Node.js and MongoDB on MongoLab: "Sockets Closed" On Every Insert - node.js

I'm trying to do something relatively simple and am running into a "server ...-a.mongolab.com:36648 sockets closed" error all of a sudden every time I try to do an "insert".
Reads seem to work without error, but inserts seem to get an error every time and I'm not sure if it's my code (which recently underwent minor changes), or a reliability problem with the free server I'm using at MongoLab (which recently showed itself to be down for a few minutes).
Oddly enough, the record itself seems to save okay, I just get the error back!
Can anyone see an issue with my code, or could this be something else?
var mongoClient = require('mongodb').MongoClient;
var http = require('http');
var connectionString = "...";
var pictureWallsCollectionName = 'PictureWalls';
//this is what barfs. see *** details
exports.saveWall = function (req, res) {
//reformat
var toSave = {
_id: req.body.wallId,
pictures: req.body.pictures
};
var status;
mongoClient.connect(connectionString, function (err, db) {
if (err) { return console.error(err); }
var collection = db.collection(pictureWallsCollectionName);
//*** no err yet... ***
collection.insert(
toSave,
function (error, response) {
//*********************
//*** err here! ******
//*********************
db.close();
if (error) {
console.error(error);
//bad
status = 500;
}
else {
console.log('Inserted into the ' + collection_name + ' collection');
//good
status = 200;
}
});
response.status(status).end(http.STATUS_CODES[status]);
});
}
//this seems to work pretty reliably. including it just in case it's relevant
exports.findByWallId = function (req, res) {
var id = req.params.id;
console.log('Retrieving wall: ' + id);
mongoClient.connect(connectionString, function (err, db) {
if (err) { return console.dir(err); }
var collection = db.collection(pictureWallsCollectionName);
collection.findOne(
{ _id: id },
function (err, item) {
db.close();
if (err) {
console.error(err);
//something bad happened
var status = 500;
res.status(status).end(http.STATUS_CODES[status]);
}
else {
console.log('Found wall with ID ' + id);
//reformat and send back in the response
res.send({
wallId: item._id,
pictures: item.pictures
});
}
}
);
});
};

EDIT: Part of my original issue was duplicate parameter names. See the linked question for detail.
ORIGINAL RESPONSE:
The issue ended up being that I was calling:
res.status(status).end(http.STATUS_CODES[status]);
...before the async insert was finished, so it barfed.
However, I'm not exactly sure how to issue the response in this case. See my new question here:
How Do I Properly Issue Response To Post When Waiting For Async Method To Complete?

Related

TypeError: validator.escape is not a function - (express-validator#6.12.1 package)

Codecademy video: link
Explanation:
As part of my Codecademy Back-End Engineer training, I have to do a project outside of their platform. The goal of this project is to make sure a node application is protected from common web attacks.
One challenge I faced was securing the code from Cross-Site Scripting (XSS) attacks. To do this, I used a package called express-validator#6.12.1. The code uses a function called validator.escape which is supposed to protect against any malicious code being inserted into an input form. However, I am getting an error in the console when I try to use it.
Terminal output :
TypeError: validator.escape is not a function
Here is the code :
const validator = require("express-validator");
app.post("/public_forum", function (request, response) {
if (request.session.loggedin) {
var comment = validator.escape(request.body.comment);
var username = request.session.username;
if (comment) {
db.all(
`INSERT INTO public_forum (username,message) VALUES ('${username}','${comment}')`,
(err, rows) => {
console.log(err);
}
);
db.all(`SELECT username,message FROM public_forum`, (err, rows) => {
console.log(rows);
console.log(err);
response.render("forum", { rows });
});
} else {
db.all(`SELECT username,message FROM public_forum`, (err, rows) => {
console.log(rows);
console.log(err);
response.render("forum", { rows });
});
}
comment = "";
} else {
response.redirect("/");
}
comment = "";
//response.end();
});
In the video of Codecademy, the guy uses this function.
Try with:
const {check, validationResult} = require('express-validator');
app.post('/public_forum', async function (request, response) {
if (request.session.loggedin) {
await check('comment').trim().escape().run(req);
const validationResult = await validationResult(req);
if (validationResult.isEmpty()) {
// Good to go...
const { comment } = req.body;
}
...
Link to official docs
I have implemented your code. I tried to add both a malicious and safe comment, but I got an error message on my browser that said, "Port 4000 Not Found." Every time I run the code, it kills the port. So I have implemented another code that works well based on what you sent me.
// This code defines a post request handler for the "/public_forum" endpoint.
app.post('/public_forum', async function (request, response) {
// Check if the user is logged in by checking the session data.
if (request.session.loggedin) {
// Trim and escape the incoming comment.
await check('comment').trim().escape().run(request);
// Get the validation result of the incoming comment.
const errors = validationResult(request);
// If the validation result contains errors, return a 400 status with the errors in a JSON format.
if (!errors.isEmpty()) {
return response.status(400).json({ errors: errors.array() });
}
// Get the comment from the request body.
const { comment } = request.body;
// If a valid comment exists, insert it into the "public_forum" database table.
if (comment) {
db.run(
`INSERT INTO public_forum (username,message) VALUES (?,?)`, [request.session.username, comment],
(err) => {
// If an error occurs while inserting the comment, log the error.
if (err) {
console.error(err);
}
}
);
}
// Select all the rows from the "public_forum" table.
db.all(`SELECT username,message FROM public_forum`, (err, rows) => {
// If an error occurs while selecting the rows, log the error.
if (err) {
console.error(err);
}
// Log the selected rows.
console.log(rows);
// Render the "forum" template, passing in the selected rows as a parameter.
response.render("forum", { rows });
});
} else {
// If the user is not logged in, redirect them to the homepage.
response.redirect("/");
}
});

Getting Error Like RequestsLimitError: You just made too many request to instagram API in node js?

I am work with isntagram api in node js. i have one array and in the array store above 20k up instagram id. and then i am do foreach on that array and one by one take instagram id and go for the take bio but that time i am getting error like this RequestsLimitError: You just made too many request to instagram API. i am try every 5 call after set time out also but still i am getting same error so how can resolved this error any one know how can fix it then please let me know.
Here this is my code =>
var InstaId = ["12345687",20k more id store here in the array]
var changesessionFlage = 0;
async.each(InstaId, function (id, callback) {
async.parallel([
function (cb) {
if (id) {
setTimeout(function () {
Client.Account.getById(sess, id).then(function (bio) {
console.log("changesessionFlage" + changesessionFlage);
changesessionFlage++
//console.log("bio : ", bio._params); // here i am getting bio one by one user
if (changesessionFlage == 6) {
changesessionFlage = 0;
}
cb(null, bio._params);
})
.catch(function (err) {
console.log("get boi: ", err)
cb(null, bio._params);
})
}, (changesessionFlage == 5) ? 10000 : 0)
}
}
], function (err, results) {
if (err) {
console.log(err);
return;
}
Result = results
callback();
});
}, function (err) {
if (err) {
console.log(err);
return;
}
else {
console.log("Result=>", Result)
if (Result) {
console.log("Result[0]=>", Result[0])
var ws = XLSX.utils.json_to_sheet(Result[0]);
var wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, "People");
var wbout = XLSX.write(wb, { bookType: 'xlsx', type: 'binary' });
res.end(wbout, 'binary');
}
}
});
any one know how can fix this issue then please help me.
Your setTimeout is use incorrectly, all API calls are made at once after 10000 delay.
Since this is a one time job, just split the 20K usernames to 4K batches and execute them every hour. This way you will be under the 5k/hr API limit

RabbitMQ request, always timesout

I have a weird problem where my callback is never published and the message goes to timeout, even though the method runs in the queue. This happens in some specific queues and after it happens once, i cannot make any other requests from client which even previously worked, they all timeout. Have to restart the client and sever to make it working again.
This is the code, where its happening, and i cant seem to understand whats wrong.
Server.js file where i am creating the queues. I have several such queues, this is one of them.
var amqp = require('amqp');
var util = require('util');
var cnn = amqp.createConnection({host:'127.0.0.1'});
var getCart = require('./services/getCart');
cnn.on('ready', function() {
cnn.queue('getCart_queue', function(q){
q.subscribe(function(message, headers, deliveryInfo, m){
// util.log(util.format( deliveryInfo.routingKey, message));
// util.log("Message: "+JSON.stringify(message));
// util.log("DeliveryInfo: "+JSON.stringify(deliveryInfo));
getCart.handle_request(message, function(err,res){
cnn.publish(m.replyTo, res, {
contentType:'application/json',
contentEncoding:'utf-8',
correlationId:m.correlationId
});
});
});
});
});
Here, the handle request function is completed successfully, but the callback never goes through and its always timeout on the other end
var cart = require('../models/cart');
function handle_request(msg, callback) {
var user_id = msg.id;
cart
.find({id:user_id})
.populate('users ads')
.exec(function(err, results){
// This works, just the callback doesnt
if(!err){
console.log(results);
callback(null, results);
} else {
console.log(err);
callback(err, null);
}
});
}
exports.handle_request = handle_request;
this is how i am calling the request
var msg_payload = {"id":id};
mq_client.make_request('getCart_queue', msg_payload, function(err, results){
console.log(results); // never prints
//stuff that is never reached
});
These are my rpc files, i dont think there should be anything wrong with these, as some other queues work fine.
And this is the error shown on client
GET /getCart - - ms - -
Error: timeout 6ee0bd2a4b2ba1d8286e068b0f674d8f
at Timeout.<anonymous> (E:\Ebay_client\rpc\amqprpc.js:32:18)
at Timeout.ontimeout [as _onTimeout] (timers.js:341:34)
at tryOnTimeout (timers.js:232:11)
at Timer.listOnTimeout (timers.js:202:5)
Hope the information is not vague, if you need more, please let me know. Thanks!
I Think the error is in this file, because i tried debugging and from the rabbitmq server, the callback is being called and it has the correlation id as well as the reply to variable, so the request is not getting picked up here.
var amqp = require('amqp')
, crypto = require('crypto');
var TIMEOUT=8000;
var CONTENT_TYPE='application/json';
var CONTENT_ENCODING='utf-8';
var self;
exports = module.exports = AmqpRpc;
function AmqpRpc(connection){
self = this;
this.connection = connection;
this.requests = {};
this.response_queue = false;
}
AmqpRpc.prototype.makeRequest = function(queue_name, content, callback){
self = this;
var correlationId = crypto.randomBytes(16).toString('hex');
var tId = setTimeout(function(corr_id){
callback(new Error("timeout " + corr_id));
delete self.requests[corr_id];
}, TIMEOUT, correlationId);
var entry = {
callback:callback,
timeout: tId
};
self.requests[correlationId]=entry;
self.setupResponseQueue(function(){
self.connection.publish(queue_name, content, {
correlationId:correlationId,
contentType:CONTENT_TYPE,
contentEncoding:CONTENT_ENCODING,
replyTo:self.response_queue});
});
};
AmqpRpc.prototype.setupResponseQueue = function(next){
if(this.response_queue) return next();
self = this;
self.connection.queue('', {exclusive:true}, function(q){
self.response_queue = q.name;
q.subscribe(function(message, headers, deliveryInfo, m){
var correlationId = m.correlationId;
if(correlationId in self.requests){
var entry = self.requests[correlationId];
clearTimeout(entry.timeout);
delete self.requests[correlationId];
entry.callback(null, message);
}
});
return next();
});
};
This is the code for your make_request() in client.js file:
var amqp = require('amqp');
var connection = amqp.createConnection({host:'127.0.0.1'});
var rpc = new (require('./amqprpc'))(connection);
function make_request(queue_name, msg_payload, callback){
rpc.makeRequest(queue_name, msg_payload, function(err, response){
if(err)
console.error(err);
else{
console.log("response", response);
callback(null, response);
}
});
}
exports.make_request = make_request;
Look at what happens when you have an err on rpc.makeRequest():
rpc.makeRequest(queue_name, msg_payload, function(err, response){
if(err)
console.error(err);
//
//HERE: should be a callback call here.
//
else{
console.log("response", response);
callback(null, response);
}
});
This could be why you are getting a timeout. I hope it helps.
There wasn't a problem with rabbitMQ but with my queries in the handle request and after responding to the request.
For others coming with this problem, check and double check every statement, as the error will not show in the console, but will only show a timeout

Trying to implement callback, or some way to wait until my get request is done. This is in node.js express

Im trying to implement some way to stop my code to redirect me before I get the response from the omdb api I am using.
My function for making a search for a movie and saving all titles in a session looks like this:
app.post('/search', isLoggedIn, function(req, res) {
function getMovies(arg, callback){
console.log('In getMovies');
console.log('searching for '+arg);
omdb.search(arg, function(err, movies) {
if(err) {
return console.error(err);
}
if(movies.length < 1) {
return console.log('No movies were found!');
}
var titles = [];
movies.forEach(function(movie) {
// If title exists in array, dont push.
if(titles.indexOf(movie.title) > -1){
console.log('skipped duplicate title of '+movie.title);
}
else{
titles.push(movie.title);
console.log('pushed '+movie.title);
}
});
// Saves the titles in a session
req.session.titles = titles;
console.log(req.session.titles);
});
// Done with the API request
callback();
}
var title = req.body.title;
getMovies(title, function() {
console.log('Done with API request, redirecting to GET SEARCH');
res.redirect('/search');
});
});
However I dont know if I implement callback in the right way, because I think there can be a problem with the api request actually executing before the callback, but not finishing before. And therefor the callback is working..
So I just want 2 things from this question. Does my callback work? And what can I do if a callback won't solve this problem?
Thankful for all answers in the right direction.
Add
callback();
To, like this
omdb.search(arg, function(err, movies) {
if (err) {
return console.error(err);
}
if (movies.length < 1) {
return console.log('No movies were found!');
}
var titles = [];
movies.forEach(function(movie) {
// If title exists in array, dont push.
if (titles.indexOf(movie.title) > -1) {
console.log('skipped duplicate title of ' + movie.title);
} else {
titles.push(movie.title);
console.log('pushed ' + movie.title);
}
});
// Saves the titles in a session
req.session.titles = titles;
callback();
});
omdb.search is asynchronous function that's why callback executed before omdb.search

Mongoose with async queue & waterfall

I aim to import large amount of data by Mongoose. As a newbie, I fail to setup the flow control properly with various mechanisms by async. Glad if someone could point to an appropriate solution. Thanks.
var async = require('async'),
mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/test');
var Cat = mongoose.model('Cat', { name: String });
// Imagine this is a huge array with a million items.
var content = ['aaa', 'bbb', 'ccc'];
var queries = [];
content.forEach(function(name) {
queries.push(function(cb) {
var obj = new Cat({ name: name });
obj.save(function(err) {
console.log("SAVED: " + name);
console.log(err);
});
return true;
});
});
// FAILED: async.parallel adds all content to db,
// but it would exhaust the resource with too many parallel tasks.
async.parallel(queries, function(err, result) {
if (err)
return console.log(err);
console.log(result);
});
// FAILED: save the first item but not the rest
async.waterfall(queries, function(err, result) {
if (err)
return console.log(err);
console.log(result);
});
// FAILED: same as async.waterfall, async.queue saves the first item only
var q = async.queue(function(name, cb) {
var obj = new Cat({ name: name });
obj.save(function(err) {
console.log("SAVED: " + name);
console.log(err);
});
})
q.push(content, function (err) {
console.log('finished processing queue');
});
I think eachLimit or eachSeries fit your situation best:
var content = ['aaa', 'bbb', 'ccc'];
async.eachLimit(content, 10, function(name, done) {
var obj = new Cat({ name : name });
obj.save(done);
// if you want to print some status info, use this instead:
//
// obj.save(function(err) {
// console.log("SAVED: " + name);
// console.log(err);
// done(err);
// });
//
}, function(err) {
// handle any errors;
});
With eachLimit, you can run an X amount of queries 'in parallel' (10 in the example above) to speed things up without exhausting resources. eachSeries will wait for the previous save before it continues with the next, so effectively saving one object at a time.
Notice that with each*, you won't get a list with (saved) objects back (it's a bit of a fire-and-forget mechanism where you're not interested in the outcome, bar any errors). If you do want a list of saved objects in the end, you can use the equivalent map* functions: mapLimit and mapSeries.

Resources