Node MongoDB memory leak issue - node.js

I'm writing a webservice using Node.js.
This webservice makes calls to a MongoDB and MSSQL.
For MSSQL i've use npm mssql library, for mongo i use the native npm mongodb library.
I use Q as my promise library.
I've a memory leak issue running a find over a MongoDB collection. I simply need to get elements from a connection. And eventually update the status of elements i get.
See my sample code below.
var Q = require('q');
var connection = require('..\connection.js'); //the connection module open a connection that can be used with pools.
function list(req, res) {
return Q.Promise(function(resolve, reject, notify) {
var collection = null;
var result = [];
var cursor = null;
Q.fcall(function(){}).then(function() {
collection = connection.collection(collectionName);
})
.then(function() {
cursor = collection.find({ fieldstatus : 0 });
return Q.Promise(function(resolve, reject, notify) {
Q.allSettled(cursor.each(function(err, item){
return Q.fcall(function(){
try {
if(item != null) {
result.push({
field1 : item.field1,
field2 : item.field2,
fieldstatus : item.fieldstatus
});
collection.update({_id: item._id}, {$set: {fieldstatus : 1}});
}
resolve(result);
} catch (err){
reject(err);
}
})
.fin(function() {
cursor.close();
});
}));
});
})
.then(function(ret) {
resolve(ret);
})
.fail(function(err) {
reject([ err.toString() ]);
})
.fin(function() {
result = null;
cursor = null;
collection = null;
});
})
.fail(function(err) {
throw([ err.toString() ]);
});
}
}
UPDATED : Answer 1
The code below seems to works without leak issuess.
var Q = require('q');
var connection = require('..\connection.js'); //the connection module open a connection that can be used with pools.
function list(req, res) {
return Q.Promise(function(resolve, reject, notify) {
var collection = null;
var result = [];
var cursor = null;
Q.fcall(function(){}).then(function() {
collection = connection.collection(collectionName);
})
.then(function() {
return Q.npost(
collection,
"find",
[
{ fieldstatus : 0 }
]
).then(function(ret){
return Q.npost(ret, "toArray").then(function(item){
return item;
});
})
.then(function(ret){
var result = [];
ret.forEach(function (item) {
result.push({
field1 : item.field1,
field2 : item.field2,
fieldstatus : item.fieldstatus
});
collection.update({_id: item._id}, {$set: {fieldstatus : 1}});
});
return result;
})
})
.then(function(ret) {
resolve(ret);
})
.fail(function(err) {
reject([ err.toString() ]);
})
.fin(function() {
collection = null;
});
})
.fail(function(err) {
throw([ err.toString() ]);
});
}
}

Related

How to await on class constructor in Node js

I have two classes as below
class AClient
class AClient {
constructor(AHeaderObj) {
logger.info("inside tenant client constructor");
this.tenant_uri = tenantHeaderObj.tenant_uri || config.TENANT_URI;
this.refresh_token = tenantHeaderObj.refresh_token;
this.tenant_header = tenantHeaderObj.tenant_header;
}
Class BClient
class BClient {
constructor(b_client) {
logger.info("Inside mongoClient/constructor ");
this.tenant_client = tenant_client;
}
async find(db_name, collection_name, selectorQuery) {
try {
logger.info("Inside mongoClient find records");
let dbName = await getDB(db_name, this.tenant_client);
let db = await getConnection(dbName, this.tenant_client);
logger.info("selectorQuery"+JSON.stringify(selectorQuery));
let doc = db.collection(collection_name).find(selectorQuery).toArray();
logger.info("****************doc**********"+JSON.stringify(doc));
return await promiseWrapper(null, doc);
} catch (err) {
logger.info("Caught error in mongoClient find records ", err);
return await promiseWrapper(err, null);
}
}
async findSome(db_name, collection_name, query, projection, sort, limit){
logger.info("Inside mongoClient findSome records");
try{
let dbName = await gettDB(db_name, this.tenant_client);
let db = await getConnection(dbName, this.tenant_client);
var selector = [{ $match : query},
{ $project : projection},
{ $sort : sort}];
if(limit > 0){
selector.push({ $limit : limit})
}
let docs = db.collection(collection_name).aggregate(selector).toArray();
return await promiseWrapper(null, docs);
} catch (err) {
logger.info("Caught error in mongoClient findSome records ", err);
return await promiseWrapper(err, null);
}
}
Now I am using it in below 2 ways . One way it works another it is not.
Case1: When it is not working
var searchAll = function (type, text, sortBy, sort, Code, AHeaderObj) {
logger.info("Inside searchAll function.");
var selector = mongoQuery.searchAll(type, text, sortBy, sort, Code);
var AObj = new AClient(AHeaderObj);
logger.info("123");
var mongoObj = new BClient(AObj);
logger.info("1235");
var findSome = callbackify(mongoObj.findSome)
logger.info("12356");
return new Promise((resolve1, reject1) => {
logger.info("123567");
findSome(dBName, collectionName, selector.query, selector.projection, selector.sort, 999999999, function (err1, res1) {
if (err1) {
logger.error("Failed to find");
reject1(err1);
} else {
logger.info("Successfully found");
resolve1(res1)
}
});
})
}
Case2: Where it is working as charm.
var getId = function (id, headerObj, callback) {
logger.info("inside getAccountById() START:");
let selector = mongoQuery.GetAccByIdQueryWithIdModelVersion("Id", id, “version", "v2.1")
let index = 0;
var AObj = new AClient(AHeaderObj);
logger.info("123");
var mongoObj = new BClient(AObj);
logger.info("1235");
mongoObj.find(dBName, collectionName, selector).then((result) => {
logger.info("123568");
if (result && result.length > 0) {
logger.info("a");
logger.info("Found an with id: " + id);
if (result[index].credentials.length > 0) {
logger.info("getById() END:"+JSON.stringify(result));
callback(null, result);
}else {
logger.info("b");
let error = {
"message": "Unable to fetch the with id: " + id
};
logger.warn(error.message);
logger.info("getById() END:");
callback(error, null);
}
}).catch((err) => {
logger.info("c");
logger.info("error" + JSON.stringify(err));
callback(err, null);
});
}
Can someone help what am I doing wrong in Case1 . I observed that ion Case 1 object creation is not awaited and db function is been called where I get below error.
Caught error in mongoClient findSome records TypeError: Cannot read property 'tenant_client' of undefined
Instead of trying to await on class initialization, make the constructor empty and provide and async init method which does the same thing as what the constructor would do.
Make sure that other methods check if the object has been instantiated via the init method and throw an error if not :)

Memory leak from node/mongoose

The below code dumps all the followers from TWITTER_USERNAME in a mongodb collection. When it's running, the memory utilization of my MongoDB database slowly increases until it crashes. I have to pause the script and restart mongo every few hours to avoid this. Is there a way for the script to get Mongo to release this memory?
var TwitterUserId = require(PATH + "/models/twitterUserId").Model;
var TwitterConnection = require(PATH + "/models/twitterConnection").Model;
util.promiseWhile = Promise.method(function(condition, action, lastValue) {
if (!condition()) return lastValue;
return action().then(promiseWhile.bind(null, condition, action));
});
var go = function(cursor) {
if(!cursor) cursor = -1;
var pollTwitter = function(cnn) {
return new Promise(function(resolve,reject) {
var parms = {
screen_name: TWITTER_USERNAME,
stringify_ids:true
cursor: cursor
};
cnn.get('followers/ids', parms,function(err,res) {
if(err) return reject(err);
cursor = res.next_cursor_str;
resolve(res.ids);
});
});
};
mongoose.connect(config.db.uri,config.db.options)
.then(function() {
return TwitterConnection.findOne({});
})
.then(function(twitterCnnCOnfig) {
return new twitterObject({
consumer_key: twitterCnnCOnfig.consumerKey
,consumer_secret: twitterCnnCOnfig.consumerSecret
,access_token_key: twitterCnnCOnfig.accessTokenKey
,access_token_secret: twitterCnnCOnfig.accessTokenSecret
});
})
.then(function(twitterCnn) {
util.promiseWhile(
function() {
return cursor;
},
function () {
return new Promise(function(outerResolve,outerReject) {
twitter.parms.cursor = cursor;
pollTwitter(twitterCnn)
.each(function(twitterId) {
return new Promise(function(resolve,reject) {
return TwitterUserId.create({twitterId: twitterId})
.then(resolve,reject);
});
})
.then(function() {
setTimeout(function() {
outerResolve();
},50 * 1000);
})
});
}
);
})
};

Completing loops inside of node.js promises

I am trying to complete a few loops over firebase objects using .forEach and I am also using promises. This isn't working out how I had planned it. My basic problem is that the loops inside of my promises complete well after the promise chain itself completes. Here is my function:
var queue = new Queue(productUpdateQueue, function(data, progress, resolve, reject) {
var incomingUpdateData = data;
var receiptID = incomingUpdateData.receiptID;
var userID = incomingUpdateData.userID;
var oldProductID = incomingUpdateData.oldProductID;
var newProductID = incomingUpdateData.newProductID;
var newReceipt = incomingUpdateData.newReceipt;
var postID = "";
var updateObject = {};
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null;
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = newReceipt;
clicks.child('VigLink').orderByChild('item').equalTo(oldProductID).once('value', function(cuidSnapshot) {
return cuidSnapshot.forEach(function(cuidSnapshot) {
var cuid = cuidSnapshot.key;
updateObject['clicks/VigLink/'+cuid+'/item'] = newProductID;
console.log('one');
progress(20);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('receipts').child(receiptID).child('items').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = data
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+newProductID] = now
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+oldProductID] = null
};
console.log('two');
progress(40);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('shops').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/shops/'+oldProductID] = null;
updateObject['userReceiptMetrics/'+userID+'/shops/'+newProductID] = data;
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+newProductID] = now;
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+oldProductID] = null;
};
console.log('three');
progress(60);
});
}).then(function() {
posts.once('value', function(postSnapshot) {
// use Promise.all and Array#map to wait for all these queries to finish
var allPosts = postSnapshot.val()
var postKeys = Object.keys(allPosts)
return Promise.all(postKeys.map(function(postKey) {
var postID = postKey;
return posts.child(postID).child('items').child(oldProductID).once('value', function(itemSnapshot) {
return itemSnapshot.forEach(function(itemSnapshot) {
var itemData = itemSnapshot.val()
console.log('post snapshot'+ itemData);
updateObject['posts/'+postID+'/items/'+oldProductID] = null
updateObject['posts/'+postID+'/items/'+newProductID] = itemData
});
});
}));
});
}).then(function() {
// Move to next item
return console.log('hey look here'+updateObject['posts/'+postID+'/items/'+newProductID]);
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
// resolve();
console.log('four');
}
});
}).then(function() {
// Move to next item
return console.log('second one'+updateObject['posts/'+postID+'/items/'+newProductID]);
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
// resolve();
console.log('four');
}
});
});
// Finish the task asynchronously
setTimeout(function() {
reject();
}, 10000);
});
And here is the output:
one
two
three
hey look hereundefined
second oneundefined
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
I think I might be using promises incorrectly but I don't know.
I figured out that I should be using Promise.all() in order to wait for my forEach loops to complete. Here is the code I used to solve my problem, enjoy:
var queue = new Queue(productUpdateQueue, function(data, progress, resolve, reject) {
var incomingUpdateData = data;
var receiptID = incomingUpdateData.receiptID;
var userID = incomingUpdateData.userID;
var oldProductID = incomingUpdateData.oldProductID;
var newProductID = incomingUpdateData.newProductID;
var newReceipt = incomingUpdateData.newReceipt;
var postID = "-KZOO0UII67uOmYo6DJh";
var postKeys = [];
var updateObject = {};
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null;
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = newReceipt;
return clicks.child('VigLink').orderByChild('item').equalTo(oldProductID).once('value', function(cuidSnapshot) {
return cuidSnapshot.forEach(function(cuidSnapshot) {
var cuid = cuidSnapshot.key;
updateObject['clicks/VigLink/'+cuid+'/item'] = newProductID;
progress(10);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('receipts').child(receiptID).child('items').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = data
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+newProductID] = now
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+oldProductID] = null
};
progress(25);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('shops').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/shops/'+oldProductID] = null;
updateObject['userReceiptMetrics/'+userID+'/shops/'+newProductID] = data;
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+newProductID] = now;
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+oldProductID] = null;
};
progress(40);
});
}).then(function() {
progress(55);
return posts.orderByChild('receipt').equalTo(receiptID).once('value');
}).then(function(postSnapshot) {
return postSnapshot.forEach(function(post) {
progress(70);
postKeys.push(post.key)
});
}).then(function() {
return Promise.all(postKeys.map(function(postKey) {
return posts.child(postKey).child('items').child(oldProductID).once('value', function(itemSnapshot) {
var itemData = itemSnapshot.val()
updateObject['posts/'+postKey+'/items/'+oldProductID] = null;
updateObject['posts/'+postKey+'/items/'+newProductID] = itemData;
});
})).then(function(results) {
progress(85);
return results;
});
}).then(function() {
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
resolve();
}
});
});
// Finish the task asynchronously
setTimeout(function() {
reject();
}, 10000);
});

Trouble in using forEach function in node.js

I have been learning about q promises and tried to build up some mock APIs to implement its functionality,While doing so I came across the following error,
Enterprise.forEach is not a function
My API code is as follows,
var mongoose = require('mongoose');
var Enterprise = mongoose.model('Enterprise_gpy');
var q = require('q');
var displayEnterprise = function(req, res) {
function displayEnterpriseName() {
var deferred = q.defer();
Enterprise.forEach(function(err, doc) {
if (err) {
console.log('Error Finding Files');
deferred.reject(err);
} else {
var name = Enterprise.enterprise_name;
deferred.resolve({
name: name
});
}
return deferred.promise;
});
}
function displayEnterpriseEmail() {
var deferred = q.defer();
Enterprise.forEach(function(err, doc) {
if (err) {
console.log('Error Finding Files');
deferred.reject(err);
} else {
var email = Enterprise.enterprise_email;
deferred.resolve({
email: email
});
}
return deferred.promise;
});
}
q.all([
displayEnterpriseName(),
displayEnterpriseEmail()
])
.then(function(success) {
console.log(500, success);
})
.fail(function(err) {
console.log(200, err);
});
}
module.exports = {
displayEnterprise: displayEnterprise
}
In your code Enterprise is a mongoose schema so when you try to do loop using forEach then got
Enterprise.forEach is not a function
you can use forEach after Enterprise.find(). so use
Enterprise.find({}, function(err, docs) {
if (err) {
console.log('Error Finding Files');
deferred.reject(err);
} else {
var names = [];
docs.forEach (function(doc) {
var name = doc.enterprise_name;
names.push(name);// pushed in names array
//.....
});
deferred.resolve({
names: names
}); // return all names
}
});
instead of
Enterprise.find().forEach
and should use
var name = doc.enterprise_name; instead of var name = Enterprise.enterprise_name;
and
var email = doc.enterprise_email; instead of var email = Enterprise.enterprise_email;
forEach only works for arrays, and you're using it on a mongoose model.
try this instead:
Enterprise.find().exec(function(err, docs) {
docs.forEach(function(doc) {
// do something with all the documents
}
// do something outside the loop
})

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Resources