nodejs mongodb timesSeries sometimes hangs without errors - node.js

Im running a script that has multiple queries using timesSeries loop.
sometimes it hangs for no reason or errors. first it usese count, to
determine how many loops for the pages. then it loops to lessen results.
then it async loops to build the document to send to google bigquery.
sometimes it just stops, then I have to kill script manually. should I close the count first also?
mongo_client.connect(mongo_url, function(err, db) {
var query = { "date_visited" : {$gt: new Date("2016-01-01T00:00:00.000Z"),$lt: new Date("2016-02-01T00:00:00.000Z")},_id:{$gte: ObjectId("56aabfe8c6edfd0d608b480e")} };
db.collection(collection_name).count(query,function(err, count){
var pages = Math.ceil(count/limit);
//var pages = 1;
console.log("count: "+count);
console.log("pages: "+pages);
var page = 0
async.timesSeries(pages, function(page, next)
{
var skip = page * limit;
var current_page = page+1;
console.log("skip: "+skip);
console.log("page: "+page);
console.log("current_page: "+current_page+" of "+pages);
var documents = [];
var x=1;
db.collection(collection_name).find(query).limit(limit).skip(skip).toArray(function(err, db_results) {
async.each(db_results, function (db_resultsx, cb) {
var account_id = db_resultsx.account_id;
delete db_resultsx.upsell_detect;
db_resultsx._id = db_resultsx._id.toString();
db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format("YYYY-MM-DD hh:mm:ss");
documents.push(db_resultsx);
//console.log(documents);
if(documents.length == db_results.length) {
bigqueryClient
.dataset(dataset)
.table(collection_name)
.insert(documents)
.then((insertErrors) => {
console.log("Inserted: "+page);
//console.error("Inserted: "+page);
//console.error(insertErrors);
next();
if (insertErrors && insertErrors.length > 0) {
console.log("start errors: "+page+"("+insertErrors.length+")");
insertErrors.forEach((err) => console.log(JSON.stringify(err, null, 4)));
console.log("done errors: "+page);
console.log("done on: "+moment().format("YYYY-MM-DD hh:mm:ss"));
}
})
.catch((err) => {
console.log('ERROR:');
console.log(JSON.stringify(err, null, 4));
console.log('done catch:'+page);
process.exit();
});
}
});
});
},
function(err, coll)
{
console.log("done all");
console.log(query);
setTimeout(function () {
db.close();
console.error("DONE");
process.exit();
}, 60000);
});
});
});

Related

NodeJS Promise and Async Problems (Firebase)

The problem is with the promises and the async function. "All moved" is supposed to be logged after everything in async.each is done. But nothing is ever logged.
Here is my exports functions:
var courier_id = data.ref.parent.key;
return admin.database().ref("firewall_queue/"+courier_id+"/orders").once('value',function(orders){
//console.log(Object.keys(orders.val()));
async.each(Object.keys(orders.val()), function (order, callback) {
if(order != "none") {
return moveToWaitingFromFirewall(order).then(callback())
}
},
function (err) {
console.log("All moved");
return admin.database().ref("/firewall_queue/"+courier_id+"/orders/").remove().then(()=>{
return pushToPending(courier_id,data.ref.key);
})
});
})
Here is my moveToWaitingFromFirewall function:
function moveToWaitingFromFirewall(order_id){
var order = {};
order.id = order_id;
var promises = [];
promises.push(new Promise((resolve) => {
admin.database().ref("orders/"+order_id+"/zone").once('value').then(function(zone){
order.zone = zone.val();
resolve();
})
}))
promises.push(new Promise((resolve) => {
admin.database().ref("orders/"+order_id+"/time_order_placed").once('value').then(function(time_order_placed){
order.time = time_order_placed.val();
resolve();
})
}))
//grab zone and time first
return Promise.all(promises).then(()=>{
return admin.database().ref(order.zone+"/wait_order_queue/"+order.id).set(order.time);
})
}
JSON Firebase
"c98" : {
"orders" : {
"0333" : 123123,
"0345" : 12,
"0911" : 123,
"none" : "none"
}
Study this a little bit, and maybe apply to your current code.
Imagine admin.database().ref("orders/"+order_id+"/time_order_placed").once('value') is like delay(time)
// let delay = time => new Promise(res=>setTimeout(res,time));
let delay = function(time){
return new Promise(function(resolve,reject){
setTimeout(function(){
resolve();
},time);
});
}
let myPromise = function(order){
return Promise.all([
delay(500),
delay(500),
delay(1000).then(function(){
console.log('Order complete: ',order);
return; // returns undefined, so cb doesn't pass anything to cb(err), but use ()=>cb() to avoid this anyways.
})
]);
}
let orders = [1,2,3];
async.each(orders,function(order,cb){
myPromise(order)
.then(()=>cb())
.catch(err=>cb(err));
},function(err,data){
if(err){
console.log('Err',err);
}else{
console.log('all Finished');
}
});
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/2.6.0/async.js"></script>

Unable to retrive data and push inside loop in node js

I am trying to retrieve attendance list along with user details.
I am using caminte.js(http://www.camintejs.com/) Cross-db ORM for database interaction.
Here is my code sample of model function "attendanceList".
exports.attendanceList = function (req, callback) {
var query = req.query;
var searchfilters = {};
if(!req.user){
callback({ code:400, status:'error', message: 'Invalid Request', data:{}});
}else{
searchfilters["vendor_id"] = parseInt(req.user._id);
}
if(query.location && parseString(query.location) != '') {
searchfilters["location"] = parseString(query.location);
}
if (query.device_details && parseString(query.device_details) != '') {
searchfilters["device_details"] = parseString(query.device_details);
}
if(query.created_on) {
searchfilters["created_on"] = query.created_on;
}
if(query.status) {
searchfilters["status"] = { regex: new RegExp(query.status.toLowerCase(), "i") };
}
var SkipRecord = 0;
var PageSize = 10;
var LimitRecord = PageSize;
var PageIndex = 1;
if(query.pagesize) {
PageSize = parseInt(query.pagesize);
}
if(query.pageindex) {
PageIndex = parseInt(query.pageindex);
}
if (PageIndex > 1) {
SkipRecord = (PageIndex - 1) * PageSize;
}
LimitRecord = PageSize;
var SortRecord = "created_on";
if(query.sortby && query.sorttype) {
var sortingBy = query.sortby;
var sortingType = 'ASC';
if(typeof query.sorttype !== 'undefined') {
sortingType = query.sorttype;
}
SortRecord = sortingBy + ' ' + sortingType;
}
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
};
I am getting only attendance list without user details. How do I force to push user details into attendance list? Any Help!!
Thank You
This behavior is asynchronous. When you're making request to DB, your code keeps running, while task to get data comes to task queue.
To keep things simple, you need to use promises while handling asynchronous jobs.
Rewrite your code from this:
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
To this:
const findAttendanceFirst = (searchFilters, SortRecord, LimitRecord, SkipRecord) => {
return new Promise((resolve, reject) => {
Attendance.find({ where: searchFilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, (err, result) => {
if(err) return reject(err);
resolve(result);
});
});
}
const findUserByIdForUserDetails = (userId) => {
return new Promise((resolve, reject) => {
User.findById(parseInt(userId), function(err, data){
if(err) return reject(err);
resolve(data);
})
});
}
const getAttendanceCount = (searchFilters) => {
return new Promise((resolve, reject) => {
Attendance.count({ where: searchFilters }, (err, count) => {
if(err) return reject(err);
resolve(count);
});
})
}
So, now we can use this separate functions to make async behavior looks like sync.
try {
const data = await findAttendanceFirst(searchFilters, SortRecord, LimitRecord, SkipRecord);
for(let userData of data){
try {
userData.userDetails = await findUserByIdForUserDetails(userData.user_id);
} catch(e) {
// Some error happened, so no user details.
// you can set here null or nothing to userDetails.
}
}
let count;
try {
count = await getAttendanceCount(searchFilters);
} catch(e){
// Same as before.
}
const callBackData = { code:200, status:'success', message:'OK', total:count, data:result };
// And here you can do whatever you want with callback data. Send to client etc.
} catch(e) {
}
NB: I've not tested this code, it will be easier for yu to play with your actual data and use Promises and async/await
Just remember that each request to db is asynchronous, and you need to make your code wait for this data.

Memory leak from node/mongoose

The below code dumps all the followers from TWITTER_USERNAME in a mongodb collection. When it's running, the memory utilization of my MongoDB database slowly increases until it crashes. I have to pause the script and restart mongo every few hours to avoid this. Is there a way for the script to get Mongo to release this memory?
var TwitterUserId = require(PATH + "/models/twitterUserId").Model;
var TwitterConnection = require(PATH + "/models/twitterConnection").Model;
util.promiseWhile = Promise.method(function(condition, action, lastValue) {
if (!condition()) return lastValue;
return action().then(promiseWhile.bind(null, condition, action));
});
var go = function(cursor) {
if(!cursor) cursor = -1;
var pollTwitter = function(cnn) {
return new Promise(function(resolve,reject) {
var parms = {
screen_name: TWITTER_USERNAME,
stringify_ids:true
cursor: cursor
};
cnn.get('followers/ids', parms,function(err,res) {
if(err) return reject(err);
cursor = res.next_cursor_str;
resolve(res.ids);
});
});
};
mongoose.connect(config.db.uri,config.db.options)
.then(function() {
return TwitterConnection.findOne({});
})
.then(function(twitterCnnCOnfig) {
return new twitterObject({
consumer_key: twitterCnnCOnfig.consumerKey
,consumer_secret: twitterCnnCOnfig.consumerSecret
,access_token_key: twitterCnnCOnfig.accessTokenKey
,access_token_secret: twitterCnnCOnfig.accessTokenSecret
});
})
.then(function(twitterCnn) {
util.promiseWhile(
function() {
return cursor;
},
function () {
return new Promise(function(outerResolve,outerReject) {
twitter.parms.cursor = cursor;
pollTwitter(twitterCnn)
.each(function(twitterId) {
return new Promise(function(resolve,reject) {
return TwitterUserId.create({twitterId: twitterId})
.then(resolve,reject);
});
})
.then(function() {
setTimeout(function() {
outerResolve();
},50 * 1000);
})
});
}
);
})
};

How to implement nested query in sqlite3

So i have this 2-layer query in node.js, each query could return multiple results. My code actually just ignores that for now. This is the best i can get, it seems working.
How to correct it please, i don't know how to callback for the 2nd one.
Also the db.close() is always called before the 2nd query finishes, even i have serialize().
var getInfo1Db = function(callback) {
var db = new sqlite3.Database("DB.sqlite3");
var cnt = 0;
var info1JsonObj = [];
db.all("select * from Info1DB",
function(err, rows) {
db.serialize(function() {
for(var ii=0, len=rows.length; ii<len; ii++) {
var t2 = rows[ii].info1;
var doorId = ...
db.all("select * from DoorDB where ObjectID=" + doorId,
function(err, row2) {
if(err) {
} else {
var doorName = row2[0]...
var info1JsonElem = {
"DoorName" : doorName
};
info1JsonObj.push(info1JsonElem);
cnt++;
if(cnt === rows.length) {
callback(null, info1JsonObj);
}
}
}
); // for the only door info based on door id
} // for each row of info1
db.close(); // why this finishes before the 2nd db.all
} ); // end of serialize
});
};
You can't implement nested query in sqlite3's normal way. ( I mean you even can't do it in the callback hell way, because the sqlite3 need to close the connection before another query called. otherwise you will always got error)
You have to use Promise, async and await to do this.
( it's worth to spend 30 minutes to learn these 3 words )
Step1. define a async function like this:
async query_1() {
new Promise(resolve => {
db = ...
db.serialize( () => {
db.get('select .. from ... where id = 1', [], (error, row) => {
// here is the KEY: put the result into resolve
// this equals to the "return" statement in non-sync method.
resolve(row)
}
})
db.close()
})
}
and also implement your query_2 function like this:
async query_2() {
let query_1_result = await this.query_1()
db = ...
db.serialize( () => {
db.get('select .. from ... where dependency_id = ' + query_1_result, [], (error, row) => {
// other code here...
}
})
db.close()
}
refer to my answer: https://stackoverflow.com/a/67881159/445908
How about using 2 function to do these ?
function db_query1(your_param,...., callback){
  // database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
function db_query2(your_param,...., callback){
  // database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
And call these function:
db_query1(....,function(err,result1){
if(err) ...// return
// do the things with result1
// And then call query2
db_query2(....,function(err,result2){
if(err) ...// return
// do the things with result1
});
});
Hope this will help :)
You can use Promises.all, an array and the second callback for node sqlite3 db.each() that is executed when all rows have been fetched. Node Sqlite3 db.each usage to simplify the nested query and
I cannot really get the meaning of the variables you are using thus I assume that each row in Info1DB has a one-to-many relationship with DoorDB on the field doorId.
async function getInfo (callback) {
sql = "select * from Info1DB;";
numVersions = 0;
countVersions = 0;
info1JsonObj = [];
db.serialize(function() {
db.each(sql, [], (err, info1Row) => {
sql = "select * from DoorDB where ObjectID=?;";
info1Row.doors = [];
doorId = ...
db.each(sql, [doorId], (err, doorRow) => {
info1Row.doors.push(new Promise((resolve, reject) => {
if (err) {
reject(err);
} else {
resolve(doorRow);
}
}));
}, (err, num) => {
Promise.all(info1Row.doors)
.then((doors) => {
info1Row.doors = doors;
info1JsonObj.push(info1Row);
countVersions++;
if (countVersions == numVersions) {
callback(null, info1JsonObj);
}
}).catch((err) => {
callback(err, null);
});
});
}, (err, versions) => {
numVersions = versions;
});
});
}

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Resources