nodejs mongodb async.Each done - node.js

How to detect when async.Each is done loop all objects/arrays loaded into the parameters. how to call the ALL FINISH log.
db.collection('tests').find(query).limit(10).toArray(function(err, db_results) {
console.log("count: "+db_results.length);
async.each(db_results, function (db_resultsx, cb) {
db_resultsx._id = db_resultsx._id.toString();
db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format("YYYY-MM-DD hh:mm:ss");
documents.push(db_resultsx);
}, function(documents) {
console.log(documents);
console.log(documents.length);
console.log("ALL FINISH");
process.exit();
});
});

The reason that the async.each function is not finishing is that the callback isn't being called within the second function. I've created a snipped with a simple example without the mongo call but with a timeout instead.
If you look at the docs https://caolan.github.io/async/docs.html#each the second argument is a callback (in your code this is the value cb) that needs to be called to signify the end of the code you are running.
function delayedReturn(callback) {
setTimeout(function(){
console.log('timeout complete');
callback(false, ['first value']);
}, 1000);
}
var documents = [];
delayedReturn(function(err, db_results) {
async.each(db_results, function (dbResult, cb) {
documents.push(dbResult);
console.log('documents pushed', dbResult)
cb();
}, function(error) {
if(error)
{
console.error(error);
}
});
});
<script src="https://rawgit.com/caolan/async/master/dist/async.min.js"></script>

There's no reason to use async.each() given that the code that it should be running isn't asynchronous.
Instead, you can use .map():
db.collection("tests").find(query).limit(10).toArray(function(err, db_results) {
console.log("count: " + db_results.length);
let documents = db_results.map(function(db_resultsx) {
db_resultsx._id = db_resultsx._id.toString();
db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format(
"YYYY-MM-DD hh:mm:ss"
);
return db_resultsx;
});
console.log(documents);
console.log(documents.length);
curl(URL, { data : documents }, function(err) {
console.log("ALL FINISH");
process.exit();
});
});

Related

Async series is not calling properly

I have 3 function has to call in series one after the other. but first function is exeucting and in between second and third are executing.
var tasklist=[api_hit,delay,mysql_check];
if(task_list.length>0){
async.series(
tasklist,
function(err, response) {
console.log(err);
console.log(response);
results.data=response;
results.message="Completed";
console.log(results);
}
);
}
Internal functions:
function api_hit(callback){
console.log("Inside api");
var ele=task_list[0];
var apidata=[];
var msg={'data':[]};
apiinfo.forEach((item,key)=>{
if(item.Method_name==ele.Parameters){
//Here checking random Int value
if(item.Value=="{{$randomInt}}"){
item.Value = generate(25);
}
apidata.push(item);
}
});
var data=[];
data['api']=apidata;
apiModel.validateAPI(data,function(res){
console.log("result api");
msg.data=res;
msg.case='api_hit';
callback(msg);
});
}
function delay(callback){
console.log("Inside delay");
var msg={'data':[]};
global_vars.sleep(1000);
msg.data='success';
msg.case='task';
console.log("after delay");
callback(msg);
}
function mysql_check(callback){
console.log("inside mysql");
var ele=task_list[2];
var dbdata=[];
var msg={'data':[]};
dbchecks.forEach((item,key)=>{
if(item.query_id==ele.Parameters){
console.log(item.query+" ::: "+ele.Parameters);
dbdata.push(item);
}
});
data['dbdata']=dbdata;
apiModel.checkmysql(data,function(err,res){
if(err) throw err;
console.log("inside mysql res");
msg.data=res;
msg.case='task2';
callback(msg);
});
}
My intention is to call these function after completing of others and all the results has to process in a single variable. but in api_hit method when it is executing another function inside of it then delay()(second function of async) is executing. how to stop this and make it in sequence. thanks in advance.
The first argument to the callback function is the error, pass null in case of success.
'use strict'
const async = require('async')
function api_hit(callback) {
setTimeout(() => {
console.log('Completed api_hit')
callback(null, 'api_hit')
}, 1000)
}
function delay(callback) {
setTimeout(() => {
console.log('Completed delay')
callback(null, 'delay')
}, 100)
}
function mysql_check(callback) {
setTimeout(() => {
console.log('Completed mysql_check')
callback(null, 'mysql_check')
}, 500)
}
var tasklist = [api_hit, delay, mysql_check];
if (tasklist.length > 0) {
async.series(
tasklist,
function (err, response) {
console.log(err);
console.log(response);
}
);
}
Doc link: https://caolan.github.io/async/docs.html#series

Hard time uderstaning async.parallel in node

I have this function in the controller
router.post('/', function(req, res, next) {
if (req.user.isPremium == false) {
// Free user - Single report
let website = req.body.website0;
let builtWithCall = `https://api.builtwith.com/free1/api.json?KEY=APIKEY&LOOKUP=${website}`;
let pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=APIKEY`;
// curl 'https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://georgiancollege.ca&strategy=mobile&key=APIKEY'
var calls = [];
calls.push(function(callback) {
// First call
https.get(builtWithCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log('BuiltWith received', data);
});
});
});
calls.push(function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
});
} else {
// Premium user - comparison report
let websites = [];
}
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
});
I am trying to run several async API calls at once. The problem is, when I try to run them like this
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
the console doesn't log anything.
When I do the console log here though
function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
}
it logs the response. The pageSpeed one gets in a weird loop and repeats itself multiple times, but at least it works.
Now what am I doing wrong with the async.parallel part? Also what is this callback in function(callback) {?
EDIT:
This is the new version of the anonymous function:
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end', function(data) {
callback(null, data);
});
resource.on('error', function(err) {
callback(err);
});
});
}
You need to call the passed in callback. Looking at your one parallel function you are not calling callback(). I'll assume your resource object has an end & error
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end' function() {
callback(null, results);
});
resource.on('error' function(err) {
callback(err);
});
});
}
How async.parallel works is all the functions called must in turn call the passed in callback function; in your case that is callback.
Once each function in the parallel calls callback then and only then will the final function be called, which is the function you defined as function(err, results) {...}.
There is one caveat, if in the callback call you pass non-null for the first argument then that final function will be called immediately where you should handle that error if it happens.

Promises not working as expected with async.each

I'm somehow not able to find the solution. Here is what I'm doing:
async.each(bottles, function(bottle) {
// set substance quantity to 0
updateQuantity(bottle[constant.SUBSTANCE_ID], 0)
.then(function() {
bottle[constant.EXPIRY] = expiry;
bottle[constant.IS_AVAILABLE] = true;
updateRecord(bottle)
.then(function() {
console.log('Updated');
},
function() {});
}, function() {});
}, function() {
console.log('Finished');
});
console.log('Done');
The method updateQuantity and updateRecord returns promises and are using Knex update in the background which also returns a promise.
I want the output of the above code to be:
Updated
Updated
Finished
Done
But the output I'm getting is:
Done
Updated
Updated
So, the callback of async.each is not working and the code is not waiting for async.each to finish.
You have to call the passed in callback from the async.each operation. As it is, async doesnt know when your callback resolves.
Also, DONE should always be the first text you see as async.each doesn't block. When you start running running the script, async.each will be registered, but because your code is asynchronous it will reach the console.log('Done') before the updateRecord and updateQuantities complete
async.each(bottles, function(bottle, callback) {
updateQuantity(bottle[constant.SUBSTANCE_ID], 0)
.then(function() {
...
updateRecord(bottle)
.then(function() {
console.log('Updated');
callback();
},
function() {});
}, function() {});
}, function() {
console.log('Finished');
});
console.log('Done');
Don't use async.js at all with promises, they are capable of what you need with their builtin methods and you don't need to fall back to callbacks.
bottles.reduce(function(prev, bottle) {
return prev.then(function() {
// set substance quantity to 0
return updateQuantity(bottle[constant.SUBSTANCE_ID], 0)
}).then(function() {
bottle[constant.EXPIRY] = expiry;
bottle[constant.IS_AVAILABLE] = true;
return updateRecord(bottle);
}).then(function() {
console.log('Updated');
});
}, Promise.resolve()).then(function() {
console.log('Finished successfully');
}, function(err) {
console.error(err)
}).then(function() {
console.log('Done'); // notice you must put this in the callback
});
console.log('Started');

nodejs loop async callback function

I am running a cron job with node with mongodb as the database. I am trying to close db connection and exit the process once the curr_1 each loop has executed completely.
However the exit() is called while function_2 is being executed. I understand this is due to the callback and is async in nature.
How do I make sure exit is called only once the curr_1.each is complete?
Any solution without promises?
function function_1(obj){
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
function_2(doc)
});
exit(obj)
}
function function_2(obj) {
coll_2.findOne({}, function(err, document) {
dosomeprocess(obj)
})
}
function exit(obj) {
// Close connection
console.log('closing connection')
obj.db.close();
process.exit();
}
It's a job for Node async....
For example:
async.each(
curr_1, // the collection to iterate over
function(doc, callback) { // the function, which is passed each
// document of the collection, and a
// callback to call when doc handling
// is complete (or an error occurs)
function_2(doc);
},
function(err) { // callback called when all iteratee functions
// have finished, or an error occurs
if (err) {
// handle errors...
}
exit(obj); // called when all documents have been processed
}
);
Without using any library:
function function_1(obj, callback) {
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
callback(err, doc);
});
}
function function_2(err, obj) {
coll_2.findOne({}, function(err, document) {
dosomeprocess(obj)
exit(err, obj);
})
}
function exit(err, obj) {
// Close connection
console.log('closing connection')
obj.db.close();
process.exit();
}
function_1(obj, function_2);
Using async module
var async = require('async');
async.waterfall([
function function_1(callback) {
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
if (err) {
callback(err, null)
} else {
allback(null, doc)
}
});
},
function function_2(obj, callback) {
coll_2.findOne({}, function(err, document) {
if (err) {
callback(err, null);
} else {
dosomeprocess(obj)
callback(null, obj);
}
})
}
], function done() {
obj.db.close();
process.exit();
});
Simply give a condition in your loop using counter.
function function_1(obj){
var curr_1 = coll_1.find({})
var curr_1Length = curr_1.length;
var counter = 0;
curr_1.each(function(err, doc) {
++counter;
//Check condition everytime for the last occurance of loop
if(counter == curr_1Length - 1){
exit(obj)
}
function_2(doc)
});
}
Hope it helps :)

nodejs async execute callbacks in series

I have a situation like:
function smth(data) {
// save data to db.
Object.findOne({ _id: ceva['id']}, function(obj) {
obj.save();
});
}
This function is called from various async calls. There is a race condition where the second findOne call runs before a previous save() runs.
Is there a way to work around this? Maybe using the async library to run things in series?
You can make use one of async control flows to ensure every iteration of smth() happens in order.
If you're not in favor of using a flow control library, you can easily achieve series execution of each event. Consider following code snippet:
function smth(data, cb) {
// save data to db.
findOne({
id: data.id
}, function (err, obj) {
if (!err && obj) {
savedb(obj, cb);
} else {
cb(err);
}
});
}
function findOne(filter, cb) {
// simulate find
setTimeout(function () {
cb(null, {
id: filter.id,
name: 'test',
role: 'test'
});
}, 500);
}
function savedb(obj, cb) {
//simulate db save
setTimeout(function () {
cb(null, obj);
}, 500);
}
// iterations count
var count = parseInt(process.argv[2], 10) || 3;
(function iterate(i) {
console.log(i);
if (i === count) {
// iterations complete
process.exit(1);
}
var data = {
id: 123 + i
};
smth(data, function (err, res) {
console.log(err || res);
iterate(++i);
});
})(0);
//make this follow async conventions with callback argument last
function smth(data, callback) {
//pseudocode database API here
db.save(data, function (error) {
if (error) {
callback(error);
return;
}
Object.findOne({ _id: ceva['id']}, function(obj) {
obj.save(callback);
});
});
}
That's the basic callback approach. You can use async.js if you like to clean it up a bit or study callbackhell.com for more ways to avoid the nested functions.

Resources