Async waterfall with mongo db - node.js

Probably just a basic concept of async. I'd like to check on how we can pass the array variable from mongo result(docs) to the second function. In below code the second console.log doesn't give any output.
// Retrieve
var db = require('monk')('localhost/db');
var async = require('async');
console.log('start');
async.waterfall([
function(callback) {
var test = db.get('test');
test.find({}, function(err, docs) {
console.log(docs); //OUTPUT OK
callback(docs);
});
},
function(docs, callback) {
console.log(docs); //NO OUTPUT
}
])

To give you a better understanding of the using multiple callbacks with the async module, let's illustrate this with an example from Seven Things You Should Stop Doing with Node.js of multiple operations with callbacks to find a parent entity, then find child entities that belong to the parent:
methodA(function(a){
methodB(function(b){
methodC(function(c){
methodD(function(d){
// Final callback code
})
})
})
})
With the async module, you can either use the series method to address the use of callbacks for nesting code of multiple methods which may result in Callback Hell:
Series:
async.series([
function(callback){
// code a
callback(null, 'a')
},
function(callback){
// code b
callback(null, 'b')
},
function(callback){
// code c
callback(null, 'c')
},
function(callback){
// code d
callback(null, 'd')
}],
// optional callback
function(err, results){
// results is ['a', 'b', 'c', 'd']
// final callback code
}
)
Or the waterfall:
async.waterfall([
function(callback){
// code a
callback(null, 'a', 'b')
},
function(arg1, arg2, callback){
// arg1 is equals 'a' and arg2 is 'b'
// Code c
callback(null, 'c')
},
function(arg1, callback){
// arg1 is 'c'
// code d
callback(null, 'd');
}], function (err, result) {
// result is 'd'
}
)
Now going back to your example, using the async waterfall method you could restructure your code to
async.waterfall([
function(callback) {
// code a
var test = db.get('test');
test.find({}, function(err, docs) {
if(err) callback(err);
console.log(docs); // OUTPUT OK
callback(null, docs);
});
}], function(err, result) {
if(err) callback(err);
console.log(result); // OUTPUT OK
}
])

You must always handle your errors. Try this!
// Retrieve
var db = require('monk')('localhost/db');
var async = require('async');
console.log('start');
async.waterfall([
function(callback) {
var test = db.get('test');
test.find({}, function(err, docs) {
if(err){
console.log(err);
callback(err);
}
else if(docs){
console.log(docs);
callback(null,docs);
}
else{
console.log("No error no docs");
callback(null);
}
});
},
function(err, docs) {
if(err)
console.log(err);
else{
console.log(docs);
}
}
])

Related

Nodejs send waterfall requests

I have dynamic array with foldername and creating a zip using zipFolder
I am facing issues as the library is async i.e
zipFolder(source,dest,callback)
Because its a async call it take times and I want to block the next request unless the first one reponse is true,
function (req, res)
var folder = ['N','PP','Y'];//dynamic just an example
for( var i = 0 ; i < folder.length ; i++) {
var source = path.join(basePath,'/download/'+folder[i]);
var dest = path.join(basePath,'/createZip/'+folder[i]+'.zip');
console.log('sending',folders[i])
//this is async
zipFolder(source,dest, function(err) {
if(err) {
console.log('oh no!', err);
} else {
console.log('EXCELLENT');//want to send the second,third,fourth if success
}
});
}
Summary:
output should be like this:
sending file 1 if success/response sending file 2 if success/response
want something like waterfall wait for callback reponse to send the second file
plus I want a method to know the done call back so I can send back the reponse
you can use each as like:
var each = require('async-each-series')
var folder = ['N','PP','Y'];
each(folder, function(items,callback){
//insert each items to db in async way
var source = path.join(basePath,'/download/'+items);
var dest = path.join(basePath,'/createZip/'+items+'.zip');
zipFolder(source,dest, function(err) {
if(err) {
console.log('oh no!', err);
} else {
console.log(items,'inserted')
callback()
}
});
},function(err){
if(err) console.log('error occured')
console.log('all items uploaded');
});
Try with async waterfall method
https://caolan.github.io/async/docs.html#waterfall
Runs the tasks array of functions in series, each passing their
results to the next in the array. However, if any of the tasks pass an
error to their own callback, the next function is not executed, and
the main callback is immediately called with the error
async.waterfall([
function(callback) {
callback(null, 'one', 'two');
},
function(arg1, arg2, callback) {
// arg1 now equals 'one' and arg2 now equals 'two'
callback(null, 'three');
},
function(arg1, callback) {
// arg1 now equals 'three'
callback(null, 'done');
}
], function (err, result) {
// result now equals 'done'
});
UPDATE
in your usecase eachSeries is more applicable.
var folder = ['N','PP','Y'];
async.eachSeries(folder, function(item,callback){
var source = path.join(basePath,'/download/'+item);
var dest = path.join(basePath,'/createZip/'+item+'.zip');
zipFolder(source,dest, function(err) {
if(err) {
console.log('oh no!', err);
callback();
} else {
console.log('EXCELLENT');//want to send the second,third,fourth if success
callback();
}
});
},function(){
console.log('all items uploaded');
}); .

Error handling in async.parallel

I have a function which uses async.parallel function to call two functions. But I don't know how to handle errors when async.parallel is used. If one of the two functions throw errors, how do I handle them?
exports.getProductconfigWithProductList = function(req, res){
var apiVersion = req.param('version', null);
var product_id = req.params.productId;
dbDriver = determineDriver('es');
async.parallel([
function(callback) {
dbDriver.getProductconfig(product_id,function(data){
callback(null, data);
});
},
function(callback) {
var productListPromise = getProductListData();
productListPromise.then(function(data) {
callback(null, data);
});
}
],
function(err, results) {
if(!err){
var data = results[0];
data[1] = results[1];
res.send(data);
res.end();
} else {
console.log('<<<<<<<<<<<<<<'+err+'>>>>>>>>>>>>>>>');
res.send(err);
res.end();
}
}
);
};
When you have:
async.parallel([
func1,
func2,
], (err, data) => {
if (err) {
// you handle the errors here
}
});
It's explained in more detail in the docs:
https://caolan.github.io/async/docs.html
and in the issue on GitHub here:
https://github.com/caolan/async/issues/334

Nodejs Async Waterfall, another flow control?

I'm using async.waterfall, and I want to do some like this:
keeping the result of async.waterfall by returning something likes Promise. So that, I can split those two parts.
function Action1(){
return async.waterfall([
function step1(callback){
...
callback(null, data);
},
function step2(data, callback){
...
callback(null, data);
},
function step3(data, callback){
...
callback(null, data);
}
]);
}
Action1().then(function(result){
// obtain the final result here
});
However, what I can find is: the final callback can only be injected into the waterfall function as an argument.
async.waterfall([
function(callback) {
...
callback(null, data);
},
function(data, callback) {
...
callback(null, data);
},
function(data, callback) {
...
callback(null, data);
}
], function (err, result) {
// final callback
...
});
Is there any suggestion for me, to achieve the expected programming structure? Or any choices other than async allow this pattern?
You should probably go for Bluebird.
The async code like this:
async.waterfall([
function(callback) {
callback(null, 'one', 'two');
},
function(arg1, arg2, callback) {
// arg1 now equals 'one' and arg2 now equals 'two'
callback(null, 'three');
},
function(arg1, callback) {
// arg1 now equals 'three'
callback(null, 'done');
}
], function (err, result) {
// result now equals 'done'
});
Can be written like this (using Blubird):
Promise.resolve(['one', 'two']).spread(function(arg1, arg2) {
// arg1 now equals 'one' and arg2 now equals 'two'
return 'three';
}).then(function(arg1) {
// arg1 now equals 'three'
return 'done';
}).then(function(result) {
// result now equals 'done'
});
You can refer to http://bluebirdjs.com/docs/coming-from-other-libraries.html#coming-from-async-module for more info.

Looping an array with async module

I have to do a query to the DB and then for each element in the result array I have to do another query, and then complete the code. I'm using async.waterfall to do the tasks and inside one function I use async.forEachSeries to do the looping.
async.waterfall([
function(callback){
connection.query(sql, params, function(err, rows, fields){
if(err) throw err;
callback(null, rows);
})
},
function(rows, callback){
var result = [];
var aux = {};
for(var i in rows){
if(typeof aux[rows[i].id] == 'undefined'){
var obj = {
'id': rows[i].id,
'title': rows[i].title,
'type': rows[i].type,
'url': rows[i].url,
'last_update': rows[i].last_update,
'unixend': rows[i].unixend,
'client_id': rows[i].id_client,
'client_name': rows[i].client_name,
'background_timeline': rows[i].background_timeline,
'description': rows[i].description,
'init': rows[i].init,
'end': rows[i].end,
'days': rows[i].days,
'timeinit': rows[i].timeinit,
'timeend': rows[i].timeend,
'lat': rows[i].lat,
'lon': rows[i].lon,
'url2': rows[i].url2,
//'locations': [ ],
};
aux[rows[i].id] = obj;
}
}
callback(null, aux, result)
},
function(aux, result, callback){
async.eachSeries(aux,
function(obj, callback){
connection.query(sql2, [obj['id'], obj['id'], obj['id']], function(err, data){
obj['locations'] = data;
callback();
})
},
function(err){
if(err) throw err;
else callback(null, aux, result)
});
},
function(aux, result, callback){
for(var i in aux){
result.push(aux[i]);
}
callback(null, result);
}
],
function(err, result){
if(err) callback1(err, []);
else callback1(false, result);
}
);
Here, callback1 is the callback passed as an argument to the global function when it's called. Also, sql and sql2 are the queries stored in variables.
The problem is that it never enters at the eachSeries method. Why is that?

How do I pass a variable into mongodb callback?

I'm trying to select the distinct values for two fields from my mongodb. Using async.series, I have the following:
function getSettings(target){
var versions = [];
var result = {};
function putResults(key, val){
result[key] = val;
}
async.series(
[
function(callback){
db.get(target).distinct('version', function(err, arr){
if(err) return err;
versions = arr;
callback();
}); //end db call
},
function(callback){
var items;
for(var i = 0; i<versions.length; i++)
{
db.get(target).distinct('instance', {version: versions[i]}, function(err, arr){
items = arr;
putResults(versions[i], items);
}); //end db call
}
}
],
function(err){
if(err) return err;
console.log(result);
}); //end async
}
With this code, I get an "undefined" exception on putResults(versions[i], items). I know it's because versions is out of scope. My question is: how do I get it in scope?
Expected result:
result = {1: [1, 2, 3], 2: [1, 2, 3, 4]}
In our case better use waterfall instead of series, and map, like this
function getSettings(target){
async.waterfall([
function (callback) {
db.get(target).distinct('version', function (err, versions) {
callback(err, versions);
});
},
function (versions, callback) {
async.map(versions, function (version, next) {
db.get(target).distinct('instance', {version: version}, function (err, items) {
var result = {};
result[version] = items;
next(err, result);
});
}, function (err, res) {
callback(err, res);
});
}
], function (err, result) {
if (err) {
return;
}
console.log(result);
});
}
As Alexander mentioned in his comment, if you want to pass the values from the first function to the next one, you should use waterfall instead of series.
As specified in the official documentation regarding waterfall:
Runs the tasks array of functions in series, each passing
their results to the next in the array. However, if any of the tasks
pass an error to their own callback, the next function is not
executed, and the main callback is immediately called with the error.
If you would like to read more about the various capabilities of async I strongly recommend their official documentation (LINK).

Resources