I'm building a server in Nodejs to retrieve data from some database. I've worked with the async library for some time now and figured out some things like putting a waterfall inside a parallel function.
I stumbled upon a problem where I first need to execute a query and then use that query's result in other queries that can be executed concurrently. The code looks something like this:
async.waterfall([
function(callback) {
connection.query( query,
function(err, rows, fields) {
if (!err) {
callback(null,rows);
} else {
callback(null,"SORRY");
}
}
);
},
async.parallel([
function(resultFromWaterfall,callback) {
connection.query(query,
function(err, rows, fields) {
if (!err) {
callback(null,rows);
} else {
callback(null,"SORRY");
}
}
);
},
function(resultFromWaterfall,callback) {
connection.query(query,
function(err, rows, fields) {
if (!err) {
callback(null,rows);
} else {
callback(null,"SORRY");
}
}
);
}
])
], finalCallback
);
Now my problem is to access the result from the waterfall function and to use it in the parallel functions.
async.waterfall([
function(callback) {
connection.query(query,
function(err, rows, fields) {
if (!err) {
callback(null, rows);
} else {
callback(null, "SORRY");
}
}
);
},
function(prevData,callback){
console.log(prevData);//Use it whereever you want.
async.parallel([
function(callbackOfAsyncParallel) {
connection.query(query1,
function(err, rows1, fields1) {
if (!err) {
callbackOfAsyncParallel(null, rows1);
} else {
callbackOfAsyncParallel(null, "SORRY1");
}
}
);
},
function(callback) {
connection.query(query2,
function(err, rows2, fields2) {
if (!err) {
callbackOfAsyncParallel(null, rows2);
} else {
callbackOfAsyncParallel(null, "SORRY2");
}
}
);
}
],function mainCBOfParallel(err,reuslts){
if(!err){
//this will be done after tasks in async.parallel are finished.
callback(null,results);
//results[0]===>rows1
//results[1]===>rows2
}
});
}
], finalCallback);
You have two mistakes in your code,
tasks should be a function to execute.
async.parallel has only callbacks in its task functions.
Update
There are callbacks (callbackOfAsyncParallel) which will be called
when a task in async.parallel is finished.
It should not invoke callback (callback) of async.waterfall.If
done probably there may be error/unexpected results.
Try this..
async.waterfall([
function(callback) {
connection.query(query,
function(err, rows, fields) {
if (!err) {
callback(null, rows);
} else {
callback(null, "SORRY");
}
}
);
},
function(resultFromWaterfall, callback) {
async.parallel([
function() {
connection.query(query,
function(err, rows, fields) {
if (!err) {
callback(null, rows);
} else {
callback(null, "SORRY");
}
}
);
},
function() {
connection.query(query,
function(err, rows, fields) {
if (!err) {
callback(null, rows);
} else {
callback(null, "SORRY");
}
}
);
}
]);
}
], finalCallback);
doWhatever(callback) {
async.waterfall([
(waterfallCallback) => {
connection.query(query, waterfallCallback);
},
(rows, fields, waterfallCallback) => {
async.parallel([
(parallelCallback) => {
connection.query(query, parallelCallback);
},
(parallelCallback) => {
connection.query(query, parallelCallback);
}
], waterfallCallback);
}
], callback);
},
That's just me trying to make it clearer…
Anyway, my exemple is not taking into account parameters and arguments of query. It's just a scaffold
Related
I want to launch a series of async tasks without using Promise & Async await.
My worfklow is :
. task 1
. task 2
. task 3 & 4 (in parallel)
I've tried the library async for that. That's ok for a series of sequential task or a series of parallel. But to implement my worklfow, I'm stuck for the moment.
const async = require('async');
const fs = require('fs');
const readFile1 = (name, callback) => {
fs.readFile(name, 'utf-8', (err, data) => {
if (err) {
callback(err);
}
console.log(data);
callback(null, data);
});
}
async.series([
function(callback1) {
async.series([
function(callback1) {
readFile1('./1.txt', callback1);
},
function(callback1) {
readFile1('./2.txt', callback1);
},
], function(err, results) {
console.log('series ', results);
});
},
function(callback2) {
async.parallel([
function(callback2) {
readFile1('./3.txt', callback2);
},
function(callback2) {
readFile1('./4.txt', callback2);
},
], function(err, results) {
console.log('parallel ', results);
});
}
]);
For now only the step 1 & 2 are executed. Have you got an idea for this (apart using a more modern code style with async / await & promises?)
Blured
You need to call the top-level callbacks before async.series() will continue to the next step:
async.series([
function (callback) {
async.series(
[
function (callback1) {
readFile1("./1.txt", callback1);
},
function (callback2) {
readFile1("./2.txt", callback2);
},
],
function (err, results) {
console.log("series ", results);
return callback(err, results); // <-- here
}
);
},
function (callback) {
async.parallel(
[
function (callback1) {
readFile1("./3.txt", callback1);
},
function (callback2) {
readFile1("./4.txt", callback2);
},
],
function (err, results) {
console.log("parallel ", results);
return callback(err, results); // <-- and here
}
);
},
]);
(renamed callback names to make things a bit more clear)
I am trying to develop an APP in which I have to increase or decrease the user count according to their hobbies/interest in the master list. I am doing it in Node js with the help of loopback. Here is my code, in which I am giving two interests(i.e sketching and horse-riding):
async.forEach(data, function (interest) {
console.log("Interest is", interest);
Interest.findOne({
where:
{
'name': interest
}
}, function (err, interestObj) {
if (err) {
//return callback(err, null);
console.log("error", err);
}
else {
//return callback(null, response);
console.log("found", interestObj);
if (!interestObj) {
Interest.create({ "name": interest, "count": 1 }, function (err, response) { });
}
else {
_count = interestObj.count + 1;
interestObj.updateAttribute('count', _count, function (e, r) { });
}
}
});
// return callback(null, {});
},function(err){
console.log("success..!!")
});
}
but it is showing me only one of them in output. Here is output:
data is [ 'horse-riding', 'skeching' ]
Interest is horse-riding
Interest is skeching
found { name: 'horse-riding', count: 1, id: 59ccff0765055a212491a6bc }
found null
I think the async function is not working properly with forEach loop in this, But I am not getting where the code went wrong. I want to show all the interest given by the user, so what course of actions should I take to do it?? Thanks in advance..!!:)
It is working now...!!!
async.each(data, function (interest, callback2) {
console.log('Processing ', interest);
Interest.findOne({
where:
{
'name': interest
}
}, function (err, interestObj) {
if (err) {
console.log("error", err);
callback2(err);
}
else {
console.log("found", interestObj);
if (!interestObj) {
Interest.create({ "name": interest, "count": 1 }, function (err, response) { });
}
else {
_count = interestObj.count + 1;
interestObj.updateAttribute('count', _count, function (e, r) { });
}
callback2();
}
});
}, function (err) {
if (err) {
console.log('Failed to process', err);
} else {
console.log('All interests have been processed successfully');
}
return callback(err);
})
};
I have this async.parallel functionality inside an aysnc.eachSeries call.
I hardcoded an error so I could pass it, to see if it was behaving the way I thought. For some reason, when I pass an error, it doesn't get thrown in the final callback named "doneWithSeries".
async.eachSeries(jsonDataArr, function iterator(item, callback) {
async.parallel([
function (cb) {
if (item.hasOwnProperty('event.type')) {
var event_type = item['event.type'];
delete item['event.type'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + event_type + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
},
function (cb) {
if (item.hasOwnProperty('status_desc')) {
var status_desc = item['status_desc'];
delete item['status_desc'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + status_desc + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
}
],
function doneWithParallel(err) {
callback(new Error('throw this baby')); //shouldn't the first incident of error pass the error straight to the doneWithSeries callback below?
})
},
function doneWithSeries(err) {
if (err) {
throw err;
}
else {
console.log('success');
}
});
here is a distilled version of the code without anything unnecessary:
var async = require('async');
async.eachSeries(['1', '2'], function (item, callback) {
async.parallel([
function (cb) {
setTimeout(function () {
cb(null, 'one');
}, 200);
},
function (cb) {
setTimeout(function () {
cb(null, 'two');
}, 100);
}
],
function doneWithParallel(err, results) {
console.log('results', results);
callback(new Error('duh'));
})
},
function doneWithSeries(err) {
if (err)
throw err;
});
indeed that works. can't figure out why my code above doesn't, accept perhaps that the array could be empty even though when I run my code the success message gets logged...weird.
I think that's expected behavior if your list is empty. async will always call the final callback with no error even if there is no input list.
I have a requirement to make several API requests and then do some processing on the combines result sets. In the example below, you can see that 3 requests are made (to /create) by duplicating the same request code however I would like to be able to specify how many to make. For example, I may wish to run the same API call 50 times.
How can I make n calls without duplicating the API call function n times?
async.parallel([
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/api/store/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
}
],
function(err, results){
if (err) {
console.log(err);
}
// do stuff with results
});
First, wrap the code that you want to call many times in a function:
var doRequest = function (callback) {
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err);
}
callback(null, res.body.id);
});
}
Then, use the async.times function:
async.times(50, function (n, next) {
doRequest(function (err, result) {
next(err, result);
});
}, function (error, results) {
// do something with your results
}
Create an array with as many references to the function as you need tasks in your workload. Then pass them to async.parallel. For example:
var async = require("async");
var slowone = function (callback) {
setTimeout(function () {
callback(null, 1);
}, 1000);
};
async.parallel(
dd(slowone, 100),
function (err, r) {
console.log(JSON.stringify(r));
}
);
// Returns an array with count instances of value.
function dd(value, count) {
var result = [];
for (var i=0; i<count; i++) {
result.push(value);
}
return result;
}
Note again that there is only one instance of the slow running function, in spite of there being many references to it.
I'm using async waterfall and i've got a question about try/catch error.
I wan't to dodge this syntax method with one global try/catch and no duplicate a try/catch by function :
async.waterfall([
function (callback) {
try {
this.foo() // Ok
this.bar() // Method bar doesn't exist so without try/catch node will crash
} catch(ex) {
//print err
}
},
function (callback) {
try {
//Again & again
} catch(ex) {
//print err
}
}
//... function() with try catch ...
], function(err, result) {
//Do something
})
If you seek a more robust error handling system; try this:
var async = require('async');
function callbackErrorHandler(fn, callback) {
try {
fn()
} catch (err) {
callback(err);
}
}
async.waterfall([
function(callback) {
callbackErrorHandler(function() {
throw new Error('error1');
}, callback);
},
function(callback) {
callbackErrorHandler(function() {
throw new Error('error2');
}, callback);
}
],
//final
function(err, result) {
console.log(err);
});
/**
* Using Promises
*/
var async = require('async');
//promises
var Promises = require('bluebird');
function callbackErrorHandler(fn) {
return new Promises(function(resolve, reject) {
try {
resolve(fn()); //should return something
} catch (err) {
reject(err);
}
});
}
async.waterfall([
function(callback) {
callbackErrorHandler(function() {
throw new Error('error1');
})
.error(callback)
},
function(callback) {
callbackErrorHandler(function() {
throw new Error('error2');
})
.error(callback)
}
],
//final
function(err, result) {
console.log(err);
});
This script no longer print in final callback and crash node server :
Test this and try to rename test() by testt()
'use strict'
var async = require('./lib/node_modules/async')
function test()
{
console.log("hi !")
}
async.waterfall([
function(callback){
callback(null)
},
function(callback){
test()
callback(null, null)
}], function (err, result) {
if (err)
{
console.log("got an error !")
}
else
{
console.log("everyting is ok")
}
}
)