I've inherited some Node.js code and I need to add some functionality. However, I'm not sure syntactically how to accomplish my goal due to the asynchronous nature of Node. Currently, I have a function defined like this:
return {
myEntryPoint: function(req, res) {
var results = getResults();
res.send(200, results);
}
};
Several pieces are calling this function already. Inside of it, I'm calling a function called getResults which is defined like this:
var getResults = function() {
var results = [];
async.series([
function(callback) {
// add to results
},
function(callback) {
// add to results
}
]);
return results;
};
My problem is, I need to wait until all of the functions inside of the async.series call are made before I return the results. How do I do this?
You could change it to add a callback. Series has an optional callback to run once all the functions have completed within.
var getResults = function(finished) {
var results = [];
async.series([
function(callback) {
// add to results
},
function(callback) {
// add to results
}
], function() {
//Called when series is finished
finished(results);
});
};
And to get the results,
return {
myEntryPoint: function(req, res) {
getResults(function(results) {
res.send(200, results);
});
}
};
The usual way to handle this is to take a callback that uses the argument in question.
results = getResults(args);
someFunction(results);
Is equivalent to
function getResults(args, callback) {
var results;
//do stuff and populate results.
callback(results);
}
getResults(args, someFunction);
Except you don't have the issue of trying to wait for async stuff to happen, which you can't do, without weird logic.
Related
The code comes from an MDN tutorial on how to use Node.js and mongoose. The idea is to make parallel request to get the count of documents in different models. I don't understand where the callback passed to each async.parallel comes from, where it is defined and what it does, it seems like a dummy function to me. Could you help me understand it? Here is the code:
var Book = require('../models/book');
var Author = require('../models/author');
var Genre = require('../models/genre');
var BookInstance = require('../models/bookinstance');
var async = require('async');
exports.index = function(req, res) {
async.parallel({
book_count: function(callback) {
Book.countDocuments({}, callback); // Pass an empty object as match condition to find all documents of this collection
},
book_instance_count: function(callback) {
BookInstance.countDocuments({}, callback);
},
book_instance_available_count: function(callback) {
BookInstance.countDocuments({status:'Available'}, callback);
},
author_count: function(callback) {
Author.countDocuments({}, callback);
},
genre_count: function(callback) {
Genre.countDocuments({}, callback);
}
}, function(err, results) {
res.render('index', { title: 'Local Library Home', error: err, data: results });
});
};
the callback is passed by the async package.
Explanation:
As async parallel function takes array or object (in your example) of asynchronous tasks and these async tasks require a callback which will get called when its execution completes or if there is an error. So parallel function provides these callback functions and will call your callback ( provided as a second parameter to parallel function call) when all of them are called or got an error in any of them.
You can check the detailed explanation from here - https://caolan.github.io/async/v3/docs.html#parallel
Update:
Consider parallel as a wrapper function like:
function parallel(tasks, userCallback) {
let tasksDone = [];
function callback(err, data){ // this is the callback function you're asking for
if(err){
userCallback(err); // calling callback in case of error
}else {
tasksDone.push(data);
if(tasks.length === tasksDone.length){
userCallback(null, tasksDone); // calling callback when all the tasks finished
}
}
}
tasks.forEach(task => {
task(callback); // calling each task without waiting for previous one to finish
})
}
Note: This is not a proper implementation of parallel function of async, this is just an example to understand how we can use callback function internally and what is its usecase
Let's say I have 3 files.
index.js makes a call to the backend like this
$.post('/test/', data, function(response) {
//handle success here
})
routes.js handles the route like this
app.post('/test/', function(req, res){
item.getItems(function(response){
res.json(response);
});
});
items.js is the model which accesses the database and makes a POST request for each item
function getItems(callback) {
database.query('SELECT * from items', function(result){
result.forEach(function(item){
request.post('/api/', item, function(req, res) {
//finished posting item
});
});
});
//callback here doesnt wait for calls to finish
}
where/when should I call the callback passed to getItems() to handle a success/failure in index.js?
Because your request.post() operations are async, you have to use some method of keeping track of when they are all done and then you can call your callback. There are multiple ways of doing that. I'll outline a few:
Manually Keeping Track of Count of Request Operations
function getItems(callback) {
database.query('SELECT * from items', function(result){
var remaining = result.length;
result.forEach(function(item){
request.post('/api/', item, function(err, res) {
--remaining;
//finished posting item
if (remaining === 0) {
callback();
}
});
});
});
}
The main problem with doing this manually is that propagating error in nested async operations is difficult when you get to actually figuring out how you're going to handle errors. This is much easier in the other methods shown here.
Using Promises
// load Bluebird promise library
var Promise = require('bluebird');
// promisify async operations
Promise.promisifyAll(request);
function queryAsync(query) {
return new Promise(function(resolve, reject) {
// this needs proper error handling from the database query
database.query('SELECT * from items', function(result){
resolve(result);
});
});
}
function getItems(callback) {
return queryAsync('SELECT * from items').then(function(result) {
return Promise.map(result, function(item) {
return request.postAsync('/api/', item);
});
});
}
getItems.then(function(results) {
// success here
}, function(err) {
// error here
})
It seems strange that you're making an API request in your server-side code, unless this is some sort of middle tier code that interacts with the API... but you're interacting with a database, so I'm still confused on why you can't just do a database insert, or have a bulk insert API call?
Anyway, if you must do it the way you're asking, I've done this in the past with a recursive method that trims down the result array... I really don't know if this is a good approach, so I'd like to hear any feedback. Something like this:
function recursiveResult(result, successfulResults, callback) {
var item = result.shift();
// if item is undefined, then we've hit the end of the array, so we'll call the original callback
if (item !== undefined) {
console.log(item, result);
// do the POST in here, and in its callback, call recursiveResult(result, successfulResults, callback);
successfulResults.push(item);
return recursiveResult(result, successfulResults, callback);
}
// make sure callback is defined, otherwise, server will crash
else if (callback) {
return callback(successfulResults);
}
else {
// log error... callback undefined
}
}
function getItems(callback) {
var successfulResults = [];
var result = [1, 2, 3, 4];
recursiveResult(result, successfulResults, callback);
}
console.log('starting');
getItems(function(finalResult) {
console.log('done', finalResult);
});
I have a loop that I need to run inside of a callback, unfortunately accessing the array outside of the callback leaves me with a blank array. I know why this happens, but I want to know the best solution to tackle this.
Gallery.prototype.getGallery = function(cb) {
self = this;
var cos = new pb.CustomObjectService();
var ms = new pb.MediaService();
var s = [];
cos.loadTypeByName('Gallery Image', function(err, gallery){
cos.findByType(gallery._id.toString(), function(err, rpy){
for(var i = 0; i < rpy.length; i++){
ms.loadById(rpy[i].Image, function(e,r){
s.push(r.location);
console.log(r.location); /* <-- logs expected data */
});
}
console.log(s[0]); /* <-- this is undefined */
});
});
};
Replace your for loop with a call to async.*; in this case async.map seems right. Pass a callback to async.map; it will be invoked when all the individual calls to ms.loadById are done, with the array of results.
async.map(
rpy,
function(elt, callback) {
ms.loadById(elt.Image, callback);
},
function(err, data) {
// comes here after all individual async calls have completed
// check errors; array of results is in data
}
);
If you want to go into the promises world, then wrap the calls to ms.loadById in a promise. Here's a roll-your-own version, but various versions of what is usually called promisify are also out there.
function loadByIdPromise(elt) {
return new Promise(function(resolve, reject) {
ms.loadById(elt.image, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
Then do a Promise.all on the resulting promises:
Promise.all(rpy.map(loadByIdPromise))
.then(function(data) {
// comes here when all individual async calls complete successfully
// data is your array of results
});
Using the promises style, your entire code would look like:
loadTypeByNamePromise('Gallery Image') .
then(function(gallery) { return findByTypePromise(gallery._id.toString(); }) .
then(function(rpy) { return Promise.all(rpy.map(loadByIdPromise)); }) .
then(function(results) { /* do something with [results] */ });
I am trying to use async module on my nodejs application with not lucky.
Suppose i have the following:
/**
* parents=['parent1Template','parent2TEmplate',...]
* children=[[{id: 'child1Parent1'}{id: 'child2Parent1'}],[{id: 'child1Parent2"}{id: 'child2Parent2'}],...]
*
*/
function createTemplate(parents,children){
var final=[]
async.each(Object.keys(parents), function(item,done){
if(children[item].length!==0) addChildsByParent(parents[item],children[item], function (result) {
final.push(result);
});
done();
});
console.log("Final results: "+final);
return final;
}
function addChildsByParent (parent,childs,callback){
var template=[];
//some operations..
async.each(childs,function(child){
Children.findone({"_id": child.id}, function (err,ch)){
template.push(ch);
}
});
return callback(template)
}
I need to get in final all results when all operations have finished.
I have seen also functions as parallel and waterfall on async module but the main problem is that i need to work always with two arrays and do the query when i get the single value.
What's the best way, maybe something like this?
async.waterfall([
each()...
async.waterfall([
each()...
])
])
I would use async map instead of each to build the response array using the map callback.
Also, I would use async parallel instead of waterfall to improve speed, since operations don't depend on each other and can be executed in parallel.
async.parallel({
final : function(next) {
async.map(Object.keys(parents), function(item,done){
if(children[item].length!==0) addChildsByParent(parents[item],children[item], function (result) {
done(null, result);
});
}, next);
},
template : function(next) {
async.map(childs, function(child, done) {
Children.findone({"_id" : child.id}, function (err, ch) {
done(err, ch);
});
});
}
}, function(error, results){
if (!error) {
console.log(results);
// This will be {final:[], template:[]}
}
});
I have code like
common.findOne('list', {'listId': parseInt(request.params. istId)}, function(err, result){
if(err) {
console.log(err);
}
else {
var tArr = new Array();
if(result.tasks) {
var tasks = result.tasks;
for(var i in tasks) {
console.log(tasks[i]);
common.findOne('tasks', {'taskId':parseInt(tasks[i])}, function(err,res){
tArr[i] = res;
console.log(res);
});
}
console.log(tArr);
}
return response.send(result);
}
});
It is not executed sequentially in node.js so I get an empty array at the end of execution. Problem is it will first execute console.log(tArr); and then execute
common.findOne('tasks',{'taskId':parseInt(tasks[i])},function(err,res){
tArr[i] = res;
console.log(res);
});
Is there any mistake in my code or any other way for doing this.
Thanks!
As you are probably aware, things run asynchronously in node.js. So when you need to get things to run in a certain order you need to make use of a control library or basically implement it yourself.
I highly suggest you take a look at async, as it will easily allow you to do something like this:
var async = require('async');
// ..
if(result.tasks) {
async.forEach(result.tasks, processEachTask, afterAllTasks);
function processEachTask(task, callback) {
console.log(task);
common.findOne('tasks', {'taskId':parseInt(task)}, function(err,res) {
tArr.push(res); // NOTE: Assuming order does not matter here
console.log(res);
callback(err);
});
}
function afterAllTasks(err) {
console.log(tArr);
}
}
The main things to see here is that processEachTask gets called with each task, in parallel, so the order is not guaranteed. To mark that the task has been processed, you will call callback in the anonymous function from findOne. This allows you to do more async work in processEachTask but still manage to signify when it is done. When every task is done, it will then call afterAllTasks.
Take a look at async to see all the helper functions that it provides, it is very useful!
I've recently created a simple abstraction named "wait.for" to call async functions in sync mode (based on Fibers): https://github.com/luciotato/waitfor
Using wait.for and async your code will be:
var wait = require('waitfor');
...
//execute in a fiber
function handleRequest(request,response){
try{
...
var result = wait.for(common.findOne,'list',{'listId': parseInt(request.params.istId)});
var tArr = new Array();
if(result.tasks) {
var tasks = result.tasks;
for(var i in tasks){
console.log(tasks[i]);
var res=wait.for(common.findOne,'tasks',{'taskId':parseInt(tasks[i])});
tArr[i] = res;
console.log(res);
}
console.log(tArr);
return response.send(result);
};
....
}
catch(err){
// handle errors
return response.end(err.message);
}
};
// express framework
app.get('/posts', function(req, res) {
// handle request in a Fiber, keep node spinning
wait.launchFiber(handleRequest,req,res);
});