In my function i have to call async series inside async foreach to compute the final result and create my json.is that possible
async.series([
function(callback) {
});
},
function(callback) {
async.forEachSeries(temp,function(quest,callback) {
}, function(err) {
if (err) return next(err);
});
callback();
}
],
function(err) {
if (err) return next(err);
res.json(output);
});
You should be able to nest as much async functions into each other, however it is better to use a naming conventions so you can easily track which callbacks are passed where and to avoid collisions due to hoisting. So basically this should work as you'd expect:
async.series([
function first(seriesCallback) {
seriesCallback();
},
// other functions in series
// ...
function (seriesCallback) {
var someArray = [];
async.each(someArray, function (value, eachCallback) {
// process the value
// return an error if there is one
eachCallback(err);
}, function(err) {
// add any additional processing you might need
// pass the control to the parent async method and handle the errors
// in a more central place if there is an error here it will
// be processed in onSeriesDone as well as all other errors
seriesCallback(err);
});
}], function onSeriesDone(err) {
next(err);
});
I am running a cron job with node with mongodb as the database. I am trying to close db connection and exit the process once the curr_1 each loop has executed completely.
However the exit() is called while function_2 is being executed. I understand this is due to the callback and is async in nature.
How do I make sure exit is called only once the curr_1.each is complete?
Any solution without promises?
function function_1(obj){
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
function_2(doc)
});
exit(obj)
}
function function_2(obj) {
coll_2.findOne({}, function(err, document) {
dosomeprocess(obj)
})
}
function exit(obj) {
// Close connection
console.log('closing connection')
obj.db.close();
process.exit();
}
It's a job for Node async....
For example:
async.each(
curr_1, // the collection to iterate over
function(doc, callback) { // the function, which is passed each
// document of the collection, and a
// callback to call when doc handling
// is complete (or an error occurs)
function_2(doc);
},
function(err) { // callback called when all iteratee functions
// have finished, or an error occurs
if (err) {
// handle errors...
}
exit(obj); // called when all documents have been processed
}
);
Without using any library:
function function_1(obj, callback) {
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
callback(err, doc);
});
}
function function_2(err, obj) {
coll_2.findOne({}, function(err, document) {
dosomeprocess(obj)
exit(err, obj);
})
}
function exit(err, obj) {
// Close connection
console.log('closing connection')
obj.db.close();
process.exit();
}
function_1(obj, function_2);
Using async module
var async = require('async');
async.waterfall([
function function_1(callback) {
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
if (err) {
callback(err, null)
} else {
allback(null, doc)
}
});
},
function function_2(obj, callback) {
coll_2.findOne({}, function(err, document) {
if (err) {
callback(err, null);
} else {
dosomeprocess(obj)
callback(null, obj);
}
})
}
], function done() {
obj.db.close();
process.exit();
});
Simply give a condition in your loop using counter.
function function_1(obj){
var curr_1 = coll_1.find({})
var curr_1Length = curr_1.length;
var counter = 0;
curr_1.each(function(err, doc) {
++counter;
//Check condition everytime for the last occurance of loop
if(counter == curr_1Length - 1){
exit(obj)
}
function_2(doc)
});
}
Hope it helps :)
I'm fixing some bugs in one project. I need synchronous cursor, to load data in loop.
I have code:
var mongo = require('mongojs'),
dataMaps = [];
mongo.penguin_friends.find({ user_id_1: id_user }, function(err, friends) {
friends.forEach(function(data) {
var cursor = mongo.penguin_user_maps3.find({ user_id: data.id_user }, { fields: { maps: 1 } });
//I need to do dataMaps.push(cursor.data);
});
console.log("processingThisSomething()");
processSomething();
});
I need to complete this request before calling processSomething(); So I need to process mongodb query inside a loop synchronously.
It's not possible to make the queries synchronous as the API doesn't support it.
You'll have to provide a callback to .find() or a cursor method to receive the results:
cursor.toArray(function (err, maps) {
if (!err) {
dataMaps.push(maps);
}
});
But, you can replace the iterator with one that's asynchronous-aware, such as async.each(), to continue when they've completed:
async.each(
friends,
function (data, callback) {
var cursor = mongo....;
cursor.toArray(function (err, maps) {
if (!err) {
dataMaps.push(maps);
}
callback(err);
});
},
function (err) {
if (!err) {
processSomething();
}
}
);
I have a function that makes an api call and a second function that loops through the data of the first function and makes an api call each iteration. I'm trying to use the async library to make this happen but the 2nd function is still running asynchronously instead of waiting to finish. So I end up running function 1 runs, function 2 starts, but final callback runs before function 2 finishes.
async.series([
function (callback) {
//api call
getShelves.execute(function (err, shelves) {
if (err) { return callback(err); }
async.forEach(shelves.items, function (shelf, callback) {
var shelfObj = {id: shelf.id, title: shelf.title, books: []};
bookShelves.push(shelfObj);
callback();
});
//sort numerically to make placing books easier
bookShelves.sort(function (a, b) {return a.id - b.id; });
callback();
});
},
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
callback();
});
}
callback();
});
});
callback();
}
], function (err) {
if (err) { console.log('error'); }
res.render('collection', { shelves: bookShelves });
});
});
EDIT: Working now thanks guys
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
console.log(book.volumeInfo.title);
//callback to continue book loop
callback();
}, function () {
//callback to continue shelf loop
callback();
});
}else{
callback();
}
});
}, function () {
//callback to end function and move to next. However this is never reached
callback();
});
}
The second function in your series calls its callback immidiately, not waiting until async.forEach iteration finishes. Instead, try this to call it afterwards:
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
//... skipped ...
}, function() {
callback();
});
}
function loadShelf(shelf, callback) {
//_.pick is handy for this FYI
var shelfObj = {id: shelf.id, title: shelf.title};
//presumably your getBooks call takes a shelf id to relate the
//getBooks is an asynchronous DB or API call presumably
getBooks(shelf.id, function (error, books) {
if (error) {
callback(error);
return;
}
//This is an in-memory array. No async needed.
shelfObj.books = books.map(function (book) {
return {title: book.volumeInfo.title};
});
callback(null, shelfObj);
});
}
getShelves.execute(function (error, dbShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
async.each(dbShelves, loadShelf, function (error, fullShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
//sort numerically to make placing books easier
var sortedShelves = fullShelves.sort(function (a, b) {return a.id - b.id; });
res.render('collection', { shelves: sortedShelves });
});
function indexArticles(callback) {
fs.readdir("posts/", function(err, files) {
async.map(files, readPost, function(err, markdown) {
async.map(markdown, parse, function(err, results) {
async.sortBy(results, function(obj, callback) {
callback(err, obj.date);
}, function(err, sorted) {
callback( {"articles": sorted.reverse()} );
});
});
});
});
}
I'm trying to figure out how to make this prettier -- as you can tell I'm using caolan's async library, but I'm not sure which of the control flow structures to use. It seems like if I use async.waterfall, for example, that results in quite a bit more code, with each step having to be wrapped in an anonymous function. For example, this is just the first two lines of the nested version with waterfall:
function indexArticles(callback) {
async.waterfall([
function(callback) {
fs.readdir("posts/", function(err, files) {
callback(err, files)
})
},
function(files, callback) {
async.map(files, readPost, function(err, markdown) {
callback(err, markdown)
})
}])
}
How would you improve this?
If there were a way to partially apply arguments NOT only from the left, then I could see doing, for example,
function indexArticles(callback) {
async.waterfall([
async.apply(fs.readdir, "posts/"),
async.apply(async.map, __, readPost),
async.apply(async.map, __, parse),
// etc...
])
}
This is an interesting problem, as you need to bind arguments both to the left and to the right of your iterator functions, so neither bind/ nor bindRight (of which there are a few implementaions on StackOverflow) will work for you. There's a few options for you here:
(1) First, in your async.waterfall example, you have:
function(callback) {
fs.readdir("posts/", function(err, files) {
callback(err, files)
})
}
which is the same as:
function(callback) {
fs.readdir("posts/", callback)
}
Using Function.bind and this method, your entire function indexArticles could be written:
function indexArticles(callback) {
async.waterfall([
fs.readdir.bind(this, 'posts/'),
function(files, cb) { async.map(files, readPost, cb); },
function(text, cb) { async.map(text, parse, cb); },
function(results, cb) { async.sortBy(results, function(obj, callback) {
callback(null, obj.date);
}, cb) }
], function(err, sorted) {
callback( {"articles": sorted.reverse()} );
});
};
Which is a bit shorter.
(2) If you really want to avoid the wrapping functions, you can use a type of partial function application. First, at the top of your file (or in a module, etc), define a function called partial:
var partial = function(fn) {
var args = Array.prototype.slice.call(arguments, 1);
return function() {
var currentArg = 0;
for(var i = 0; i < args.length && currentArg < arguments.length; i++) {
if (args[i] === undefined)
args[i] = arguments[currentArg++];
}
return fn.apply(this, args);
};
}
This function takes a function and any number of arguments, and replaces undefined values in the arguments list with the actual arguments when the function is called. You would then use it like this:
function indexArticles(callback) {
async.waterfall([
fs.readdir.bind(this, 'posts/'),
partial(async.map, undefined, readPost, undefined),
partial(async.map, undefined, parse, undefined),
partial(async.sortBy, undefined, function(obj, callback) {
callback(null, obj.date);
}, undefined)
], function(err, sorted) {
callback( {"articles": sorted.reverse()} );
});
}
So, partial(async.map, undefined, readPost, undefined) returns a function that, when called by the Async library as fn(files, callback), it fills in files for the first undefined, and callback for the second undefined, ending in a call to async.map(files, readPost, callback).
(3) There is also a version of partial for Function.prototype at this StackOverflow answer, allowing you to use the syntax: async.map.partial(undefined, readPost, undefined); however, I would probably recommend against modifying Function.prototype in this way, and just use partial as a function.
In the end, it's up to you which method is the most readable and maintainable.
Looks like I have some overlap with Brandon's answer, but here's my take:
var async = require("async")
//dummy function
function passThrough(arg, callback){
callback(null, arg)
}
//your code rewritten to only call the dummy.
//same structure, didn't want to think about files and markdown
function indexArticles(callback) {
passThrough("posts/", function(err, files) {
async.map(files, passThrough, function(err, markdown) {
async.map(markdown, passThrough,
function(err, results) {
async.sortBy(results, function(obj, callback) {
callback(err, obj);
},
function(err, sorted) {
callback( {"articles": sorted.reverse()} );
});
});
});
});
}
indexArticles(console.log)
//version of apply that calls
//fn(arg, arg, appliedArg, apliedArg, callback)
function coolerApply(fn) {
var args = Array.prototype.slice.call(arguments, 1);
return function () {
var callback = Array.prototype.slice.call(arguments, -1)
var otherArgs = Array.prototype.slice.call(arguments, 0, -1)
return fn.apply(
null, otherArgs.concat(args).concat(callback)
);
};
};
//my version of your code that uses coolerAppl
function indexArticles2(callback){
async.waterfall([
async.apply(passThrough, "posts/"),
coolerApply(async.map, passThrough),
coolerApply(async.map, passThrough),
coolerApply(async.sortBy, function(obj, callback){callback(null,obj)})
],
function(err, sorted){
callback({"articles": sorted.reverse()})
})
}
//does the same thing as indexArticles!
indexArticles2(console.log)
Here's what I've ended up with so far.
function indexArticles(callback) {
var flow = [
async.apply(fs.readdir, "posts/"),
function(data, callback) { async.map(data, readPost, callback); },
function sortByDate(parsed, callback) {
var iterator = function(obj, callback) {
if (obj.date) { callback(null, obj.date); }
else { callback("Article has no date.") }
}
// Note that this sorts in reverse lexicographical order!
async.sortBy(parsed, iterator,
function(err, sorted) { callback(err, {"articles": sorted.reverse()} ); }
);
}
];
async.waterfall(flow, async.apply(callback))
}
I've recently created a simple abstraction named WaitFor to call async functions in sync mode (based on Fibers): https://github.com/luciotato/waitfor
I've not tested it with the async package, but it should work. If you run into problems, contact me.
Using wait.for and async your code will be:
var wait = require('waitfor');
var async = require('async');
function indexArticles(callback) {
var files = wait.for(fs.readdir,"posts/");
var markdown = wait.for(async.map, files, readPost);
var results = wait.for(async.map, markdown, parse);
var sorted = wait.for(async.sortBy, results, function(obj, callback) {
callback(null, obj.date);
});
callback( null, {"articles": sorted.reverse()} );
}
to call your fn (async-mode):
//execute in a fiber
wait.launchFiber(indexArticles,function(err,data){
// do something with err,data
});
to call your fn (sync-mode):
//execute in a fiber
function handleRequest(req,res){
try{
...
data = wait.for(indexArticles); //call indexArticles and wait for results
// do something with data
res.end(data.toString());
}
catch(err){
// handle errors
}
}
// express framework
app.get('/posts', function(req, res) {
// handle request in a Fiber, keep node spinning
wait.launchFiber(handleRequest,req,res);
});