Can't make Mocha before() nested callback clause behave synchronously - node.js

Running into async hell again with the following inner callback in a test I am writing. I've commented the callback that doesn't wait. I'm using both an async.series to marshall the functions, and async.each to keep the inner iteration synchronous. Mocha compalins "done() was called multiple times" - why isn't the code waiting?
describe('Report Generation (R subsystem)', function () {
before(function (done) {
//clear test files
async.series([function (callback) { //1st
console.log('Delete local test files');
_.each(output_file_template, function (test_file) {
if (fs.existsSync(__dirname + '/../reports/' + test_file + user_file_code + '.png')) {
fs.unlinkSync(__dirname + '/../reports/' + test_file + user_file_code + '.png');
};
}); //..._.each
callback();
}, function (callback) { //2nd
console.log('Delete remote test files');
async.each(output_file_template, function (test_file, cb) {
console.log(test_file);
s3.del('reports/' + test_file + user_file_code + '.png', function (err, res) {
console.log("Delete err", err);
console.log("Delete result", res);
cb();
}, function(err) {callback(err);}); //s3.head
}); //...async.each
}], function (err, res) { //3rd
done(); //this should tell the begin() clause to complete
}); //...async.series
}); //...before
it('should not start this test until before() has finished!', function (done) {
console.log("1st test here");
});
});

, What I can see is, you are doing async.series with an array of 3 functions, but no controlling function at the end.
I assume, your code in it('should not start... is the one which should be in there.
So (I think) your code should look like this:
describe('My Test 1', function () {
//clear test files
async.series([
function (callback) { //1st
console.log('Delete local test files');
...
callback();
},
function (callback) { //2nd
console.log('Delete remote test files');
async.each(
output_file_template,
function (test_file, cb) {
console.log(test_file);
s3.del('reports/' + test_file + user_file_code + '.png', function (err, res) { //I can't get the following nested callback to wait
console.log("Delete err", err);
console.log("Delete result", res);
cb();
}); //...s3.head
},
function( err ) { // this is the control function for async.each, so now it waits after all s3 have finished
callback( err );
}
); //...s3.head
},
function (callback) { //3rd -> will be called now after the async.each (I don't think, you use it so can be deleted anyway)
callback();
}
],
function( err, result ) {
done(); // whatever this has to do with the last "it" -> here is the point, where the "before" is completely done
}
});
I didn't test the source, so maybe there are typos inside, but I think it shows the picture.

Related

Hard time uderstaning async.parallel in node

I have this function in the controller
router.post('/', function(req, res, next) {
if (req.user.isPremium == false) {
// Free user - Single report
let website = req.body.website0;
let builtWithCall = `https://api.builtwith.com/free1/api.json?KEY=APIKEY&LOOKUP=${website}`;
let pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=APIKEY`;
// curl 'https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://georgiancollege.ca&strategy=mobile&key=APIKEY'
var calls = [];
calls.push(function(callback) {
// First call
https.get(builtWithCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log('BuiltWith received', data);
});
});
});
calls.push(function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
});
} else {
// Premium user - comparison report
let websites = [];
}
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
});
I am trying to run several async API calls at once. The problem is, when I try to run them like this
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
the console doesn't log anything.
When I do the console log here though
function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
}
it logs the response. The pageSpeed one gets in a weird loop and repeats itself multiple times, but at least it works.
Now what am I doing wrong with the async.parallel part? Also what is this callback in function(callback) {?
EDIT:
This is the new version of the anonymous function:
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end', function(data) {
callback(null, data);
});
resource.on('error', function(err) {
callback(err);
});
});
}
You need to call the passed in callback. Looking at your one parallel function you are not calling callback(). I'll assume your resource object has an end & error
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end' function() {
callback(null, results);
});
resource.on('error' function(err) {
callback(err);
});
});
}
How async.parallel works is all the functions called must in turn call the passed in callback function; in your case that is callback.
Once each function in the parallel calls callback then and only then will the final function be called, which is the function you defined as function(err, results) {...}.
There is one caveat, if in the callback call you pass non-null for the first argument then that final function will be called immediately where you should handle that error if it happens.

Nodejs send waterfall requests

I have dynamic array with foldername and creating a zip using zipFolder
I am facing issues as the library is async i.e
zipFolder(source,dest,callback)
Because its a async call it take times and I want to block the next request unless the first one reponse is true,
function (req, res)
var folder = ['N','PP','Y'];//dynamic just an example
for( var i = 0 ; i < folder.length ; i++) {
var source = path.join(basePath,'/download/'+folder[i]);
var dest = path.join(basePath,'/createZip/'+folder[i]+'.zip');
console.log('sending',folders[i])
//this is async
zipFolder(source,dest, function(err) {
if(err) {
console.log('oh no!', err);
} else {
console.log('EXCELLENT');//want to send the second,third,fourth if success
}
});
}
Summary:
output should be like this:
sending file 1 if success/response sending file 2 if success/response
want something like waterfall wait for callback reponse to send the second file
plus I want a method to know the done call back so I can send back the reponse
you can use each as like:
var each = require('async-each-series')
var folder = ['N','PP','Y'];
each(folder, function(items,callback){
//insert each items to db in async way
var source = path.join(basePath,'/download/'+items);
var dest = path.join(basePath,'/createZip/'+items+'.zip');
zipFolder(source,dest, function(err) {
if(err) {
console.log('oh no!', err);
} else {
console.log(items,'inserted')
callback()
}
});
},function(err){
if(err) console.log('error occured')
console.log('all items uploaded');
});
Try with async waterfall method
https://caolan.github.io/async/docs.html#waterfall
Runs the tasks array of functions in series, each passing their
results to the next in the array. However, if any of the tasks pass an
error to their own callback, the next function is not executed, and
the main callback is immediately called with the error
async.waterfall([
function(callback) {
callback(null, 'one', 'two');
},
function(arg1, arg2, callback) {
// arg1 now equals 'one' and arg2 now equals 'two'
callback(null, 'three');
},
function(arg1, callback) {
// arg1 now equals 'three'
callback(null, 'done');
}
], function (err, result) {
// result now equals 'done'
});
UPDATE
in your usecase eachSeries is more applicable.
var folder = ['N','PP','Y'];
async.eachSeries(folder, function(item,callback){
var source = path.join(basePath,'/download/'+item);
var dest = path.join(basePath,'/createZip/'+item+'.zip');
zipFolder(source,dest, function(err) {
if(err) {
console.log('oh no!', err);
callback();
} else {
console.log('EXCELLENT');//want to send the second,third,fourth if success
callback();
}
});
},function(){
console.log('all items uploaded');
}); .

nodejs mongodb async.Each done

How to detect when async.Each is done loop all objects/arrays loaded into the parameters. how to call the ALL FINISH log.
db.collection('tests').find(query).limit(10).toArray(function(err, db_results) {
console.log("count: "+db_results.length);
async.each(db_results, function (db_resultsx, cb) {
db_resultsx._id = db_resultsx._id.toString();
db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format("YYYY-MM-DD hh:mm:ss");
documents.push(db_resultsx);
}, function(documents) {
console.log(documents);
console.log(documents.length);
console.log("ALL FINISH");
process.exit();
});
});
The reason that the async.each function is not finishing is that the callback isn't being called within the second function. I've created a snipped with a simple example without the mongo call but with a timeout instead.
If you look at the docs https://caolan.github.io/async/docs.html#each the second argument is a callback (in your code this is the value cb) that needs to be called to signify the end of the code you are running.
function delayedReturn(callback) {
setTimeout(function(){
console.log('timeout complete');
callback(false, ['first value']);
}, 1000);
}
var documents = [];
delayedReturn(function(err, db_results) {
async.each(db_results, function (dbResult, cb) {
documents.push(dbResult);
console.log('documents pushed', dbResult)
cb();
}, function(error) {
if(error)
{
console.error(error);
}
});
});
<script src="https://rawgit.com/caolan/async/master/dist/async.min.js"></script>
There's no reason to use async.each() given that the code that it should be running isn't asynchronous.
Instead, you can use .map():
db.collection("tests").find(query).limit(10).toArray(function(err, db_results) {
console.log("count: " + db_results.length);
let documents = db_results.map(function(db_resultsx) {
db_resultsx._id = db_resultsx._id.toString();
db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format(
"YYYY-MM-DD hh:mm:ss"
);
return db_resultsx;
});
console.log(documents);
console.log(documents.length);
curl(URL, { data : documents }, function(err) {
console.log("ALL FINISH");
process.exit();
});
});

Callback not getting called while using async.series

I have the following code, where i am calling a callback after my async task is completed:
var async = require("async");
function _callback(err, result){
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
var result = {};
tasks.push(function(_callback) {
console.log('Getting some data');
_callback(null, result);
});
tasks.push(function(_callback) {
console.log('Second function called');
_callback(null, result);
});
async.series(tasks, function(){
console.log('All done');
});
}
tasks();
I have checked against the syntax expected in the async library. Code looks similar to me.
Can someone point out whats needs to be changed here.
Try it like this:
var async = require("async");
function _callback(err, result){
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
var result = {};
tasks.push(function(next) {
console.log('Getting some data');
next(null, result);
});
tasks.push(function(_callback) {
console.log('Second function called');
next(null, result);
});
async.series(tasks, function(err, result){
console.log('All done');
_callback(err, result);
});
}
tasks();
Async gives every single task a callback which you need to call to let async now that this task finished.
After all tasks are finished you can call your callback.
If you want your callback to get called after every single task is finished just add before every
next(null, result) also an _callback(null, result)
The reason that wouldn't work on your implementation is that you overwrite the global _callback method with the argument that has the same name.
Hope that makes sense :)
As it stands in your code right now, the _callback function you define above should never get called, as the _callback in the tasks functions is scope to the one passed by async.
// this will never get called
function _callback(err, result){
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
tasks.push(function(_callback) { // the _callback here overrides the one above for this closure
console.log('Getting some data');
_callback(null, result);
});
// ... more tasks
async.series(tasks, function(){
console.log('All done');
});
}
If you want to reuse the logic in your _callback function, I'd recommend passing it the async callback as a parameter:
// rename some functions to avoid silly js scoping mysteries
function myCallback(err, result, asyncCallback){
// shared logic here
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
asyncCallback(null, result);
}
// ...
tasks.push(function(asyncCallback) { // the _callback here overrides the one above for this closure
console.log('Getting some data');
myCallback(null, result, asyncCallback);
});
// ... more tasks
async wants its callback to be called so it knows when to continue, but that doesn't mean you can't intercept it with your own handlers :P.
You don't need the function call to _callback. You could simplify it like this.
var async = require("async");
function doSomeLogic(item, callback) {
console.log(item);
callback(null, item + 2);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
var result = {};
tasks.push(function(callback){
// do some more stuff ...
callback(null, 'one');
});
tasks.push(function(callback){
// do some more stuff ...
callback(null, 'two');
});
async.series(tasks, function(error, results) {
if (error) {
console.log(error);
}
console.log(results);
console.log('All done');
});
}
tasks();

async forEach loops making api calls

I have a function that makes an api call and a second function that loops through the data of the first function and makes an api call each iteration. I'm trying to use the async library to make this happen but the 2nd function is still running asynchronously instead of waiting to finish. So I end up running function 1 runs, function 2 starts, but final callback runs before function 2 finishes.
async.series([
function (callback) {
//api call
getShelves.execute(function (err, shelves) {
if (err) { return callback(err); }
async.forEach(shelves.items, function (shelf, callback) {
var shelfObj = {id: shelf.id, title: shelf.title, books: []};
bookShelves.push(shelfObj);
callback();
});
//sort numerically to make placing books easier
bookShelves.sort(function (a, b) {return a.id - b.id; });
callback();
});
},
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
callback();
});
}
callback();
});
});
callback();
}
], function (err) {
if (err) { console.log('error'); }
res.render('collection', { shelves: bookShelves });
});
});
EDIT: Working now thanks guys
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
console.log(book.volumeInfo.title);
//callback to continue book loop
callback();
}, function () {
//callback to continue shelf loop
callback();
});
}else{
callback();
}
});
}, function () {
//callback to end function and move to next. However this is never reached
callback();
});
}
The second function in your series calls its callback immidiately, not waiting until async.forEach iteration finishes. Instead, try this to call it afterwards:
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
//... skipped ...
}, function() {
callback();
});
}
function loadShelf(shelf, callback) {
//_.pick is handy for this FYI
var shelfObj = {id: shelf.id, title: shelf.title};
//presumably your getBooks call takes a shelf id to relate the
//getBooks is an asynchronous DB or API call presumably
getBooks(shelf.id, function (error, books) {
if (error) {
callback(error);
return;
}
//This is an in-memory array. No async needed.
shelfObj.books = books.map(function (book) {
return {title: book.volumeInfo.title};
});
callback(null, shelfObj);
});
}
getShelves.execute(function (error, dbShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
async.each(dbShelves, loadShelf, function (error, fullShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
//sort numerically to make placing books easier
var sortedShelves = fullShelves.sort(function (a, b) {return a.id - b.id; });
res.render('collection', { shelves: sortedShelves });
});

Resources