async forEach loops making api calls - node.js

I have a function that makes an api call and a second function that loops through the data of the first function and makes an api call each iteration. I'm trying to use the async library to make this happen but the 2nd function is still running asynchronously instead of waiting to finish. So I end up running function 1 runs, function 2 starts, but final callback runs before function 2 finishes.
async.series([
function (callback) {
//api call
getShelves.execute(function (err, shelves) {
if (err) { return callback(err); }
async.forEach(shelves.items, function (shelf, callback) {
var shelfObj = {id: shelf.id, title: shelf.title, books: []};
bookShelves.push(shelfObj);
callback();
});
//sort numerically to make placing books easier
bookShelves.sort(function (a, b) {return a.id - b.id; });
callback();
});
},
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
callback();
});
}
callback();
});
});
callback();
}
], function (err) {
if (err) { console.log('error'); }
res.render('collection', { shelves: bookShelves });
});
});
EDIT: Working now thanks guys
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
console.log(book.volumeInfo.title);
//callback to continue book loop
callback();
}, function () {
//callback to continue shelf loop
callback();
});
}else{
callback();
}
});
}, function () {
//callback to end function and move to next. However this is never reached
callback();
});
}

The second function in your series calls its callback immidiately, not waiting until async.forEach iteration finishes. Instead, try this to call it afterwards:
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
//... skipped ...
}, function() {
callback();
});
}

function loadShelf(shelf, callback) {
//_.pick is handy for this FYI
var shelfObj = {id: shelf.id, title: shelf.title};
//presumably your getBooks call takes a shelf id to relate the
//getBooks is an asynchronous DB or API call presumably
getBooks(shelf.id, function (error, books) {
if (error) {
callback(error);
return;
}
//This is an in-memory array. No async needed.
shelfObj.books = books.map(function (book) {
return {title: book.volumeInfo.title};
});
callback(null, shelfObj);
});
}
getShelves.execute(function (error, dbShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
async.each(dbShelves, loadShelf, function (error, fullShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
//sort numerically to make placing books easier
var sortedShelves = fullShelves.sort(function (a, b) {return a.id - b.id; });
res.render('collection', { shelves: sortedShelves });
});

Related

Async series is not calling properly

I have 3 function has to call in series one after the other. but first function is exeucting and in between second and third are executing.
var tasklist=[api_hit,delay,mysql_check];
if(task_list.length>0){
async.series(
tasklist,
function(err, response) {
console.log(err);
console.log(response);
results.data=response;
results.message="Completed";
console.log(results);
}
);
}
Internal functions:
function api_hit(callback){
console.log("Inside api");
var ele=task_list[0];
var apidata=[];
var msg={'data':[]};
apiinfo.forEach((item,key)=>{
if(item.Method_name==ele.Parameters){
//Here checking random Int value
if(item.Value=="{{$randomInt}}"){
item.Value = generate(25);
}
apidata.push(item);
}
});
var data=[];
data['api']=apidata;
apiModel.validateAPI(data,function(res){
console.log("result api");
msg.data=res;
msg.case='api_hit';
callback(msg);
});
}
function delay(callback){
console.log("Inside delay");
var msg={'data':[]};
global_vars.sleep(1000);
msg.data='success';
msg.case='task';
console.log("after delay");
callback(msg);
}
function mysql_check(callback){
console.log("inside mysql");
var ele=task_list[2];
var dbdata=[];
var msg={'data':[]};
dbchecks.forEach((item,key)=>{
if(item.query_id==ele.Parameters){
console.log(item.query+" ::: "+ele.Parameters);
dbdata.push(item);
}
});
data['dbdata']=dbdata;
apiModel.checkmysql(data,function(err,res){
if(err) throw err;
console.log("inside mysql res");
msg.data=res;
msg.case='task2';
callback(msg);
});
}
My intention is to call these function after completing of others and all the results has to process in a single variable. but in api_hit method when it is executing another function inside of it then delay()(second function of async) is executing. how to stop this and make it in sequence. thanks in advance.
The first argument to the callback function is the error, pass null in case of success.
'use strict'
const async = require('async')
function api_hit(callback) {
setTimeout(() => {
console.log('Completed api_hit')
callback(null, 'api_hit')
}, 1000)
}
function delay(callback) {
setTimeout(() => {
console.log('Completed delay')
callback(null, 'delay')
}, 100)
}
function mysql_check(callback) {
setTimeout(() => {
console.log('Completed mysql_check')
callback(null, 'mysql_check')
}, 500)
}
var tasklist = [api_hit, delay, mysql_check];
if (tasklist.length > 0) {
async.series(
tasklist,
function (err, response) {
console.log(err);
console.log(response);
}
);
}
Doc link: https://caolan.github.io/async/docs.html#series

Hard time uderstaning async.parallel in node

I have this function in the controller
router.post('/', function(req, res, next) {
if (req.user.isPremium == false) {
// Free user - Single report
let website = req.body.website0;
let builtWithCall = `https://api.builtwith.com/free1/api.json?KEY=APIKEY&LOOKUP=${website}`;
let pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=APIKEY`;
// curl 'https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://georgiancollege.ca&strategy=mobile&key=APIKEY'
var calls = [];
calls.push(function(callback) {
// First call
https.get(builtWithCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log('BuiltWith received', data);
});
});
});
calls.push(function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
});
} else {
// Premium user - comparison report
let websites = [];
}
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
});
I am trying to run several async API calls at once. The problem is, when I try to run them like this
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
the console doesn't log anything.
When I do the console log here though
function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
}
it logs the response. The pageSpeed one gets in a weird loop and repeats itself multiple times, but at least it works.
Now what am I doing wrong with the async.parallel part? Also what is this callback in function(callback) {?
EDIT:
This is the new version of the anonymous function:
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end', function(data) {
callback(null, data);
});
resource.on('error', function(err) {
callback(err);
});
});
}
You need to call the passed in callback. Looking at your one parallel function you are not calling callback(). I'll assume your resource object has an end & error
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end' function() {
callback(null, results);
});
resource.on('error' function(err) {
callback(err);
});
});
}
How async.parallel works is all the functions called must in turn call the passed in callback function; in your case that is callback.
Once each function in the parallel calls callback then and only then will the final function be called, which is the function you defined as function(err, results) {...}.
There is one caveat, if in the callback call you pass non-null for the first argument then that final function will be called immediately where you should handle that error if it happens.

async waterfall callback was already called error in nodejs

I am newbee to nodejs. I have been scratching my head for long time on why below code throwing error.I found some similar in stack overflow, but couldn't get help in finding the bug.
var albums_coll,photos_coll;
async.waterfall([
function(cb){
MongoClient.connect(url,
(err,dbase)=>{
if(err) {
console.log('bad');
process.exit(-1);
}
console.log("I have a connection!");
db=dbase;
cb(null);
}
);
},
function(cb){
db.collection("albums",cb);
},
function(album_obj,cb){
albums_coll = album_obj;
db.collection("photos",cb);
},
function(photos_obj,cb){
photos_coll = photos_obj;
cb(null);
},
function(cb){
albums_coll.InsertOne(a1,cb);
},
function(inserted_doc,cb){
console.log("I have Inserted a document!!");
console.log(inserted_doc);
cb(null);
}
],
function(err,results){
console.log("Done!!!");
console.log(err);
console.log(results);
db.close();
});
Please suggest!!
the following is the error its showing.
I have a connection!
C:\Users\thathine\NodeLive\Chapter08\mongotest\node_modules\mongodb\lib\mongo_client.js:433
throw err
^
Error: Callback was already called.
at C:\Users\thathine\NodeLive\Chapter08\mongotest\node_modules\async\dist\async.js:903:32
at Db.collection (C:\Users\thathine\NodeLive\Chapter08\mongotest\node_modules\mongodb\lib\db.js:466:27)
.
.
.
The callback() function you are passing into the db.collection() function is being called twice. The first time this happens is inside the MongoDB library while invoking db.collection(). It is happening in the following block - I added a comment pointing to line 466 as shown in the stack:
if(options == null || !options.strict) {
try {
var collection = new Collection(this, this.s.topology, this.s.databaseName, name, this.s.pkFactory, options);
if(callback) callback(null, collection);
return collection;
} catch(err) {
// if(err instanceof MongoError && callback) return callback(err);
if(callback) return callback(err); // <-- Line 466 - first time callback is called
throw err;
}
}
The second time it gets called, is by the async library when it needs to determine if it has to execute the next task in the array or invoke the final callback; by then, the callback is already null. The error you're seeing is being raised in the onlyOnce() function that is part of the async library:
function onlyOnce(fn) {
return function() {
if (fn === null) throw new Error("Callback was already called.");
var callFn = fn;
fn = null;
callFn.apply(this, arguments);
};
}
So the following code will raise the error:
async.waterfall([
function(callback) {
MongoClient.connect('mongodb://localhost:27017/test', (error, db) => {
if (error) {
console.error(error);
process.exit(-1);
}
callback(null, db);
});
},
function(db, callback) {
console.log('Querying albums collection...');
// This will cause the exception
db.collection('albums', callback);
},
function(db, albums, callback) {
console.log('Got albums...');
console.log('Querying photos collection...');
db.collection('photos', (error, photos) => {
if (error) {
return callback(error);
}
callback(null, db, albums, photos);
});
},
function(db, albums, photos, callback) {
console.log('Got photos...');
callback(null, 'DONE');
}
], function (error, results) {
console.error(error);
console.log(results);
process.exit(0);
});
While this version of the code will execute as expected:
async.waterfall([
function(callback) {
MongoClient.connect('mongodb://localhost:27017/test', (error, db) => {
if (error) {
console.error(error);
process.exit(-1);
}
callback(null, db);
});
},
function(db, callback) {
console.log('Querying albums collection...');
db.collection('albums', (error, albums) => {
if (error) {
return callback(error);
}
callback(null, db, albums);
});
},
function(db, albums, callback) {
console.log('Got albums...');
console.log('Querying photos collection...');
db.collection('photos', (error, photos) => {
if (error) {
return callback(error);
}
callback(null, db, albums, photos);
});
},
function(db, albums, photos, callback) {
console.log('Got photos...');
callback(null, 'DONE');
}
], function (error, results) {
console.error(error);
console.log(results);
process.exit(0);
});
/* Output:
Querying albums collection...
Got albums...
Querying photos collection...
Got photos...
null
DONE
*/
Hope this helps!

async.parallel inside async.series

I have this async.parallel functionality inside an aysnc.eachSeries call.
I hardcoded an error so I could pass it, to see if it was behaving the way I thought. For some reason, when I pass an error, it doesn't get thrown in the final callback named "doneWithSeries".
async.eachSeries(jsonDataArr, function iterator(item, callback) {
async.parallel([
function (cb) {
if (item.hasOwnProperty('event.type')) {
var event_type = item['event.type'];
delete item['event.type'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + event_type + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
},
function (cb) {
if (item.hasOwnProperty('status_desc')) {
var status_desc = item['status_desc'];
delete item['status_desc'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + status_desc + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
}
],
function doneWithParallel(err) {
callback(new Error('throw this baby')); //shouldn't the first incident of error pass the error straight to the doneWithSeries callback below?
})
},
function doneWithSeries(err) {
if (err) {
throw err;
}
else {
console.log('success');
}
});
here is a distilled version of the code without anything unnecessary:
var async = require('async');
async.eachSeries(['1', '2'], function (item, callback) {
async.parallel([
function (cb) {
setTimeout(function () {
cb(null, 'one');
}, 200);
},
function (cb) {
setTimeout(function () {
cb(null, 'two');
}, 100);
}
],
function doneWithParallel(err, results) {
console.log('results', results);
callback(new Error('duh'));
})
},
function doneWithSeries(err) {
if (err)
throw err;
});
indeed that works. can't figure out why my code above doesn't, accept perhaps that the array could be empty even though when I run my code the success message gets logged...weird.
I think that's expected behavior if your list is empty. async will always call the final callback with no error even if there is no input list.

Call same function many times and process combined result set

I have a requirement to make several API requests and then do some processing on the combines result sets. In the example below, you can see that 3 requests are made (to /create) by duplicating the same request code however I would like to be able to specify how many to make. For example, I may wish to run the same API call 50 times.
How can I make n calls without duplicating the API call function n times?
async.parallel([
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/api/store/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
}
],
function(err, results){
if (err) {
console.log(err);
}
// do stuff with results
});
First, wrap the code that you want to call many times in a function:
var doRequest = function (callback) {
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err);
}
callback(null, res.body.id);
});
}
Then, use the async.times function:
async.times(50, function (n, next) {
doRequest(function (err, result) {
next(err, result);
});
}, function (error, results) {
// do something with your results
}
Create an array with as many references to the function as you need tasks in your workload. Then pass them to async.parallel. For example:
var async = require("async");
var slowone = function (callback) {
setTimeout(function () {
callback(null, 1);
}, 1000);
};
async.parallel(
dd(slowone, 100),
function (err, r) {
console.log(JSON.stringify(r));
}
);
// Returns an array with count instances of value.
function dd(value, count) {
var result = [];
for (var i=0; i<count; i++) {
result.push(value);
}
return result;
}
Note again that there is only one instance of the slow running function, in spite of there being many references to it.

Resources