I have this function in the controller
router.post('/', function(req, res, next) {
if (req.user.isPremium == false) {
// Free user - Single report
let website = req.body.website0;
let builtWithCall = `https://api.builtwith.com/free1/api.json?KEY=APIKEY&LOOKUP=${website}`;
let pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=APIKEY`;
// curl 'https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://georgiancollege.ca&strategy=mobile&key=APIKEY'
var calls = [];
calls.push(function(callback) {
// First call
https.get(builtWithCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log('BuiltWith received', data);
});
});
});
calls.push(function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
});
} else {
// Premium user - comparison report
let websites = [];
}
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
});
I am trying to run several async API calls at once. The problem is, when I try to run them like this
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
the console doesn't log anything.
When I do the console log here though
function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
}
it logs the response. The pageSpeed one gets in a weird loop and repeats itself multiple times, but at least it works.
Now what am I doing wrong with the async.parallel part? Also what is this callback in function(callback) {?
EDIT:
This is the new version of the anonymous function:
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end', function(data) {
callback(null, data);
});
resource.on('error', function(err) {
callback(err);
});
});
}
You need to call the passed in callback. Looking at your one parallel function you are not calling callback(). I'll assume your resource object has an end & error
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end' function() {
callback(null, results);
});
resource.on('error' function(err) {
callback(err);
});
});
}
How async.parallel works is all the functions called must in turn call the passed in callback function; in your case that is callback.
Once each function in the parallel calls callback then and only then will the final function be called, which is the function you defined as function(err, results) {...}.
There is one caveat, if in the callback call you pass non-null for the first argument then that final function will be called immediately where you should handle that error if it happens.
Related
I have 3 function has to call in series one after the other. but first function is exeucting and in between second and third are executing.
var tasklist=[api_hit,delay,mysql_check];
if(task_list.length>0){
async.series(
tasklist,
function(err, response) {
console.log(err);
console.log(response);
results.data=response;
results.message="Completed";
console.log(results);
}
);
}
Internal functions:
function api_hit(callback){
console.log("Inside api");
var ele=task_list[0];
var apidata=[];
var msg={'data':[]};
apiinfo.forEach((item,key)=>{
if(item.Method_name==ele.Parameters){
//Here checking random Int value
if(item.Value=="{{$randomInt}}"){
item.Value = generate(25);
}
apidata.push(item);
}
});
var data=[];
data['api']=apidata;
apiModel.validateAPI(data,function(res){
console.log("result api");
msg.data=res;
msg.case='api_hit';
callback(msg);
});
}
function delay(callback){
console.log("Inside delay");
var msg={'data':[]};
global_vars.sleep(1000);
msg.data='success';
msg.case='task';
console.log("after delay");
callback(msg);
}
function mysql_check(callback){
console.log("inside mysql");
var ele=task_list[2];
var dbdata=[];
var msg={'data':[]};
dbchecks.forEach((item,key)=>{
if(item.query_id==ele.Parameters){
console.log(item.query+" ::: "+ele.Parameters);
dbdata.push(item);
}
});
data['dbdata']=dbdata;
apiModel.checkmysql(data,function(err,res){
if(err) throw err;
console.log("inside mysql res");
msg.data=res;
msg.case='task2';
callback(msg);
});
}
My intention is to call these function after completing of others and all the results has to process in a single variable. but in api_hit method when it is executing another function inside of it then delay()(second function of async) is executing. how to stop this and make it in sequence. thanks in advance.
The first argument to the callback function is the error, pass null in case of success.
'use strict'
const async = require('async')
function api_hit(callback) {
setTimeout(() => {
console.log('Completed api_hit')
callback(null, 'api_hit')
}, 1000)
}
function delay(callback) {
setTimeout(() => {
console.log('Completed delay')
callback(null, 'delay')
}, 100)
}
function mysql_check(callback) {
setTimeout(() => {
console.log('Completed mysql_check')
callback(null, 'mysql_check')
}, 500)
}
var tasklist = [api_hit, delay, mysql_check];
if (tasklist.length > 0) {
async.series(
tasklist,
function (err, response) {
console.log(err);
console.log(response);
}
);
}
Doc link: https://caolan.github.io/async/docs.html#series
I want to do async call from my getData function to getImage function but i am unable to get return data from getImage().Since the getData() does't wait for the completion of getImage(),as getImage() has further async db calls and therefore getData() always returns undefined.
What is the best way to do this instead doing nested callbacks?
var getData = function(id){
async.series([
function(callback){
var res = getImages(id);
callback(null, res);
}
],
// optional callback
function(err, results){
if (err) {
console.log("ERROR : " + err);
}else
{
console.log("Result: "+results);
}
});
}
var getImages = function(id){
async.series([
function(callback){
Image.find({id: id }).exec(
function(err, image) {
if (err) {
console.log(err);
callback(err, 0);
}else
{ console.log("Count: "+ image.length);
callback(null, image);
}
});
}
],
// optional callback
function(err, results){
if (err) {
console.log("ERROR : " + err);
}else
{
return results;
}
});
}
getData(1);
As you said you need to wait for getImages() to return, and you do that using promises.
Use any promise library, like q for instance:
var q = require('q')
...
var getImages = function(id){
var deferred = q.defer();
...
//do async logic that that evaluates some res obj you wish to return
db.find(..., function() {
deferred.resolve(res);
}
return deferred.promise;
}
Then, from getData(), you call it in the following matter:
getImages(id).then(
function(res) {
callback(null, res);
},
function(err) {
console.log("error:" + err);
}
);
As you are already using async - just use the waterfall functionality: https://github.com/caolan/async#waterfalltasks-callback
This way you will be able to run functions one after another and wait for the previous to finish, while still getting it's return value.
I'm trying to wrap my head around the async library, but I'm pretty wobbly in NodeJs and I can't figure out async.parallel. The code below produces error TypeError: undefined is not a function on the line where the parallel tasks are to be executed. Am I correct in that tasks to be run in async.parallel should have a callback() when they are done? (irrelevant parts of the function are redacted)
function scrapeTorrents(url, callback) {
request(url, function(err, res, body) {
if(err) {
callback(err, null);
return;
}
var $ = cheerio.load(body);
var results = [];
var asyncTasks = [];
$('span.title').each(function(i, element){
// scrape basic info
var show = {title: info.title, year: info.year};
asyncTasks.push(
getOmdbInfo(show, function (err, res) {
if (res) {
omdbInfo = res;
results.push({
// add basic info and Omdb info
});
}
callback();
})
);
});
async.parallel(asyncTasks, function(){
callback(null, results);
});
});
}
In the section where you define async tasks, be sure to specify a closure with a parameter method to call once the task is complete (named differently than callback so as to avoid hoisting).
asyncTasks.push(
function (done) {
getOmdbInfo(show, function (err, res) {
if (err) {
return done(err);
}
if (res) {
omdbInfo = res;
results.push({
// add basic info and Omdb info
});
}
return done();
})
}
);
I have the following code, where i am calling a callback after my async task is completed:
var async = require("async");
function _callback(err, result){
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
var result = {};
tasks.push(function(_callback) {
console.log('Getting some data');
_callback(null, result);
});
tasks.push(function(_callback) {
console.log('Second function called');
_callback(null, result);
});
async.series(tasks, function(){
console.log('All done');
});
}
tasks();
I have checked against the syntax expected in the async library. Code looks similar to me.
Can someone point out whats needs to be changed here.
Try it like this:
var async = require("async");
function _callback(err, result){
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
var result = {};
tasks.push(function(next) {
console.log('Getting some data');
next(null, result);
});
tasks.push(function(_callback) {
console.log('Second function called');
next(null, result);
});
async.series(tasks, function(err, result){
console.log('All done');
_callback(err, result);
});
}
tasks();
Async gives every single task a callback which you need to call to let async now that this task finished.
After all tasks are finished you can call your callback.
If you want your callback to get called after every single task is finished just add before every
next(null, result) also an _callback(null, result)
The reason that wouldn't work on your implementation is that you overwrite the global _callback method with the argument that has the same name.
Hope that makes sense :)
As it stands in your code right now, the _callback function you define above should never get called, as the _callback in the tasks functions is scope to the one passed by async.
// this will never get called
function _callback(err, result){
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
tasks.push(function(_callback) { // the _callback here overrides the one above for this closure
console.log('Getting some data');
_callback(null, result);
});
// ... more tasks
async.series(tasks, function(){
console.log('All done');
});
}
If you want to reuse the logic in your _callback function, I'd recommend passing it the async callback as a parameter:
// rename some functions to avoid silly js scoping mysteries
function myCallback(err, result, asyncCallback){
// shared logic here
if(err) console.log('Error Occurred');
console.log('Callback called');
console.dir(result);
asyncCallback(null, result);
}
// ...
tasks.push(function(asyncCallback) { // the _callback here overrides the one above for this closure
console.log('Getting some data');
myCallback(null, result, asyncCallback);
});
// ... more tasks
async wants its callback to be called so it knows when to continue, but that doesn't mean you can't intercept it with your own handlers :P.
You don't need the function call to _callback. You could simplify it like this.
var async = require("async");
function doSomeLogic(item, callback) {
console.log(item);
callback(null, item + 2);
}
function tasks() {
console.log('Start executing tasks');
var tasks = [];
var result = {};
tasks.push(function(callback){
// do some more stuff ...
callback(null, 'one');
});
tasks.push(function(callback){
// do some more stuff ...
callback(null, 'two');
});
async.series(tasks, function(error, results) {
if (error) {
console.log(error);
}
console.log(results);
console.log('All done');
});
}
tasks();
I have a function that makes an api call and a second function that loops through the data of the first function and makes an api call each iteration. I'm trying to use the async library to make this happen but the 2nd function is still running asynchronously instead of waiting to finish. So I end up running function 1 runs, function 2 starts, but final callback runs before function 2 finishes.
async.series([
function (callback) {
//api call
getShelves.execute(function (err, shelves) {
if (err) { return callback(err); }
async.forEach(shelves.items, function (shelf, callback) {
var shelfObj = {id: shelf.id, title: shelf.title, books: []};
bookShelves.push(shelfObj);
callback();
});
//sort numerically to make placing books easier
bookShelves.sort(function (a, b) {return a.id - b.id; });
callback();
});
},
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
callback();
});
}
callback();
});
});
callback();
}
], function (err) {
if (err) { console.log('error'); }
res.render('collection', { shelves: bookShelves });
});
});
EDIT: Working now thanks guys
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
console.log(book.volumeInfo.title);
//callback to continue book loop
callback();
}, function () {
//callback to continue shelf loop
callback();
});
}else{
callback();
}
});
}, function () {
//callback to end function and move to next. However this is never reached
callback();
});
}
The second function in your series calls its callback immidiately, not waiting until async.forEach iteration finishes. Instead, try this to call it afterwards:
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
//... skipped ...
}, function() {
callback();
});
}
function loadShelf(shelf, callback) {
//_.pick is handy for this FYI
var shelfObj = {id: shelf.id, title: shelf.title};
//presumably your getBooks call takes a shelf id to relate the
//getBooks is an asynchronous DB or API call presumably
getBooks(shelf.id, function (error, books) {
if (error) {
callback(error);
return;
}
//This is an in-memory array. No async needed.
shelfObj.books = books.map(function (book) {
return {title: book.volumeInfo.title};
});
callback(null, shelfObj);
});
}
getShelves.execute(function (error, dbShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
async.each(dbShelves, loadShelf, function (error, fullShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
//sort numerically to make placing books easier
var sortedShelves = fullShelves.sort(function (a, b) {return a.id - b.id; });
res.render('collection', { shelves: sortedShelves });
});