I am currently trying to learn the async module for node.js. Below is an example demonstrating how to use async.forEach.
var async = require('async'),
request = require('request');
var results = {};
function done(err) {
if (err) {
throw err;
}
console.log('done! results: %j', results);
}
var collection = [1, 2, 3, 4];
function iterator(value, callback) {
request.post({
uri: 'http://localhost:8080',
body: JSON.stringify(value)
},
function(err, res, body) {
if (err) {
return callback(err);
}
results[value] = JSON.parse(body);
callback();
});
}
async.forEach(collection, iterator, done);
So async.forEach iterated through each element in the collection, and each element is then passed into "value" in the iterator function...but I don't really understand whats being passed into callback? (which is then executed at the end of iterator)
It will receive an array of the results of the individual operations – whatever you pass to the callback.
Since you don't pass anything to the callback, it will receive an array of undefined.
Related
I am using the libraries mongoose 5.4.20, request 2.88.0, q 1.5.1, and deasync 0.1.14.
When using .cursor() and .eachAsync() the promises don't get resolved in the order I would expect. In particular, I make a request to the data base, and for each returned document, I make several external requests via an async series. The sample code is as follows:
Request Promise:
function Request (options) {
const result = Q.defer();
request(options, function (error, response, body) {
if (error) {
result.reject(error);
} else {
result.resolve(body);
}
});
return result.promise;
}
Async Series Promise:
function Series (entry) {
const result = Q.defer();
const errors = [];
//Iterate over some given array
async.eachSeries(array, function (doc, cb) {
//Make some async request
return Request(doc)
.then(function (body) {
//do stuff with 'body' and 'entry'
})
.then(function () {
//Move to the next doc in 'array'
return cb(null);
})
.fail(function (err) {
//Handle errors
errors.push(err);
//Move to the next doc in 'array'
return cb(null);
});
}, function (err, results) {
if (err) {
errors.push(err);
}
if (errors.length !== 0) {
result.reject(errors);
} else {
console.log('A');
result.resolve();
}
});
return result.promise;
}
Mongoose:
mongooseModel
.find()
.cursor()
.eachAsync(entry => Series(entry))
.then(function () {
console.log('B');
}, function (err) {
console.log(err);
});
What confuses me is that the final callback in .then() after .eachAsync() seems to be called before the promise in .eachAsync() is resolved, i.e. 'console.log('B')' is called before 'console.log('A')'. I would have expected, that .eachAsync() will wait until the promise is resolved, then pulls the next document from the collection and so on, and only at the very end, when all promises are resolved, the final .then() is called.
Any suggestion as to what I am doing wrong would be greatly appreciated. Thanks.
I have the following code
var getUrl = function(callback) {
ref.once('value')
.then(function(snap) {
let article_url = snap.val();
console.log(article_url);
callback(null, article_url); // I dont want to throw an error, so I pass null for the error argument
});
};
var getImage = function(err, data, callback) {
//if (err) throw err; // Check for the error and throw if it exists.
textapi.extract({
url: data,
best_image: true
}, function(error, response) {
if (error === null) {
//console.log(response.image);
if (response.image == null || response.image == "") {
image = "https://www.wefornews.com/wp-content/uploads/2017/01/news-3.jpg"
} else {
image = response.image;
}
}
});
callback(null, image); // Otherwise proceed as usual.
};
var updateDatabase = function(err, data) {
//if (err) throw err; // Check for the error and throw if it exists.
refupdate.update ({
"Article_Image_URL": data
});
console.log('success'); // Otherwise proceed as usual
};
getUrl(getImage);
getImage(updateDatabase);
However I get an error callback is not a function at the line callback(null, image);
The thing is if I remove the updateDatabase function then I get no errors. However when it is within the code I get the error mentioned above.
Can someone please tell me what can be causing this error?
When you call getUrl(getImage), the getImage function is passed as the callback parameter to getUrl. getUrl then calls this callback parameter, passing two parameters.
callback(null, article_url)
but getImage expects three parameters
var getImage = function(err, data, callback)
so callback inside the getImage function is undefined, and when you try to call undefined as a function
callback(null, image)
... you get your callback is not a function error.
Solution
You need to nest your callbacks so the functions are called in the correct order, with the correct parameters.
getUrl(function (err, article_url) {
getImage(err, article_url, function (err, image) {
updateDatabase(err, image);
});
});
This could also be simplified since you want to call updateDatabase directly after getImage completes.
getUrl(function (err, article_url) {
getImage(err, article_url, updateDatabase);
});
Also, in your getImage function, the callback is called before your textapi.extract function completes, which is probably not want you want.
Before
var getImage = function(err, data, callback) {
textapi.extract({
url: data,
best_image: true
}, function(error, response) {
/* code */
});
callback(null, image); // This will get called immediately, without waiting for textapi.extract
};
After
var getImage = function(err, data, callback) {
textapi.extract({
url: data,
best_image: true
}, function(error, response) {
/* code */
callback(null, image); // Callback moved here, so it gets called after textapi.extract completes
});
};
const updateDatabase = async (value) => {
try {
let snap = await ref.once(value);
let article_url = snap.val();
let image = await textapi.extract({
url: article_url,
best_image: true
});
if (image==='') {image = 'https://www.wefornews.com/wp-content/uploads/2017/01/news-3.jpg'}
refupdate.update({
"Article_Image_URL": article_url
});
} catch (err) {
throw err;
}
}
because you don't provide any callback on this call getImage(updateDatabase); (last line) ?
I am trying to make multiple HTTP requests and cumulate, display the results in NodeJS using the following code:
const async = require('async');
const request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function(err, res, body) {
console.log("invoked")
callback(err, body);
}
).on('error', function(err) {
console.log(err)
});
}
const urls= [
"http://1.2.3.4:30500/status/health/summary",
"http://5.6.7.8:30505/status/health/summary"
];
async.map(urls, httpGet, function (err, res){
if (err)
console.log(err);
else
console.log(res);
});
The problem here is, if the first request(http://1.2.3.4:30500/status/health/summary) fails (like connection refused etc.), the second one does not go through. I know that I am making a silly mistake but cannot find it. Any help appreciated !
In async.map if one of the calls passes an error to its callback, the main callback (for the map function) is immediately called with the error(this is the problem in your case). In order not to terminate on the first error, don't call the callback with err param in your httpGet.
Use async each, it receives a list of arguments and a function, and calls the function with each element, make sure in your httpGet inside on error you call the callback, without the err, this will make rest of the calls to continue even if there was an error in some of the calls. This can work for map too but, I think the more suitable function for your case is async.each, instead of map, also you can limit the number of concurrent calls with eachLimit method.
Check https://caolan.github.io/async/docs.html#each
const async = require('async');
const request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function(err, res, body) {
if (err){
console.log(err);
callback();
return;
}
console.log("invoked")
callback(null, body);
}
).on('error', function(err) {
console.log(err);
callback();
});
}
const urls= [
"http://1.2.3.4:30500/status/health/summary",
"http://5.6.7.8:30505/status/health/summary"
];
async.each(urls, httpGet, function (err, res) {
}, function (err, res) {
});
If you want async.map NOT to fail fast you could do it like this
const async = require('async');
const request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function alwaysReportSuccess(err, res, body) {
callback(null, {
success: !err,
result: err ? err : body
});
}
).on('error', function(err) {
console.log(err)
});
}
const urls= [
"http://1.2.3.4:30500/status/health/summary",
"http://5.6.7.8:30505/status/health/summary"
];
async.map(urls, httpGet, function alwaysOk(_, res){
console.log(res); // will be an array with success flags and results
});
I am using async series to run 2 functions takes2Seconds and function takes5Seconds.Why the final callback not showing any result?
var async = require('async'),
operations = [];
operations.push(takes2Seconds(1,function(){}));
operations.push(takes5seconds(2,function(){}));
async.series(operations, function (err, results) {
if(err){return err;}
console.log(results);
});
function takes2Seconds(a,callback) {
results='Took 2 sec'+a;
callback(null, results);
}
function takes5seconds(b,callback) {
results='Took 5sec'+b;
callback(null, results);
}
It looks like you are pushing two undefined values to operations.
When running async.series the operations array needs to contain function(s) that has callback as it's parameter.
When you do operations.push(takes2Seconds(1, function() {})); you are invoking the takes2Seconds function straight away, and because there is no return statement in the takes2Seconds function, it is pushing undefined to the operations array.
Below, I have added a return statement into your takesXSeconds functions. They now return a function with callback as the parameter, and the returned function gets pushed to the operations array.
I have also remove the callback param from takesXSeconds as it is not needed at this point.
When running async.series(...) now, each function (which we returned from takesXSeconds) gets invoked.
var async = require('async'),
operations = [];
operations.push(takes2Seconds(1));
operations.push(takes5seconds(2));
async.series(operations, function (err, results) {
if(err){return err;}
console.log(results);
});
function takes2Seconds(a) {
var results='Took 2 sec'+a;
return function(callback) {
callback(null, results);
}
}
function takes5seconds(b) {
var results='Took 5sec'+b;
return function(callback) {
callback(null, results);
}
}
First your takes2Seconds function executed then function takes5seconds executed.
var takes2Seconds = function (a, callback) {//first this function executed
results = 'Took 2 sec' + a;
callback(null, results);
};
async.eachSeries(takes2Seconds, takes5seconds, function (err) {
if (err) {
res.json({status: 0, msg: "OOPS! How is this possible?"});
}
res.json("Series Processing Done");
})
var takes5seconds = function (b, callback) { // second this function executed
results = 'Took 5sec' + b;
callback(null, results);
}
I'm trying to select the distinct values for two fields from my mongodb. Using async.series, I have the following:
function getSettings(target){
var versions = [];
var result = {};
function putResults(key, val){
result[key] = val;
}
async.series(
[
function(callback){
db.get(target).distinct('version', function(err, arr){
if(err) return err;
versions = arr;
callback();
}); //end db call
},
function(callback){
var items;
for(var i = 0; i<versions.length; i++)
{
db.get(target).distinct('instance', {version: versions[i]}, function(err, arr){
items = arr;
putResults(versions[i], items);
}); //end db call
}
}
],
function(err){
if(err) return err;
console.log(result);
}); //end async
}
With this code, I get an "undefined" exception on putResults(versions[i], items). I know it's because versions is out of scope. My question is: how do I get it in scope?
Expected result:
result = {1: [1, 2, 3], 2: [1, 2, 3, 4]}
In our case better use waterfall instead of series, and map, like this
function getSettings(target){
async.waterfall([
function (callback) {
db.get(target).distinct('version', function (err, versions) {
callback(err, versions);
});
},
function (versions, callback) {
async.map(versions, function (version, next) {
db.get(target).distinct('instance', {version: version}, function (err, items) {
var result = {};
result[version] = items;
next(err, result);
});
}, function (err, res) {
callback(err, res);
});
}
], function (err, result) {
if (err) {
return;
}
console.log(result);
});
}
As Alexander mentioned in his comment, if you want to pass the values from the first function to the next one, you should use waterfall instead of series.
As specified in the official documentation regarding waterfall:
Runs the tasks array of functions in series, each passing
their results to the next in the array. However, if any of the tasks
pass an error to their own callback, the next function is not
executed, and the main callback is immediately called with the error.
If you would like to read more about the various capabilities of async I strongly recommend their official documentation (LINK).