I have an array of URLs and I want to loop through them and fetch thr content. After I have looped through them and fetched thr content I want a callback function to be called.
I know I can do this via async library but I want to do this without using any library.
Sample of what kind of code I want is below
['yahoo.com', 'gmail.com'].each(function(item){
//code to fetch URL content
},someCallbackFunctionToBeExecutedAtTheEndOfLoop);
This is typically the type of thing you do using promises (But you would need a library), with a code like:
var ops = [];
urls.forEach(function(url) {
ops.push(fetchUrl(url));
});
P.all(ops).then(callback);
function fetchUrl(url) {
var defer = P.defer();
//do stuff
// call defer.resolve(result);
return defer.promise;
}
If you don't want to use promises, you can use a counter of operations, like:
var ops = urls.length;
urls.forEach(function(url) {
// do stuff
ops--;
if (ops === 0) {
callback();
}
});
If you chose the promises, I advice to use p-promise module, which is far more optimized than Q.
If you want to do it without any sort of library like async, then you have to write your own counter to keep track of when all the async responses have been completed:
var request = require('request');
function loadAll(list, fn) {
var cnt = list.length;
var responses = [];
list.forEach(function(url, index) {
request(url, function(error, response, body) {
if (error) {
fn(error);
} else {
responses[index] = response;
--cnt;
if (cnt === 0) {
fn(0, responses);
}
}
});
})
}
loadAll(['http://www.yahoo.com', 'http://www.gmail.com'], function(err, results) {
if (!err) {
// process results array here
}
});
If you're going to be doing many async operations in node.js, then getting a promise library like Bluebird will save you a lot of time. For example, I think you could do the above in something like this (untested):
var Promise = require("bluebird");
var requestP = Promise.promisfy(require("request"));
Promise.map(['http://www.yahoo.com', 'http://www.gmail.com'], requestP).then(function(results) {
// process the array of results here
});
Related
I'm trying my damndest to avoid callback hell with my Node JS. But I'm trying to make a large number of api-requests and insert these into my database.
My issue here (of course) is that my for-loop runs and increments i before I finish my request and database insertion.
for(var i = 0; i <= 1 ; i++){
apiRequest = data[i];
apicall(apiRequest);
}
function apicall(urlApi){
request((urlApi), function(error, response, body){
if(error){
console.log("error");
} else if(!error && response.statusCode == 200){
var myobj = JSON.parse(body);
dbInsert(myobj);
}
});
}
function dbInsert(obj) {
//insert into database
}
If someone else would come by this question I can truly recommend this blogpost which I found after reading the response by joshvermaire:
http://www.sebastianseilund.com/nodejs-async-in-practice
There are a number of ways to approach this type of problem. Firstly, if you can run all the API calls in parallel (all in flight at the same time) and it doesn't matter what order they are inserted in your database, then you can get a result a lot faster by doing that (vs. serializing them in order).
In all the options below, you would use this code:
const rp = require('request-promise');
function apicall(urlApi){
return rp({url: urlApi, json: true}).then(function(obj){
return dbInsert(obj);
});
}
function dbInsert(obj) {
//insert into database
// return a promise that resolves when the database insertion is done
}
Parallel Using ES6 Standard Promises
let promises = [];
for (let i = 0; i <= data.length; i++) {
promises.push(apicall(data[i]));
}
Promise.all(promises).then(() => {
// all done here
}).catch(err => {
// error here
});
Parallel using Bluebird Promise Library
With the Bluebird Promise library, you can use Promise.map() to iterate your array and you can pass it the concurrency option to control how many async calls are in flight at the same time which might keep from overwhelming either the database or the target API host and might help control max memory usage.
Promise.map(data, apiCall, {concurrency: 10}).then(() => {
// all done here
}).catch(err => {
// error here
});
In Series using Standard ES6 Promises
If you have to serialize them for some reason such as inserting into the database in order, then you can do that like this. The .reduce() pattern shown below is a classic way to serialize promise operations on an array using standard ES6:
data.reduce(data, (p, item) => {
return p.then(() => {
return apicall(item);
});
}, Promise.resolve()).then(() => {
// all done here
}).catch(err => {
// error here
});
In Series Using Bluebird Promises
Bluebird has a Promise.mapSeries() that iterates an array in series, calling a function that returns a promise on each item in the array which is a little simpler than doing it manually.
Promise.mapSeries(data, apiCall).then(() => {
// all done here
}).catch(err => {
// error here
});
I'd recommend using something like async.each. Then you could do:
async.each(data, function(apiRequest, cb) {
apicall(apiRequest, cb);
}, function(err) {
// do something after all api requests have been made
});
function apicall(urlApi, cb){
request((urlApi), function(error, response, body){
if(error){
console.log("error");
cb(error);
} else if(!error && response.statusCode == 200){
var myobj = JSON.parse(body);
dbInsert(myobj, cb);
}
});
}
function dbInsert(obj, cb) {
doDBInsert(obj, cb);
}
When the dbInsert method completes, make sure the cb callback is called. If you need to do this in a series, look at async.eachSeries.
I want to call from a parent rest service a child rest service. The number of times child service is called depends on parameters to parent rest services. Once I call all child service instance concurrently with different parameters. I want to combine the responses from all instances of child service. I am using below snippet. But I don't want to use timeout. It should either be timeout or when all calls of child service are over which ever is lesser.
for( i=0; i<length; i++)
{
url=accountID[i] +'+'+sortcode[i] +'+' +accountHolderName[i];
micro(url ,filter[i],function(resp)
{
this.resutlObject[count]=resp;
console.log("count"+count);
count=count+1;
}.bind( {resutlObject: resutlObject} ));
}//end of for
setTimeout(function () {
console.log("in time out");
res.end(JSON.stringify(resutlObject || {}, null, 2));
},500);
Also you could use Promises. Suppose service call returns promise, then you wait while all of them are fulfilled. Node.js supports promises starting from v4. If you have earlier version of node, just use some library.
//Instead of
function micro(url, filter, cb) {
var resp = "result of async job";//do some async work
cb(resp)
}
//Modify your service to return a promise
function micro(url, filter) {
return new Promise(function(resolve, reject) {
var resp = "result of async job using `url` and `filter`";
if (resp) {
resolve(resp);
} else {
reject("reason");
}
});
}
//Create a list of service calls.
var promises = [];
for( i=0; i<length; i++)
{
url=accountID[i] +'+'+sortcode[i] +'+' +accountHolderName[i];
promises.push(micro(url, filter[i]));
}
//Wait for all off them to fulfill
Promise.all(promises)
.then(function(resultObject) {
//Response
res.end(JSON.stringify(resultObject || {}, null, 2));
}, function(reason) {
res.sendStatus(500);
console.error(reason);
});
you can use async module async. It provides the parallel foreach loop.
var obj = {dev: "/dev.json", test: "/test.json", prod: "/prod.json"};
var configs = {};
async.forEachOf(obj, function (value, key, callback) {
fs.readFile(__dirname + value, "utf8", function (err, data) {
if (err) return callback(err);
try {
configs[key] = JSON.parse(data);
} catch (e) {
return callback(e);
}
callback();
})
}, function (err) {
if (err) console.error(err.message);
// configs is now a map of JSON data
doSomethingWith(configs);
})
here in the example it is reading files listed in parameters.
similarly you can do for your task
You could use async module. It's designed to do the stuff you're after. Something like this:
var async = require('async');
var collection = [];
for(i=0;i<length;i++) {
collection.push(
(function(i) {
return function(callback) {
url=accountID[i] +'+'+sortcode[i] +'+' +accountHolderName[i];
micro(url ,filter[i],function(resp) {
callback(null, resp);
});
}
})(i)
);
}//end of for
async.parallel(collection, function(err, results) {
console.log(results) // array of results from all requests
})
What happens
async.parallel takes an array of functions as an argument. Each function receives callback as an argument. Callback is a function, which takes error and result as an argument.
After all callback are executed async calls the final callback which receives the array of results from all other callbacks.
In the loop we are creating just that, a collection of functions. In this example the code is a bit more complex because we use closure in order to preserve the value of i for each of these functions.
I have a loop that I need to run inside of a callback, unfortunately accessing the array outside of the callback leaves me with a blank array. I know why this happens, but I want to know the best solution to tackle this.
Gallery.prototype.getGallery = function(cb) {
self = this;
var cos = new pb.CustomObjectService();
var ms = new pb.MediaService();
var s = [];
cos.loadTypeByName('Gallery Image', function(err, gallery){
cos.findByType(gallery._id.toString(), function(err, rpy){
for(var i = 0; i < rpy.length; i++){
ms.loadById(rpy[i].Image, function(e,r){
s.push(r.location);
console.log(r.location); /* <-- logs expected data */
});
}
console.log(s[0]); /* <-- this is undefined */
});
});
};
Replace your for loop with a call to async.*; in this case async.map seems right. Pass a callback to async.map; it will be invoked when all the individual calls to ms.loadById are done, with the array of results.
async.map(
rpy,
function(elt, callback) {
ms.loadById(elt.Image, callback);
},
function(err, data) {
// comes here after all individual async calls have completed
// check errors; array of results is in data
}
);
If you want to go into the promises world, then wrap the calls to ms.loadById in a promise. Here's a roll-your-own version, but various versions of what is usually called promisify are also out there.
function loadByIdPromise(elt) {
return new Promise(function(resolve, reject) {
ms.loadById(elt.image, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
Then do a Promise.all on the resulting promises:
Promise.all(rpy.map(loadByIdPromise))
.then(function(data) {
// comes here when all individual async calls complete successfully
// data is your array of results
});
Using the promises style, your entire code would look like:
loadTypeByNamePromise('Gallery Image') .
then(function(gallery) { return findByTypePromise(gallery._id.toString(); }) .
then(function(rpy) { return Promise.all(rpy.map(loadByIdPromise)); }) .
then(function(results) { /* do something with [results] */ });
This is what I would like to do
var response = [];
Model.find().then(function(results){
for(r in results){
MyService.getAnotherModel(results[r]).then(function(magic){
response.push(magic);
});
}
});
//when finished
res.send(response, 200);
however it returns just [] because the async stuff is not ready yet. I am using sails.js that uses Q promise. Any ideas how to return a response when all async calls are finished?
https://github.com/balderdashy/waterline#query-methods (promise methods)
Since waterline uses Q, you can use the allSettled method.
You can find more details on Q documentation.
Model.find().then(function(results) {
var promises = [];
for (r in results){
promises.push(MyService.getAnotherModel(results[r]));
}
// Wait until all promises resolve
Q.allSettled(promises).then(function(result) {
// Send the response
res.send(result, 200);
});
});
You simply can't do that, you have to wait for the asynchronous functions to complete.
You can either create something yourself, or use the async middleware, or use built in features, as noted in Florent's answer, but I'll add the other two here anyway :
var response = [];
Model.find().then(function(results){
var length = Object.keys(results).length,
i = 0;
for(r in results){
MyService.getAnotherModel(results[r]).then(function(magic){
response.push(magic);
i++;
if (i == length) {
// all done
res.send(response, 200);
}
});
}
});
or with async
var response = [];
Model.find().then(function(results){
var asyncs = [];
for(r in results){
asyncs.push(function(callback) {
MyService.getAnotherModel(results[r]).then(function(magic){
response.push(magic);
callback();
})
});
}
async.series(asyncs, function(err) {
if (!err) {
res.send(response, 200);
}
});
});
Take a look at jQuery deferred objects:
http://api.jquery.com/category/deferred-object/
Specifically, .when()
http://api.jquery.com/jQuery.when/
I have code like
common.findOne('list', {'listId': parseInt(request.params. istId)}, function(err, result){
if(err) {
console.log(err);
}
else {
var tArr = new Array();
if(result.tasks) {
var tasks = result.tasks;
for(var i in tasks) {
console.log(tasks[i]);
common.findOne('tasks', {'taskId':parseInt(tasks[i])}, function(err,res){
tArr[i] = res;
console.log(res);
});
}
console.log(tArr);
}
return response.send(result);
}
});
It is not executed sequentially in node.js so I get an empty array at the end of execution. Problem is it will first execute console.log(tArr); and then execute
common.findOne('tasks',{'taskId':parseInt(tasks[i])},function(err,res){
tArr[i] = res;
console.log(res);
});
Is there any mistake in my code or any other way for doing this.
Thanks!
As you are probably aware, things run asynchronously in node.js. So when you need to get things to run in a certain order you need to make use of a control library or basically implement it yourself.
I highly suggest you take a look at async, as it will easily allow you to do something like this:
var async = require('async');
// ..
if(result.tasks) {
async.forEach(result.tasks, processEachTask, afterAllTasks);
function processEachTask(task, callback) {
console.log(task);
common.findOne('tasks', {'taskId':parseInt(task)}, function(err,res) {
tArr.push(res); // NOTE: Assuming order does not matter here
console.log(res);
callback(err);
});
}
function afterAllTasks(err) {
console.log(tArr);
}
}
The main things to see here is that processEachTask gets called with each task, in parallel, so the order is not guaranteed. To mark that the task has been processed, you will call callback in the anonymous function from findOne. This allows you to do more async work in processEachTask but still manage to signify when it is done. When every task is done, it will then call afterAllTasks.
Take a look at async to see all the helper functions that it provides, it is very useful!
I've recently created a simple abstraction named "wait.for" to call async functions in sync mode (based on Fibers): https://github.com/luciotato/waitfor
Using wait.for and async your code will be:
var wait = require('waitfor');
...
//execute in a fiber
function handleRequest(request,response){
try{
...
var result = wait.for(common.findOne,'list',{'listId': parseInt(request.params.istId)});
var tArr = new Array();
if(result.tasks) {
var tasks = result.tasks;
for(var i in tasks){
console.log(tasks[i]);
var res=wait.for(common.findOne,'tasks',{'taskId':parseInt(tasks[i])});
tArr[i] = res;
console.log(res);
}
console.log(tArr);
return response.send(result);
};
....
}
catch(err){
// handle errors
return response.end(err.message);
}
};
// express framework
app.get('/posts', function(req, res) {
// handle request in a Fiber, keep node spinning
wait.launchFiber(handleRequest,req,res);
});