Node.js For Loop async request - node.js

Question

I have for loop with a request (and another request in callback).
I have problem with memory usage (when i am go through a lot of request at same time)
here is the sample code :
var request = require('request');
for(var j=1;j<=10;j++){
request({
method: 'GET',
url: 'https://api.domain.com/items/?page='+j+'&limit=1000',
headers: {
'Content-Type': 'application/json'
}}, function (error, response, body) {
var data = JSON.parse(body)
for(var i=0;i<data.length;i++){
request({
method: 'GET',
url: 'https://api.domain.com/itemDetail/'+data[i].id,
headers: {
'Content-Type': 'application/json',
}}, function (error, response, body) {
var itemDetails = JSON.parse(body);
// save items to mongodb
}
}
});
}
The solution suggested to me it was using async module.
After reading documentation i find out eachlimit suit my needs.
The problem i have its i can use eachlimit for second loop but for first loop i dont know how use it (because first loop in not an array).
here code so far i get , and its not working:
var request = require('request');
var mongodb = require('mongodb');
var async = require('async');
var MongoClient = mongodb.MongoClient;
var url = 'mongodb://localhost:27017/subtitle';
for(var j=1;j<=10;j++){
request({
method: 'GET',
url: 'https://api-v2.domain.com/news/popular?page='+j+'&limit=1000',
headers: {
'Content-Type': 'application/json',
'api-version': '2',
'api-key': 'my-api-key'
}}, function (error, response, body) {
try {
var data = JSON.parse(body);
} catch (e) {
// An error has occured, handle it, by e.g. logging it
console.log(e);
}
async.each(data, function(item, callback) {
request({
method: 'GET',
url: 'https://api-v2.domain.com/news/'+item.id,
headers: {
'Content-Type': 'application/json',
'api-version': '2',
'api-key': 'my-api-key'
}}, function (error, response, body) {
// sava item in mongodb
})
callback();
}, function(err){
if( err ) {
console.log('A item failed to process');
} else {
console.log('All items have been saved successfully');
}
});
})
}

Answer 1

You can achieve this in a number of ways, one of them would be using whilst. With your code it'd look something like this (removed error handling and request options for readability here):
var requestCount = 0;
async.whilst(
function () {
return requestCount < 10;
},
function (firstCallback) {
request({url: "https://api-v2.domain.com/news/popular?page=" + requestCount + "&limit=1000"}, function (err1, res1, body1) {
var data = JSON.parse(body1);
async.each(
data,
function(item, secondCallback) {
request({url: "https://api-v2.domain.com/news/" + item.id}, function (err2, res2, body2) {
// sava item in mongodb
secondCallback();
});
},
function(err){
requestCount++;
firstCallback();
}
);
});
},
function (err) {
//all requests done
}
);
Once you start to nest so many callbacks, you should probably refactor this and put the second loop in it's own function. For example:
var requestCount = 0;
async.whilst(
function () {
return requestCount < 10;
},
function (callback) {
request({url: "https://api-v2.domain.com/news/popular?page=" + requestCount + "&limit=1000"}, function (err, res, body) {
var items = JSON.parse(data);
getNews(items, function(newsErr){
requestCount++;
callback(newsErr);
});
});
},
function (err) {
//all requests done
}
);
//----------------------------------------------------
function getNews(items, callback){
async.each(items, function(item, itemCallback) {
request({url: "https://api-v2.domain.com/news/" + item.id}, function (err2, res2, body2) {
// sava item in mongodb
itemCallback();
});
}, callback);
}
Also make sure to call the async callbacks within the callbacks of the request (you weren't doing it so in your code).