Node.js Async mapLimit and memory - node.js

Question

SOLVED, see the answer please.
I have a list of urls I fetch using request and for some reason I am unable to save more than 1720 records into my database when I try to fetch 2000 or more URL's at a time.
If I try 1000 to 2000 and 2000 to 3000, I get 3000 results in total. But when I try 1000 to 3000 or 4000 to 6000, my script stops after fetching the 1720th result.
What could be the reason for that?
I use mapLimit in order to limit concurrent connections.
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 1000; a < 3000; a++) {
people.push("http://www.example.com/" + a + "/person.html");
}
async.mapLimit(people, 20, function(url, callback) {
// iterator function
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function(error, response, body) {
if (!error && response.statusCode == 200) {
async.series([
// add this person into database
function(callback) {
var $ = cheerio.load(body);
var name = entities.decodeHTML($('span[itemprop="name"]').text());
new person({
name: name,
url: url
}).save();
callback();
},
function(callback) {
async.waterfall([
function(callback) {
var $ = cheerio.load(body);
var jobs = $('span[itemprop="jobtitle"]').length;
if (jobs == 0) {
console.log("no job");
var jobsArr = 0;
} else {
var jobsArr = [];
for (var aa = 0; aa < jobs; aa++) {
jobsArr.push(entities.decodeHTML($('span[itemprop="jobtitle"]').eq(aa).text()));
}
}
callback(null, jobsArr);
},
function(jobsArr, callback) {
if (jobsArr == 0) {
console.log("this person has no jobs");
} else {
async.map(jobsArr, function(jobs, callback) {
personRole.where('job_name', jobs).fetch({
require: true
}).then(function(data1) {
data1 = data1.toJSON();
person.where('url', url).fetch().then(function(data2) {
data2 = data2.toJSON();
new personPersonRole({
person_id: data2.id,
personrole_id: data1.id
}).save();
});
}).catch(function(err) {
new personRole({
job_name: jobs
}).save().then(function(data3) {
data3 = data3.toJSON();
person.where('url', url).fetch().then(function(data4) {
data4 = data4.toJSON();
new personPersonRole({
person_id: data4.id,
personrole_id: data3.id
}).save();
});
});
});
});
}
callback(null, "yes");
}
], function(err, result) {
if (err) {
console.log(err);
}
});
callback();
}
], function(err, result) {
if (err) {
console.log("err3");
}
});
} else {
console.log("err4");
}
});
callback();
});
});
EDIT #2
The following code is also problematic, adds only 1747 records and it stops after that. If I stop my node app and start again, it also stops at 1747.
var person = require('./models').person;
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("random url");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
callback();
});
});
db.js
var knex = require('knex')({
client: 'mysql',
connection: {
host: '127.0.0.1',
port: 8889,
user: 'root',
password: 'root',
database: 'mydatabase',
charset: 'utf8'
},
pool: {
min: 0,
max: 100
}
});
var db = require('bookshelf')(knex);
module.exports = db;
models.js
var person = db.Model.extend({
tableName: 'people'
});
module.exports = {
person : person
};
EDIT #3
Okay, I think I've found the solution.
18K-18.5K - no problem
18K-19K - no problem
18K-19.7K - no problem
18K-20K - RangeError: Maximum call stack size exceeded at new Object
(native)
I just wrapped my callbacks into a wrapper, like below.
async.setImmediate(function () {
callback();
});
app.get('/async22', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("yes");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
async.setImmediate(function () {
callback();
});
});
});

Answer 1

It was in front of my eyes all the time, actually this solution isn't unique, it's already included async library's database.
https://github.com/caolan/async#common-pitfalls-stackoverflow
Here's how you do it.
async.setImmediate(function () {
callback();
});
Example
app.get('/async22', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("yes");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
async.setImmediate(function () {
callback();
});
});
});

Answer 2

This is still not an answer, but it is too big for the comment.
I suggest to reduce the code to some minimal example and try if it works (example is below and it works for me).
Second thing - is to add a monitoring route (see the /apptest below), so you can check if you app still works and the processing progress.
If the minimal sample works, start to gradually add more code with your logic to it and check if it still works.
The code, server.js:
var util = require('util');
var express = require('express');
var async = require('async');
var request = require('request');
var cheerio = require('cheerio');
var app = express.createServer();
app.successCount = 0;
app.errorCount = 0;
app.get('/apptest', function(req, res) {
res.send(
util.format(
'I am OK, successCount: %s, errorCount: %s',
app.successCount, app.errorCount
), 200
);
});
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 1000; a < 3000; a++) {
people.push("http://www.example.com/" + a + "/person.html");
}
async.mapLimit(people, 20, function(url, callback) {
// iterator function
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function(error, response, body) {
if (!error) {
console.log('success requesting: ' + options2.url);
var $ = cheerio.load(body);
app.successCount += 1;
} else {
console.log(
'error requesting: %s, error: %s, status: %s',
options2.url, error, response.statusCode
);
app.errorCount += 1;
}
callback();
});
});
});
app.listen(3000, function() {
console.log(
"Express server listening on port %d in %s mode",
app.address().port, app.settings.env
);
});
Dependencies, package.json:
{
"name": "application-name",
"version": "0.0.1",
"private": true,
"dependencies": {
"async": "^1.5.2",
"cheerio": "^0.19.0",
"express": "2.5.8",
"request": "^2.67.0"
},
"devDependencies": {}
}
Run the example as node server.js and then open http://localhost:3000/asynctest in the browser, you should see success requesting: xxxx in the console. While it is running (or when it stops running) - open http://localhost:3000/apptest to check if app is OK and how many urls are processed.