Node.js : How to pause/resume a for loop - node.js

Question

I am running a for loop to generate all possible IPv4 addresses and then looking them up in maxmind's mmdb file. The problem is, while the for loop is too fast, maxmind lookup is relatively very slow and hence the processs slows down and eventually my system freezes after some 400k iterations. If I were reading from some stream, I would have paused the stream after reading every 10k ip addresses and would have resumed only when all of them have been looked up from mmdb file. But how can I have such control for a for loop ?
connectToMaxMindDatabases().then(function (done){
for(var i=0; i<256; i++){
for(var j=0; j<256; j++){
for(var k=0; k<256; k++){
for(var l=0; l<256; l++){
count++;
if(count % 1000 == 0){
console.log("count", count);
}
var newIP = getIPV4([i,j,k,l])
ispDB.getGeoDataAsync(newIP).then(function (result){
if(result){
console.log(count, newIP, result);
// process.exit();
}
});
}
}
}
}
})
function getIPV4(bytes){
return bytes.join(".")
}

Answer 1

I'm not sure, but process.nextTick might help
https://nodejs.org/api/process.html#process_process_nexttick_callback_arg
I'm really not sure, but the following code might be used:
connectToMaxMindDatabases().then(function (done){
for(var i=0; i<256; i++){
process.nextTick(function(){
for(var j=0; j<256; j++){
process.nextTick(function(){
for(var k=0; k<256; k++){
process.nextTick(function(){
for(var l=0; l<256; l++){
process.nextTick(function(){
count++;
if(count % 1000 == 0){
console.log("count", count);
}
var newIP = getIPV4([i,j,k,l])
ispDB.getGeoDataAsync(newIP).then(function (result){
if(result){
console.log(count, newIP, result);
// process.exit();
}
});
}
}
})
}
}
}
}
}
})
According to my knowledge,here's what happening:
all the for loops are running synchronously, and the async ispDB.getGeoDataAsync(newIP) call is getting queued, but its callback(.then function) is not able to execute because the event loop was blocked by the 4 for loops.

Answer 2

The root of the problem is definitely in resource management of the underlying OS/Node server. A for loop with such iteration number will freeze on that particular hardware while it probably would only freeze later or none at all on a much stronger computer. The solution is to collect all IP addresses in an array first, then process them in a way that only a segment is processed at once, wait until that is finished and move to the next segment.
A possible solution can be achieved with this kind of segmentation and Bluebird's Promse.each.
If the iterator function returns a promise or a thenable, then the
result of the promise is awaited, before continuing with next
iteration.
var Promise = require('bluebird'),
_ = require('lodash'),
allIPs = [];
connectToMaxMindDatabases().then(function (done){
for(var i=0; i<256; i++){
for(var j=0; j<256; j++){
for(var k=0; k<256; k++){
for(var l=0; l<256; l++){
count++;
if(count % 1000 == 0){
console.log("count", count);
}
var newIP = getIPV4([i,j,k,l])
allIPs.push(newIP);
}
}
}
}
return allIPs;
})
.then(function(allIPs) {
// Create an array of arrays of IPs, each inner array with 1000 IPs
var from = 0, to = 1000, segments = [], promises;
do {
segments.push(allIPs.slice(from, to));
from += 1000; to += 1000;
} while(to < allIPs.lenght);
// Process a segment of 1000 IPs and wait before moving the next
// segment until this segment is fully resolved.
Promise.each(segments, function(segmentArr) {
promises = [];
_.forOwn(segmentArr, function(IP) {
promises.push(ispDB.getGeoDataAsync(IP)
.then(function(result) {
// save the result
})
);
});
return Proimse.all(promises);
});
});
Of course if the resources in your case are limited in a way the allIPs array cannot even be produced because it starts freezing before that, this will not solve the problem, but chances are the resources are eaten up by all those getGeoDataAsync calls in which case this should help.