Why does this Bottleneck-ed app stall? - node.js

Question

I have the app below and it stalls (code below). And I have no idea why. I suspect I might be using the Bottleneck module the wrong way.
Disclaimer: I am trying to learn programming and NodeJS myself using this project. Please help.
Intro
The point of the app is to fetch data missing from documents in a DB by requesting a webpage and parsing it jQuery-style. Then saving the returned data to new keys in the document. The database consists of ~92 000 documents. The app uses the bottleneck, cheerio and request modules. I run the app on OS X.
The problem
If I set a limit to the number of requests, such as
var limiter = new bottleneck(5, 0);
The app stalls after the first batch (5 in this case). But why? I suspect something might be wrong with Bottleneck and how it expects my program to work. Something to do with callbacks per Bottleneck "Gotchas" maybe?
If I set no limit, the app kind-of-works. It fetches webpages and writes to the DB. However with a lot of errors due to resources being overloaded and thus slowly. This is how I tell bottleneck not to limit:
var limiter = new bottleneck(0, 0);
These are the kind of errors I get:
{ [Error: getaddrinfo ENOTFOUND www.vestnikverejnychzakazek.cz www.vestnikverejnychzakazek.cz:443]
code: 'ENOTFOUND',
errno: 'ENOTFOUND',
syscall: 'getaddrinfo',
hostname: 'www.vestnikverejnychzakazek.cz',
host: 'www.vestnikverejnychzakazek.cz',
port: 443 }
{ [Error: connect EMFILE 65.52.146.11:443 - Local (undefined:undefined)]
code: 'EMFILE',
errno: 'EMFILE',
syscall: 'connect',
address: '65.52.146.11',
port: 443 }
App code
'use strict';
var express = require('express');
var router = express.Router();
var assert = require('assert');
var mongo = require('mongoskin');
var path = require('path');
var ObjectID = require('mongodb').ObjectID;
var db = mongo.db("mongodb://localhost:27017/zak", {
native_parser: true
});
var database = db.collection("zakazky");
var cheerio = require("cheerio");
var request = require("request");
var fs = require("fs");
var toJs = (path.join(__dirname, '../public/javascripts', 'jquery.min.js'));
var jquery = fs.readFileSync(toJs).toString();
var bottleneck = require("bottleneck");
var limiter = new bottleneck(5, 0);
/* GET home page. */
router.get('/', function(req, res) {
var cursor = database.find();
cursor.each(function(err, data) {
assert.equal(err, null);
if (data != null) {
var vvz = "vestnikverejnychzakazek";
var praha = "zakazky.praha.eu";
var id = data["_id"];
var zdroj = data["zdroj"];
if (zdroj.indexOf(vvz) > -1) {
if ((data["cpv"] == null) || (data["predpokladana_hodnota"] == null)) {
limiter.submit(getCPV, id, zdroj, null);
// getCPV(id, zdroj);
} else {
// console.log("we're good");
return
}
} else if (zdroj.indexOf(praha) > -1) {
// console.log("pha");
}
} else {
// callback();
}
});
var getCPV = function(id, zdroj, callback) {
console.log("CPV started");
var zdroj = zdroj.replace("http://", "https://");
console.log("zdroj: " + zdroj);
var cpv = [];
var retryWrapper = function(retries) {
var retries; // I added this
if (retries === 3) {
return;
} else if (retries === undefined) {
retries = 0;
} else if (retries > 0) {
console.log("trying again");
}
request(zdroj, function(err, resp, data) {
if (err) {
console.log(err);
return retryWrapper(retries + 1);
}
var $ = cheerio.load(data);
var predpokladnaHodnota = $("[id*='Hodnota1_']").first().attr("value");
$("[id*='HlavniSlovnik']").each(function() {
cpv.push(this.attribs.value);
});
// let's check what we've got is actual data
if (cpv.length === 0) {
return
} else {
// send it off
writeCPV(id, "cpv", cpv)
}
if (predpokladnaHodnota == undefined || predpokladnaHodnota == null) {
return
} else {
// send it off
writeCPV(id, "predpokladana_hodnota", predpokladnaHodnota)
}
callback();
});
}; // end of retryWrapper
retryWrapper();
};
var writeCPV = function(id, key, value) {
id = ObjectID(id);
(function() {
console.log("starting DB write 1");
database.update({
"_id": id
}, {
$set: {
[key]: value
}
}, function(err, results) {
if (err) {
console.log("error in Mongo DB: \n------------------------\n" + err);
}
console.log("Mongo success!:\n ----------------------\n" + results);
// callback();
});
})();
};
// send the browser we're done
res.sendStatus(200);
});
// ---------------------
module.exports = router;
Here is a sample document from the DB including the fetched keys:
{
"_id": ObjectId("568d91396912101c1007ab4e"),
"cena": 1636363,
"cena_celkem": 1500000,
"cena_dopocitano": false,
"created": "2015-04-07T13:45:10.420739",
"datum_zadani": "2015-02-16",
"dodavatel": "/api/v1/dodavatel/381836/",
"druh_rizeni": "/api/v1/druh_rizeni/1116/",
"id": 1312587,
"modified": "2015-04-18T14:22:10.765733",
"nazev": "Pohostinství",
"pocet_nabidek": 2,
"podporeno_eu": true,
"popis": "Kurzy v oblasti pohostinství (formou profesní kvalifikace)",
"ramcova_smlouva": true,
"resource_uri": "/api/v1/zakazka/1312587/",
"skupina": "490648-ISVZUS_2011",
"typ_zakazky": "/api/v1/typ_zakazky/193/",
"zadavatel": "/api/v1/zadavatel/131528/",
"zdroj": "http://www.vestnikverejnychzakazek.cz/en/Form/Display/568547",
"zdroj_nazev": "isvzus.cz",
"cpv": ["80000000-4", "80400000-8", "", "", ""],
"predpokladana_hodnota": "1 500 000,00"
}
Sample URL being requested:
http://www.vestnikverejnychzakazek.cz/en/Form/Display/568547

Answer 1

This has been up here for a bit but in case anyone else stumbles upon this hopefully this will help someone!
The limiter is calling getCPV and it does call the callback at the end of the sequence, however, there are some conditional statements in retryWrapper that would allow an early return and as a result, never call the callback. The limiter will get piled up until it fires, so always make sure the callback will get fired in all of the scenarios.