Node.js promises and bookshelf.js - node.js

Question

I'm new into Promises.
I use Bookshelf.js as ORM.
I fetch a number of webpages, get person info (about actors) from those pages and add them into my database if they don't exist.
But there's a problem, even though console.log(name) returns actor names in the right order, my query checks for only one actor, the latest one, which is 9.
What's wrong here?
var entities = require("entities");
var request = require('request');
var cheerio = require('cheerio');
// create promisified version of request()
function requestPromise(options) {
return new Promise(function (resolve, reject) {
request(options, function (err, resp, body) {
if (err) return reject(err);
resolve(body);
});
});
}
var person = require('./models').person;
app.get('/fetch', function (req, res) {
var promises = [];
var headers = {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
};
for (var i = 1; i < 10; i++) {
promises.push(requestPromise({url: "http://www.example.com/person/" + i + "/personname.html", headers: headers}));
}
Promise.all(promises).then(function (data) {
// iterate through all the data here
for (var i = 0; i < data.length; i++) {
if ($ = cheerio.load(data[i])) {
var links = $("#container");
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("null name returned, do nothing");
} else {
name = entities.decodeHTML(name);
console.log(name); // returns names in the right order
// does this person exist in the database?
person.where('id', i).fetch().then(function (result) {
if (result) {
console.log(i + "exists");
} else {
console.log(i + " doesn't exist");
// returns "9 doesn't exists" 9 times instead of
// checking each ID individually, why?
}
});
}
} else {
console.log("can't open");
}
}
}, function (err) {
// error occurred here
console.log(err);
});
});
EDIT #2
Now the order is broken and my ID's aren't the same with the site's I fetch data from. I see ID's like 11 and 13 even though I iterate from 1 to 5 and it seems to overrule something since it adds duplicate entries.
Here's what I'm trying to do in a nutshell. "Visit these urls in order and add the data you fetch (e.g. names) in the same order (id1 = name1; id2 = name2, etc) to the database".
app.get('/fetch', function (req, res) {
var promises = [];
var headers = {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
};
for (var i = 1; i < 5; i++) {
promises.push(requestPromise({url: "http://example.com/person/ + i + "/personname.html", headers: headers}));
}
Promise.all(promises).then(function (data) {
// iterate through all the data here
data.forEach(function (item, i) {
var $ = cheerio.load(item);
var name = $("#container span[itemprop='name']").text();
if (!name) {
console.log("null name returned, do nothing");
} else {
// name exists
person.where('id', i).fetch({require: true}).then(function (p) {
console.log(i + " exists");
}).catch(function () {
console.log(i + " does not exist");
new person({id: i, name: name}).save(null, {method: 'insert'}).then(function () {
console.log("success" + i);
});
});
}
}, function (err) {
// error occurred here
console.log(err);
});
});
});

Answer 1

When you run your code through jshint, you will see a warning that says
Don't make functions within a loop.
In this piece of code the callback inside then does not run in sync with the enclosing for loop. It runs whenever the database has fetched your result.
person.where('id', i).fetch().then(function (result) {
if (result) {
console.log(i + "exists");
} else {
console.log(i + " doesn't exist");
}
});
Therefore, when that callback runs eventually, the loop has long finished. Your callback function holds a reference to the loop counter i - which, by now, has the value 9.
It's better to use a function that accepts a parameter than to refer to a loop counter.
Luckily node makes this easy, you can use the forEach array function:
data.forEach(function (item, i) {
var $ = cheerio.load(item);
var name = $("#container span[itemprop='name']").text();
if (!name) {
console.log("null name returned, do nothing");
} else {
console.log("successfully scraped name: " + name);
person.where('id', i).fetch({require: true}).then(function (p) {
console.log(i + " exists");
}).catch(function () {
console.log(i + " does not exist");
});
}
});
Note that you can make Bookshelf.js throw instead of silently passing over non-existing records with {require: true}.
More generally speaking, I don't see a real connection between scraping the name from a website and retrieving a model from the database. These two things should probably be done in separate functions that each return an individual promise for the respective thing. That way, requests to the database can run in parallel with requests to the web server.

Answer 2

It looks like you need a closure for person.where('id', i).fetch().
also use node-fetch instead of hand rolling request-promise.