nodejs request running last - node.js

Question

I am trying to figure out how to make the callback function in request run in order. Currently, my loop runs 10 times but does not wait for the callback function in request to finish before moving to the next iteration. My output is nothing like what I'd expect it to be and I'm not sure why certain things are being printed before others. Here is how my code is as of now:
var express = require('express');
var path = require('path');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var app = express();
var url;
for(var i=0; i < 10; i++ ){
url = "http://www.ratemyprofessors.com/ShowRatings.jsp?tid=" + i;
request(url, function(err, resp, body){
console.log("hello");
var $ = cheerio.load(body);
if($('.error').text().substring(0, 14) == "Page Not Found"){
console.log("sorry page not found");
return;
}else{
console.log($('.error').text().substring(0, 14) );
var pfname = $('.pfname');
var plname = $('.plname');
var professorName = pfname.text().replace(/\s/g, '') + " " +plname.text().replace(/\s/g, '');
console.log(professorName);
console.log(url);
return;
}
});
}
Here is the output I am getting:
hello
sorry page not found
hello
sorry page not found
hello
sorry page not found
hello
sorry page not found
hello
sorry page not found
hello
sorry page not found
hello
sorry page not found
hello
Michael Beeson
http://www.ratemyprofessors.com/ShowRatings.jsp?tid=9
hello
Sami Khuri
http://www.ratemyprofessors.com/ShowRatings.jsp?tid=9
hello
aaa aaa
http://www.ratemyprofessors.com/ShowRatings.jsp?tid=9
Here is the proper output:
aaa aaa
http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1
Sami Khuri
http://www.ratemyprofessors.com/ShowRatings.jsp?tid=2
Michael Beeson
http://www.ratemyprofessors.com/ShowRatings.jsp?tid=3
sorry page not found
sorry page not found
sorry page not found
sorry page not found
sorry page not found
sorry page not found
sorry page not found

Answer 1

There are multiple issues in your code, but the main issue is that you're running an async operation inside the for loop so your for loop will start all the async operations and then they will, one-by-one complete later. Any variables shared between the loop invocations will tromp one another.
So, in a nutshell, I did:
Removed all shared variables so each loop invocation has its own variables (no conflicts).
Switched over to request-promise so we can use Promise.all() to more easily tell us when they are all done.
Returned the value we want from each .then() handler so that will be collected by Promise.all() as the final values for each invocation of the loop.
Because there appears to be no reason to sequence your operations, I let them all run in a parallel (that's faster) and then let Promise.all() put the results in order for us in the final array of results.
Here's the code:
const express = require('express');
const path = require('path');
const rp = require('request-promise');
const cheerio = require('cheerio');
const fs = require('fs');
const app = express();
let promises = [];
for (let i = 0; i < 10; i++ ) {
let url = "http://www.ratemyprofessors.com/ShowRatings.jsp?tid=" + i;
promises.push(rp(url).then(function(body) {
console.log(url);
let $ = cheerio.load(body);
if($('.error').text().substring(0, 14) == "Page Not Found"){
console.log("sorry page not found");
return null;
} else {
console.log($('.error').text().substring(0, 14) );
let pfname = $('.pfname');
let plname = $('.plname');
let professorName = pfname.text().replace(/\s/g, '') + " " +plname.text().replace(/\s/g, '');
console.log(professorName);
return professorName;
}
}));
}
// see when they are all done
Promise.all(promises).then(results => {
// array of results, some entries that were not found may be null
console.log(results);
}).catch(err => {
console.log(err);
});
If you want to sequence them one at a time so the second request doesn't start until the first one is done, that could be done like this using async/await:
const express = require('express');
const path = require('path');
const rp = require('request-promise');
const cheerio = require('cheerio');
const fs = require('fs');
const app = express();
async function run() {
let results = [];
for (let i = 0; i < 10; i++ ) {
let url = "http://www.ratemyprofessors.com/ShowRatings.jsp?tid=" + i;
try {
let body = await rp(url);
console.log("hello");
let $ = cheerio.load(body);
if($('.error').text().substring(0, 14) == "Page Not Found"){
console.log("sorry page not found");
results.push(null);
} else {
console.log($('.error').text().substring(0, 14) );
let pfname = $('.pfname');
let plname = $('.plname');
let professorName = pfname.text().replace(/\s/g, '') + " " +plname.text().replace(/\s/g, '');
console.log(professorName);
console.log(url);
results.push(professorName);
}
} catch(e) {
console.log(url, e);
results.push(null);
}
}
return results;
}
run().then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
EDIT Jan, 2020 - request() module in maintenance mode
FYI, the request module and its derivatives like request-promise are now in maintenance mode and will not be actively developed to add new features. You can read more about the reasoning here. There is a list of alternatives in this table with some discussion of each one. I have been using got() myself and it's built from the beginning to use promises and is simple to use.