rate limit of github using node.js - node.js

Question

I am writing an app using node.js which sends a request to github and fetches the html page of github project issues. when I send more request for 40th page I am getting 429 response for request. how can i overcome this RATE limit of github?
function requestPage(pageNo){
var changedUrl = url+"?page="+pageNo+"&q=is%3Aissue+is%3Aopen"; //URL for requesting all the pages individually
request(changedUrl, function(error, response, html){ //requesting thee web page
if(error){
return error;
}
else{
var $ = cheerio.load(html);
if(pageNo == 40){
console.log(response.statusCode);
fs.writeFile("page.html", html ,'utf8',function(err){
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
}
//functions
}
});
}
for (var i = 1; i <= noOfPages; i++) {
requestPage(i);
}

Answer 1

using the right tool for the right job
First, please note that you should better use github API instead of requesting the HTML pages. It should make your work easier, as you'll directly get the data you are interested in in JSON format, without having to parse HTML.
Issues API endpoints are documented here.
You also should check whether github allows you to crawl their page, and make sure you don't overload their servers with your requests.
fixing your code
However, in case you want to continue the way you're doing, you arrange your code this way:
handle rate limiting
Inside request function callback, you should add a condition to check the response from github:
request(changedUrl, function(error, response, html){
if(error){
return error;
}
else{
if (response.headers.status === '429 Too Many Requests') {
setTimeout(function() {requestPage(pageNo)}, 54000000}) // retry in some delay (find out the appropriate timeout value)
}
else {
... //continue the processing
behaving well
Another change required in your code may avoid you from being rate-limited. It should also spare your CPU and github servers' too.
This is bad:
for (var i = 1; i <= noOfPages; i++) {
requestPage(i);
}
Reason: your are sending noOfPages requests to github quasi-simultaneously.
How to fix: use a recursive function instead of a for loop, and set a delay between the calls.
Fixed code:
function requestPage(pageNo){
var changedUrl = url+"?page="+pageNo+"&q=is%3Aissue+is%3Aopen"; //URL for requesting all the pages individually
request(changedUrl, function(error, response, html){ //requesting thee web page
if(error){
return error;
}
else{
if (response.headers.status === '429 Too Many Requests') {
// retry to request the same page after some delay
setTimeout(function() {requestPage(pageNo)}, 54000000})
}
else {
var $ = cheerio.load(html);
if(pageNo == 40){
console.log(response.statusCode);
fs.writeFile("page.html", html ,'utf8',function(err){
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
if (i < noOfPages) {
// request next page after a small delay
setTimeout(function() {requestPage(pageNo+1)}, 15000})
}
}
}
});
}
// request the first page immediately
requestPage(1);
I leave it up to you to cleanup the hard-coded if(pageNo == 40){, which probably should be if(pageNo == noOfPages){ and could be arranged with the following if clause.

Answer 2

In ES7 with babel, just one page at a time:
import req from 'request-promise';
async function getPages(urls) {
for (let url of urls) {
let html = await req(url);
console.log(html);
}