I'm trying to limit my use of an external API in my node.js code.
I've set up node rate limiter, but it doesn't seem to be working. I still hit 429's. What else should I be doing that I'm not?
var RateLimiter = require('limiter').RateLimiter; // Rate limits
var limiter = new RateLimiter(1, 2000); // one call every two seconds
self.riotAPI = function(options, cb){
limiter.removeTokens(1, function() {
https.request(options, function(response) {
// Error handling
response.on('error', function (e) {
console.log(e);
});
var str = '';
// Another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
// Parse and return the object
response.on('end', function () {
if(response.statusCode >= 400) {
var err = "HTTP response "+response.statusCode;
console.log(err);
cb(new Error("err"), null);
}
else {
cb(null, JSON.parse(str));
}
});
}).end();
});
}
I switched to Bottleneck and got everything functioning as desired.
self.riotAPI = function(options, cb){
limiter.submit( function(lcb) {
https.request(options, function(response) {
// Error handling
response.on('error', function (e) {
console.log(e);
});
var str = '';
// Another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
// Parse and return the object
response.on('end', function () {
if(response.statusCode >= 400) {
var err = "HTTP response "+response.statusCode;
console.log(err);
// If it's a 429, retry
if(response.statusCode == 429) {
console.log("retrying...");
self.riotAPI(options, cb);
}
// If not, fail
else {
cb(new Error("err"), null);
lcb();
}
}
else {
cb(null, JSON.parse(str));
lcb();
}
});
}).end();
}, null);
}
Related
Well, i'm visiting an array of urls making a request for each one, when one request ends the method executes the next. The array is something like this: [link1,link2,link3]. If i try to open first the link3 in browser i'll get an error (error 404) but opening the link1 and link2 first i'll have the desired response. In the browser works without problems, but isn't working in my code because i got "status:200" using the first two links, but a 404 with the third.
(If i open link2 and link2 in the browser the problem ends, but i want to do that not using the browser)
The code:
function websiteOpener(links_array, index, final) {
var methodStr = className + '::websiteOpener';
try {
log.info(methodStr + '>> Open the link: ' + links_array[index]);
var protocol;
var _host;
var rawhost;
if (links_array[index].match(/https:\/\/[^\/]+/)) {
rawhost = links_array[index].match(/https:\/\/[^\/]+/);
_host = rawhost[0].replace(/https:\/\//, '');
protocol = 'https:'
_path = links_array[index].replace(rawhost, '');
incremental = index + 1;
var options = {
host: _host,
path: _path,
method: 'GET',
headers: { 'Content-type': 'text/html' },
protocol: protocol,
agent: new https.Agent({
rejectUnauthorized: false,
})
}
} else {
incremental = index + 1;
var options =links_array[index];
}
if (incremental < final) {
if (links_array[index].match(/https:\/\/[^\/]+/)) {
var request = https.request(options, function (response) {
console.log(response.statusCode);
//if (response.statusCode === 200) {
var data;
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', function () {
websiteOpener(links_array, incremental, final);
});
//}
});
request.end();
} else {
var request = http.request(options, function (response) {
//if (response.statusCode === 200) {
var data;
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', function () {
websiteOpener(links_array, incremental, final);
});
//}
});
request.end();
}
} else {
options.headers = { 'Content-type': 'applcation/pdf' };
var request = https.request(options, function (response) {
console.log(response.statusCode);
//if (response.statusCode === 200) {
var data;
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', function () {
log.info(methodStr + '>>link found ' + links_array[index]);
});
//}
});
request.end();
}
} catch (e) {
log.error(methodStr + ">> Server error: ", e);
reject({ statusCode: 500, flag: 'ERR_PROCESS' });
}
}
Following is the node-js code used for HTTP requests. This code is giving "This deferred has already been resolved" error on production servers when I try to use requestPromise.resolve(str) in request end. Can someone please suggest what might be the error?
Libraries used : http and node-promise
var Promise = require('node-promise').Promise;
var requestPromise = new Promise();
callback = function (response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
if (!(response && response.statusCode >= 200 && response.statusCode < 300)) {
requestPromise.resolve(str);
return;
}
var resp;
try {
resp = JSON.parse(str);
} catch (ex) {
resp = str;
}
requestPromise.resolve(str);
});
});
var request = http.request(options, callback);
request.on('error', function (err) {
requestPromise.resolve(err);
});
request.write(postObject);
request.end();
I think you cannot use new Promise() (because it need resolver).
You can use this code:
new Promise(function(resolve, reject) {
callback = function (response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
if (!(response && response.statusCode >= 200 && response.statusCode < 300)) {
resolve(str);
return;
}
var resp;
try {
resp = JSON.parse(str);
} catch (ex) {
resp = str;
}
resolve(resp);
});
});
var request = http.request(options, callback);
request.on('error', function (err) {
reject(err);
});
request.write(postObject);
request.end();
});
Facebook.create executes before fbUtils.getLongTermToken finishes retrieving the long term token. This is causing validation issues when trying to create the user. Do I need to use promises to manage the asynchronous execution that's happening here?
var userData = req.body;
userData.email = userData.email.toLowerCase();
userData.fbAuthToken = fbUtils.getLongTermToken(userData.fbAuthToken);
FacebookUser.create(userData, function (err, fbUser) {
console.log(userData.fbAuthToken);
if (err) {
console.log(err.message);
if (err.toString().indexOf('E11000') > -1) {
err = new Error('Email already exists.');
}
res.status(400);
return res.sendStatus({reason: err.toString()});
} else {
res.send('success');
}
});
Here's the code for fbUtils.getLongTermToken:
getLongTermToken: function (token) {
var options = {
host: 'graph.facebook.com',
path: '/oauth/access_token?grant_type=fb_exchange_token'
+ '&client_id=' + CONSTANTS.FACEBOOK_APP_ID
+ '&client_secret=' + CONSTANTS.FACEBOOK_APP_SECRET
+ '&fb_exchange_token=' + token
};
var str = '';
http.get(options, function (res) {
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
console.log(str);
console.log(fbUtils.parseToken(str));
return fbUtils.parseToken(str);
});
}).on('error', function(err) {
console.log('longterm error: ' + err.message);
});
return str;
}
You need to update getLongTermToken to take a callback too, which will be called when the token retrieval is complete. For example:
getLongTermToken: function (token, cb) {
var options = {
host: 'graph.facebook.com',
path: '/oauth/access_token?grant_type=fb_exchange_token'
+ '&client_id=' + CONSTANTS.FACEBOOK_APP_ID
+ '&client_secret=' + CONSTANTS.FACEBOOK_APP_SECRET
+ '&fb_exchange_token=' + token
};
var str = '';
http.get(options, function (res) {
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
console.log(str);
console.log(fbUtils.parseToken(str));
return cb(fbUtils.parseToken(str));
});
}).on('error', function(err) {
console.log('longterm error: ' + err.message);
});
}
Then you need to update your calling code to wait for this token:
var userData = req.body;
userData.email = userData.email.toLowerCase();
fbUtils.getLongTermToken(userData.fbAuthToken, function (token) {
userData.fbAuthToken = token;
FacebookUser.create(userData, function (err, fbUser) {
console.log(userData.fbAuthToken);
if (err) {
console.log(err.message);
if (err.toString().indexOf('E11000') > -1) {
err = new Error('Email already exists.');
}
res.status(400);
return res.sendStatus({reason: err.toString()});
} else {
res.send('success');
}
});
});
Because fbUtils.getLongTermToken is async, Facebook.create should be called in its callback, after the token is retrieved and parsed.
getLongTermToken: function (token) {
var options = {
host: 'graph.facebook.com',
path: '/oauth/access_token?grant_type=fb_exchange_token'
+ '&client_id=' + CONSTANTS.FACEBOOK_APP_ID
+ '&client_secret=' + CONSTANTS.FACEBOOK_APP_SECRET
+ '&fb_exchange_token=' + token
};
var str = '';
http.get(options, function (res) {
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
console.log(str);
console.log(fbUtils.parseToken(str));
var parsedToken = fbUtils.parseToken(str);
// The earliest point where the token can be used
Facebook.create(...);
return parsedToken;
});
}).on('error', function(err) {
console.log('longterm error: ' + err.message);
});
return str;
}
Am using a function to save data. Using the http.request option. It is working fine. If I call the same function in a loop, some of the data not saving in database. Also getting the parse error message for some response.
How Can I call the function of http.request in a loop?
for (var i = 1; i <= 23; i++) {
turn_timer(i);
}
function turn_timer(nos) {
try {
var str = "num=" + nos;
var len = str.length;
win_settings.headers = {
'Content-length': len,
'Content-Type': 'application/x-www-form-urlencoded'
}
var request = http.request(win_settings, function(response) {
response.on('data', function(data) {
});
response.on('error', function(err) {
});
response.on('end', function() {
});
});
request.on('error', function(err) {
});
request.write(str + "\0");
request.end();
} catch (err) {
console.log(err);
}
}
Check the scope of your variable that stores message because async function call may overwrite it.
I believe your problem is because you are using a for loop, instead of going with a more asynchronous approach. Here is a quick attempt to solve your problem. I have omitted some of your code, as it seemed to be incomplete. I have left the important parts and added a few things based on an answer to a similar question.
var http = require('http'),
async = require('async');
function turn_timer(n, callback) {
var var str = "num=" + n,
len = str.length,
req;
win_settings.headers = {
'Content-length': len,
'Content-Type': 'application/x-www-form-urlencoded'
};
req = http.request(options, function(response) {
...
callback(null);
});
req.on('error', function(err) {
...
callback(err);
});
req.end();
}
async.timesSeries(23, function (n, next) {
turn_timer(options, function(err) {
next(err);
});
});
For more information, you can read more about async.timesSeries here: https://github.com/caolan/async#timesseriesn-callback
For some reason when the status code is not 200 and I return then call the callback the executed script just hangs there, not exiting. Why is that?
var http = require('http');
var qs = require('querystring');
var args = {
q: 'dfdfdf'
};
var opts = {
hostname: 'api.openweathermap.org',
path: '/data/2.5/weather?' + qs.stringify(args)
};
function cb(err, result) {
console.log(err, result);
}
http.get(opts, function(res) {
var buffer = new Buffer(0);
if (res.statusCode !== 200) return cb(new Error('Unable to fulfill request.'));
res.on('readable', function() {
return buffer = Buffer.concat([buffer, this.read()]);
});
res.on('end', function() {
return cb(null, JSON.parse(buffer.toString('utf8')));
});
});
Command line:
$ node plugins/weather.js
[Error: Unable to fulfill request.] undefined
# I have to ctrl+c at this point
You still need to consume the stream until it emits end event:
http.get(opts, function(res) {
var buffer = new Buffer(0);
res.on('readable', function() {
return buffer = Buffer.concat([buffer, this.read()]);
});
res.on('end', function() {
if (res.statusCode !== 200) return cb(new Error('Unable to fulfill request.'))
return cb(null, buffer.toString());
});
});
You never call res.end(). You should also do some res.write calls before res.end().