limit maxsockets request package - node.js

I try to limit request package maxsockets ,
I use fiddler for testing how much it uses concurrent.
As I see ,it doesnt apply the 10 limit that i try to set.
i dont want request module use more than 10 concurrent
what may i be doing wrong?
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var loglar='idler2.txt';
var url = require('url');
var util = require('util');
var http = require('http');
var https = require('https');
var request = require('request');
var fs = require('fs');
var linkler = [];
var starttime = Math.round(new Date().getTime() / 1000);
http.globalAgent.maxSockets = 10;
https.globalAgent.maxSockets = 10;
var timeoutsure = 30 * 1000;
var success=0,fail=0;
process.on('exit', onExit);
function onExit() {
console.log('\n%d secs,\n%d suc ,\n%d fail\n---------------------\n',
(Math.round(new Date().getTime() / 1000)) - starttime,success,fail);
}
function logla(data)
{
var fd = fs.openSync(loglar, 'a+');
fs.writeSync(fd, data);
fs.closeSync(fd);
}
for(i=0;i<1000;i++)
{
sorgutest();
}
var r = request.defaults({'proxy':'http://127.0.0.1:8888',pool: {maxSockets: 10}});
function sorgutest()
{
r.get({url:'https://freegeoip.net/json/',pool: {maxSockets: 10}}, function optionalCallback(err, httpResponse, body) {
if (err) {
fail++;
return console.error('failed: 49', err);
}
else {
try {bodyjson=JSON.parse(body);
logla(body);
success++;
}
catch(e){console.log("hamina 54");}
}
});
}

As stated in the documentation:
Note that if you are sending multiple requests in a loop and creating multiple new pool objects, maxSockets will not work as intended. To work around this, either use request.defaults with your pool options or create the pool object with the maxSockets property outside of the loop.
try to change the pool option of request.defaults or create a pool object and use it on all request calls.
Edit:
I've noticed that you already used request.default, the fix should be to simply remove the pool option in the r.get call.
from:
r.get({url:'https://freegeoip.net/json/',pool: {maxSockets: 10}}, function optionalCallback(err, httpResponse, body) {
to: r.get({url:'https://freegeoip.net/json/'}, function optionalCallback(err, httpResponse, body) {

Related

Strange 401 error in nodejs. when calling 2 API it gives auth error

When I am calling one API from the nodejs it is giving proper reply. But when I am adding one more call it is giving 401 error. I dont know if I have to close some parameter before calling another request.
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var request = require('request')
var username = "shruti111";
var password = 'Welcome1';
var planId;
baseUrl = 'https://50d5a18993c046e585b90bc8cc5e1f80-jcs.oci.cloudonline.ml:443';
var baseUrlwoHttps = baseUrl.substring(8);
process.env["NO_PROXY"] = baseUrlwoHttps;
var getUrl = baseUrl + '/IMCMCSREST/rest/v1/PlannedCosts';
var options = {
url: getUrl,
auth: {
user: username,
password: password
}
}
request(options, function (err, res, body) {
if (err) {
console.dir(err)
return
}
var json = JSON.parse(body);
var arr = [];
for (i = 0; i < json.items.length; i++) {
if (json.items[i].PlanCode == 'Material Cost Planning - PO')
planId = json.items[i].PlanId;
//arr.push(json.items[i].PlanId, json.items[i].PlanCode);
}
console.log(planId);
})
Upto this point it is working properly. If I add below code in the same file it gives 401 error for both call. Otherwise it runs properly.
var getUrl = baseUrl + 'IMCMCSREST/rest/v1/PlannedCosts/' + planId + '/ child / CmiCpPlanCostTypesView';
var options = {
url: getUrl,
auth: {
user: username,
password: password
}
}
request(options, function (err, res, body) {
if (err) {
console.dir(err)
return
}
console.log(body);
var json = JSON.parse(body);
console.log(json);
var arr = [];
var x;
for (i = 0; i < json.items.length; i++) {
arr[i] = json.items[i].CostTypeId;
//arr.push(json.items[i].PlanId, json.items[i].PlanCode);
}
console.log(arr[i]);
})
I think the first problem here is the plandId variable you're using on second request does not have a value. What you can try is calling the second request on the callback of first request.
Another problem seems to be you are redefining existing variables, though its not fully clear as you didn't show the file as a whole.

Rate limit API requests in node.js

First off, I'm new here, so please...be gentle...I've been teaching myself node.js over the last few months, mostly with the desire to scrape a bunch of data from the FlightAware website API.
I am trying to request from their site a list of flights for aircraft, here is what I have so far.
var aircraft = [array,of,aircraft,tail,numbers]
for (i=0; i < aircraft.length; i++) {
faGetFlight(aircraft[i],function doneLookup(data) {
dbUpdateFlight(collectionName, data)
})
}
This code works, but as soon as there is more than 10 aircraft in the list, it fails, because is sending more than 10 API requests in a minute. What are some easy/straightforward ways to slow this down a little. I would like to send about 50-60 API requests total each time this runs, so I need it spaced over 5-6 minutes. The faGetFlight() function uses the 'request' module. I've tried the request-rate-limit module instead of the request module, but I can't seem to make it work. I don't think the authorization works properly with the request-rate-limiter module. Getting an error about anonymous user. For what it's work, it works with just the request module instead, but I run into the rate limit issues.
Here is my faGetFlight() code.
var RateLimiter = require('request-rate-limiter');
const REQS_PER_MIN = 10; // that's 25 per second
var limiter = new RateLimiter(REQS_PER_MIN);
//API Variables //
var apiUrl = 'url'
var apiEndpoint = 'endpoint'
var apiAuth = 'apikey'
var apiExtData = 0
var apiHowMany = 15 //Number of results to request.
var options = { method: 'GET',
url: apiUrl + apiEndpoint,
qs: { ident: acIdent
},
headers:
{
Authorization: apiAuth }
};
limiter.request(options, function doneDownload(error, response, body) {
if (error) throw new error(error);
callback(body)
});
}
Sorry if this isn't clear...it's my first post!
You can do a naive implementation using functions and a simple setTimeout.
See:
var aircrafts = [array,of,aircraft,tail,numbers];
var REQS_PER_MIN = 10;
var MAX_AMOUNT_REQUESTS = 100;
var timeout = (1 / REQS_PER_MIN) * 60 * 1000;
processAircraft(0);
function processAircraft(index){
if(index >= MAX_AMOUNT_REQUESTS)
return console.log("All done!");
//On start of function, schedule next processing in "timeout" ms
setTimeout(function(){
processAircraft(index+1);
}, timeout);
faGetFlight(aircrafts[index], function doneLookup(data) {
dbUpdateFlight(collectionName, data)
})
}

Encapsulate node.js code in a function

So far I have the below code which I need to get the id from a JSON object:
var http = require("http");
var fs = require('fs');
//var bodyParser = require("body-parser");
//var moment = require("moment");
var options = {
"method" : "GET",
"hostname" : "xxx.xxx.xxx.xxx",
"port" : "18080",
"path" : "/api/v1/applications/"
};
var req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = JSON.parse(Buffer.concat(chunks));
var arrFound = Object.keys(body).filter(function(key) {
if (body[key].name.indexOf("TestName") > -1) {
return body[key].name;
}
}).reduce(function(obj, key){
obj = body[key].id;
return obj;
}, {});;
//console.log("Name: ", arrFound);
console.log("ID: ", arrFound);
});
});
req.end();
I know it's reading the id as I currently write it out to the console(console.log("ID: ", arrFound);).
What I would like to do is have this available for use in other parts of my program. I assume I need to do something like encapsulate it in a function that I can call but node.js is new to me and I'm not entirely sure how/what to do.
Can anybody advise how to do this or point me to some resource that might help?
You should research the module pattern and exports. You are right, the basic idea is to export your code as a function. Based on how you've approached this, it should take a callback function. I've made a naive attempt with what you have.
Notice I pass a null as the first argument to the callback function. It is conventional when using callbacks in node to use the first argument for errors. You probably need some error handling.
Having said that, you night look into some libraries like requests, or the ES6 fetch function. These will allow you to organize your code more neatly. You'll probably wind up using promises instead of callbacks.
var http = require("http");
var fs = require('fs');
//var bodyParser = require("body-parser");
//var moment = require("moment");
var options = {
"method" : "GET",
"hostname" : "xxx.xxx.xxx.xxx",
"port" : "18080",
"path" : "/api/v1/applications/"
};
exports.getId = function(callback) {
var req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = JSON.parse(Buffer.concat(chunks));
var arrFound = Object.keys(body).filter(function(key) {
if (body[key].name.indexOf("TestName") > -1) {
return body[key].name;
}
}).reduce(function(obj, key){
obj = body[key].id;
return obj;
}, {});;
//console.log("Name: ", arrFound);
callback(null, arrFound));
});
});
req.end();
}
You call it like this:
exports.getId(function(err, id){
console.log(id)
})

How to concurrent download files using cheerio and nodejs?

I have a website with multiple pages, each page lists download links which I want to scrap and download.
I have few issues with it:
My script only downloads about 4-5 files and getting stuck.
I would like to concurrently download as much files as my CPU can.
I got stuck with maximum event emitters, I don't understand why is that so I just go
How to follow redirects purely using request module (without follow-redirects)?
How to download the file like the browser does without mentioning it's name? there is no content-disposition but I think the browser follow redirects and the redirected URL has the filename in it's path.
My current code looks like so:
var request = require('request');
var cheerio = require('cheerio');
var https = require('follow-redirects').https;
require('events').EventEmitter.prototype._maxListeners = 1000;
for(var i = 1; i <= 10000; i++) {
(function(i){
url = 'http://mywebsite.com/files?page=' + i;
request(url, gotHTML)
})(i);
}
function gotHTML(err, resp, html) {
var $ = cheerio.load(html);
$('.file-header').each(function() {
var data = $(this);
var fileLink = data.children().first().children().first().attr('href');
var fileName = fileLink.substring(10);
var downloadLink = 'https://mywebsite.com/api/download/' + fileName;
download(downloadLink, function() {
console.log('downloaded');
})
})
}
function download(url, cb) {
var request = https.get(url, function(response) {
var location = request.res.headers.location;
console.log(location);
location = location.split('/').pop();
console.log(location);
var file = fs.createWriteStream(location);
response.pipe(file);
file.on('finish', function() {
file.close(cb);
});
});
}
The default HTTP/HTTPS Agent only uses a maximum of 5 sockets (maxSockets) for requests to the same origin. So this could be causing some issues for you.
Try changing this:
var request = https.get(url, function(response) {
to this:
var options = require('url').parse(url);
options.agent = false; // or use a custom https.Agent with a higher `maxSockets`
var request = https.get(options, function(response) {

Expressjs/connect - Simulate slow file upload

Is there a way of setting a mode or value in connect or express
to simulate slow file uploads??
First, install node-limiter then create a Transform stream that will throttle any Streams:
var util = require('util');
var Transform = require('stream').Transform;
var TokenBucket = require('limiter').TokenBucket;
function BucketStream(rate, interval, parentBucket, options) {
Transform.call(this, options);
this.bucket = new TokenBucket(Number.POSITIVE_INFINITY, rate, interval, parentBucket);
}
util.inherits(BucketStream, Transform);
BucketStream.prototype._transform = function(chunk, encoding, callback) {
this.bucket.removeTokens(chunk.length, function(err) {
callback(err, chunk);
});
};
Then the request is a ReadableStream:
var bucketStream = new BucketStream(1024 * 500, 'second'); // 500KB/sec
req.pipe(bucketStream);
Now read from bucketStream (instead from req) as fast as you want, you'll only get 500KB/sec
I did this on top of my head so beware :)

Resources