var request = require('request');
var async = require('async');
require('http').globalAgent.maxSockets = 30;
var count = 0;
var urlArr = ['url1','url2'.......] // ten thousand links to request
async.eachLimit(urlArr, 30, iterator, function(err) {
console.log('complete');
});
function iterator(url,callback) {
request(url, function(error, response, body) {
console.log(count++);
if (!error && response.statusCode === 200) {
return callback(null);
} else{
return callback(null);
}
});
}
The code above stop working after a few minutes, no error happen, the process is hoding and not exit, what's the problem with it?
Related
I have created one NODE JS URL request to get JSON and pass it to the new variable var1.
Var1 can get data within request function when call var1 outside of request it returns null value why? how to get data from URL request and pass it to a new variable outside the Request URL?
function datadata() {
var var1 = 0;
var request = require('request');
request('http://x.x.x.x/json', function (error, response, body) {
if (!error && response.statusCode == 200) {
var importedJSON = JSON.parse(body);
var1 = importedJSON.result.data;
// show value of var1
console.log(var1);
}
});
// cannot show value of var1
console.log(var1);
return var1;
}
function datadata(){
var var1 = 0;
var request = require('request');
return new Promise(function(resolve, reject){
request('http://x.x.x.x/json', function (error, response, body) {
if (err) return reject(err);
try {
if (!error && response.statusCode == 200) {
var importedJSON = JSON.parse(body);
var1 = importedJSON.result.data;
console.log(var1);
} catch(e) {
reject(e);
}
});
});
}
datadata().then(function(res) {
console.log(res);
}).catch(function(err) {
console.err(err);
});
you can't get something outside the request that easy anyway you can try to use Promise
Example
const request = require('request');
var p = new Promise((resolve, reject) => {
request('http://x.x.x.x/json', function (error, response, body) {
if (!error && response.statusCode == 200) {
var importedJSON = JSON.parse(body);
resolve(importedJSON.result.data)
}
});
p.then((data) => {
console.log(data);
})
You can use a callback to the datadata like so
Check whether the body is found on the response, in that case change your code to
request('http://x.x.x.x/json', function (error,response) {
and
var importedJSON = JSON.parse(response.body);
You should do this because your importedJSON may not be having data because according to your code it's data comes from (body)
Mostly likely the json data will be found on the response.body.
Then we callback the specific data you want, in your case it's result.data. Again make sure the result.data can be found on the raw json data that the request gives you otherwise it won't output anything because there's nothing to output, most likely you will get errors
function datadata() {
var var1 = 0;
var request = require('request');
request('http://x.x.x.x/json', function (error,body,response) {
if (error && response.statusCode == 200) {
callback("There was an error")
}else{
var importedJSON = JSON.parse(body);
callback(importedJSON.result.data);
// show value of var1
console.log(var1);
});
}
datadata((error, var1) => {
if(error){
return console.log(error)
}
console.log(var1
})
I'd like to read some information from an remote CSV file using a callback function. Not sure, how exactly to do this.
function:
function getRoomsFromCSV(allRoomsArray) {
var request = require('request');
request('http://localhost:3333/rooms.csv', function (error, response, body) {
if (!error && response.statusCode == 200) {
...
allRoomsText = allRoomsText.substr(0,allRoomsText.length-1) + ']}';
var allRoomsArray = JSON.parse(allRoomsText);
}
})
}
I'd like to call the function and loop through the result array.
var rooms = [];
getRoomsFromCSV( function (rooms) {
for(var i = 0; i < rooms.length; i++) {
console.log("i:",i);
}
However, the for loop is never called and the result (room) seems to be empty.
Try like this
function getRoomsFromCSV(allRoomsArray) {
var request = require('request');
request('http://localhost:3333/rooms.csv', function (error, response, body) {
if (!error && response.statusCode == 200) {
...
allRoomsText = allRoomsText.substr(0,allRoomsText.length-1) + ']}';
allRoomsArray(JSON.parse(allRoomsText)); //response params to callback
}
})
}
You sent callback to retrieve response. so call that callback inside the async function
Right now I have a piece of code which makes two http requests in parallel.
var request = require('request');
var async = require('async');
var fs = require('fs');
module.exports = {
req : function(options) {
//populate list with 2 lookups
var lookup_list = [];
for (var i = 0; i < 2; i++) {
if(i == 1) {
options.timeout = 200;
}
lookup_list.push(options);
}
//iterate through list with and do request calls
async.map(lookup_list, function(options, next) {
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
var body = JSON.parse(body);
next(null, body);
} else {
next(error || response.statusCode);
}
});
//after both requests are finished, deal with responses
}, function(err, results) {
if (!err) {
console.log(results);
/*
fs.appendFile('log.txt', JSON.stringify(results[1]) + "\n",function callback(error){
if (error) {
return console.error('write failed: ', error);
}
});
*/
} else {
console.error('request failed: ', err);
}
});
}
}
I want a to return the array results from the async.map callback when parallel.req() is called.
I've tried to provide a callback for req() and have it be called in the callback for async.map but no luck. I also tried other returns and callbacks elsewhere but have had no success. I would like some guidance to resolve this issue.
Can somebody see in my code why the variable oauthToken is defined on the server but not defined when returned to the client in the result of Meteor.call
I make a call to initiate a post request on the server
The body is parsed and I store a value into the variable oauthToken
This prints out on the server but does not print out on the client in my 'result'
Is this because the client is running a simulation? Can we do a 'return' in an asynchronous function?
Server.js
Meteor.methods({
getGoodreads: function () {
request.post('http://www.goodreads.com/oauth/request_token', {oauth:{
consumer_key: '89hdg8pEoMzRdg',
consumer_secret: 'dfgdfgHthtdtjtt' }}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var a = querystring.parse(body)
oauthToken = a.oauth_token
console.log(oauthToken); //prints value i need
return oauthToken
}else{
console.log('there is an error ' + error);
}
});
}
});
client.js
Template.profile.events({
'click #goodreads': function (event) {
event.preventDefault();
Meteor.call('getGoodreads', function(error, result) {
if (error) {
console.log('this is an error ');
} else {
console.log(result); //THIS IS UNDEFINED...possibilities?
}
});
}
});
Use futures to return values from async functions:
var Future = Npm.require("fibers/future")
Meteor.methods({
getGoodreads: function () {
var f = new Future();
request.post('http://www.goodreads.com/oauth/request_token', {oauth:{
consumer_key: '89hdg8pEoMzRdg',
consumer_secret: 'dfgdfgHthtdtjtt' }}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var a = querystring.parse(body)
oauthToken = a.oauth_token
console.log(oauthToken); //prints value i need
return f.return(oauthToken)
}else{
f.throw(error);
console.log('there is an error ' + error);
}
});
return f.wait()
}
});
It's because you are starting an asynchronous request in the method body. The method returns immediately, and the callback from the request doesn't really have anywhere to return the result.
I don't really know how to do this with methods, but you could do it with publish/subscribe. Something like this:
Server.js
var result = "";
Meteor.setInterval(function() {
request.post('http://www.goodreads.com/oauth/request_token', {oauth:{
consumer_key: '89hdg8pEoMzRdg',
consumer_secret: 'dfgdfgHthtdtjtt' }},
function(error, response, body) {
if (!error && response.statusCode == 200) {
var a = querystring.parse(body);
oauthToken = a.oauth_token;
result = oauthtoken;
}
});
});
}, 10*1000);
Meteor.publish("goodReads", function() {
this.added("goodReads", 1, {reads: result});
Meteor.setInterval(function() {
this.changed("goodReads", 1, {reads: result});
}, 10*1000);
this.ready();
});
Client.js
goodReads = new Meteor.Collection("goodReads");
Meteor.subscribe("goodReads"); // May be done in waitOn in the router
This was taken off the top of my head, and is not tested. It may or may not work, but is at least a pointer to how you could do it
var url = posts.paging.next;
for (var i = 0; i < 3; i++) {
(function(url,i) {
request(url,{json:true}, function (error, response, body) {
if (!error && response.statusCode == 200) {
url = body.paging.next;
console.log(i+"+++"+url)
}
});
})(url,i);
};//for
According to async request response (body.paging.next), İ want to change value of var url which is top of code .Please Help
NOTE: i'm trying to get all comments from facebook api , To do that , i need to get page links . Because of this , i wrote this codes , if you have any other alternative way please suggest them thanx
Assuming that you're trying to perform the requests in sequence (e.g. the next request relies on the url from the previous request), you could do something like (using the async module):
var async = require('async');
// ...
var url = posts.paging.next, i = 0;
async.whilst(
function() { return i < 3; },
function(callback) {
++i;
request(url, { json: true }, function(err, response, body) {
// TODO: handle `err`
//return callback(err);
if (!err && response.statusCode === 200) {
url = body.paging.next;
console.log(i + '+++' + url);
callback();
}
});
},
function(err) {
// done!
}
);