Right now I have a piece of code which makes two http requests in parallel.
var request = require('request');
var async = require('async');
var fs = require('fs');
module.exports = {
req : function(options) {
//populate list with 2 lookups
var lookup_list = [];
for (var i = 0; i < 2; i++) {
if(i == 1) {
options.timeout = 200;
}
lookup_list.push(options);
}
//iterate through list with and do request calls
async.map(lookup_list, function(options, next) {
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
var body = JSON.parse(body);
next(null, body);
} else {
next(error || response.statusCode);
}
});
//after both requests are finished, deal with responses
}, function(err, results) {
if (!err) {
console.log(results);
/*
fs.appendFile('log.txt', JSON.stringify(results[1]) + "\n",function callback(error){
if (error) {
return console.error('write failed: ', error);
}
});
*/
} else {
console.error('request failed: ', err);
}
});
}
}
I want a to return the array results from the async.map callback when parallel.req() is called.
I've tried to provide a callback for req() and have it be called in the callback for async.map but no luck. I also tried other returns and callbacks elsewhere but have had no success. I would like some guidance to resolve this issue.
Related
We are using method called flask_api which has the parameter.
Our need is, we need to pass that parameter inside request.post
Can anyone please help as we are beginners in node.js
async flask_api(turnContext){
var res;
var fs = require('fs');
var request = require('request');
var formData = {image1: fs.createReadStream('D:/my_app/demo.png')}
var json_body = "" ;
var obj_value= ""
function initPromise(turnContext)
{return new Promise(function(resolve,reject)
{
console.log("print turn context",turnContext); //we are getting the value in turncontext
console.error('before post call...:1');
request.post({url:'http://127.0.0.1:5002/',formData: formData}, function(err, httpResponse, body)
{
if (err) {
console.log("response", reject(err));
return JSON.stringify(err);
}
else{
while(1==1){
if(body != undefined){
break;
}
}
json_body = JSON.parse(body);
resolve([json_body,turnContext]);//here we are not able to get the turncontext
}
});
})
}
Try using this:
var req = request.post({ url: 'http://127.0.0.1:5002/' }, (err, resp, body) => {
if (err) {
res.status(500).send(err);
} else {
res.send(resp);
}
});
var form = req.form();
form.append('file', fs.createReadStream('D:/my_app/demo.png'));
In your case:
async flask_api(turnContext){
var res;
var fs = require('fs');
var request = require('request');
var formData = {image1: fs.createReadStream('D:/my_app/demo.png')}
var json_body = "" ;
var obj_value= ""
var tempContext;
function initPromise(turnContext)
{return new Promise(function(resolve,reject)
{
tempContext = turnContext;
console.log("print turn context",tempContext); //we are getting the value in turncontext
console.error('before post call...:1');
request.post({url:'http://127.0.0.1:5002/',formData: formData}, (err, httpResponse, body) =>
{
if (err) {
console.log("response", reject(err));
return JSON.stringify(err);
}
else{
while(1==1){
if(body != undefined){
break;
}
}
json_body = JSON.parse(body);
resolve([json_body,tempContext]);//here we are not able to get the turncontext
}
});
})
}
Try this. Note that I refactored your code (how I would have done it).
Server (for test purposes)
var http = require('http');
var querystring = require('querystring');
var server = http.createServer().listen(8124);
server.on('request', function(request,response) {
if (request.method == 'POST') {
var body = '';
// append data chunk to body
request.on('data', function (data) {
body += data;
});
// data transmitted
request.on('end', function () {
var post = querystring.parse(body);
console.log(post);
response.writeHead(200, {'Content-Type': 'text/plain'});
response.end(JSON.stringify('body\n'));
});
}
});
console.log('server listening on 8124');
Client
function initPromise(turnContext) {
var res;
var fs = require('fs');
var request = require('request');
var formData = {image1: "hello"}
var json_body = "" ;
var obj_value= "";
return new Promise(function(resolve,reject) {
console.log("print turn context",turnContext); //we are getting the value in turncontext
console.error('before post call...:1');
request.post({url:'http://localhost:8124/',formData: formData}, function(err, httpResponse, body) {
if (err) {
reject(err);
}
else{
if (body != undefined) {
json_body = JSON.parse(body);
resolve([json_body,turnContext]); //here we are not able to get the turncontext
} else {
reject('body undefined');
}
}
});
});
}
async function flask_api(turnContext) {
var result = initPromise(turnContext);
console.log("result", result);
result.then(
// fulfillment
function (val) { //this is where you get the value passed to resolve()
console.log(val);
}).catch(
//log the rejection reason
(reason) => {
console.error(reason);
}
);
}
flask_api({test: 'hello'});
The result on the client side is:
print turn context { test: 'hello' }
before post call...:1
result Promise { <pending> }
[ 'body\n', { test: 'hello' } ]
Note:
That line of code return JSON.stringify(err); isn't necessary. The callback in a promise is not expecting any return unless you want to use it for flow control purposes.
if body is indeed undefined, you will have an infinite loop.
Try using this:
This will work for the latest node.js versions.
async initPromise(turnContext)
{
var res;
var fs = require('fs');
var request = require('request');
var formData1 = {image1: "Hello"};
var path = require('path')
var base_dir=path.dirname(path.dirname(__filename))
console.log(path.join( base_dir, "resource", "output.png"));
var image_name = path.join( base_dir, "resource", "output.png")
var formData = { image1: fs.createReadStream(image_name) };
var json_body = "" ;
var obj_value= "";
return await new Promise(function(resolve,reject)
{
console.log("pls work",turnContext);
//\var turnContext1 = turnContext;
//\console.log(typeof(turnContext));
console.error('before post call...:1');
request.post({url:'http://127.0.0.1:5002/',formData: formData}, (err, httpResponse, body) =>
{
console.log("formdata....body....",formData);
console.log("inside request post");
if (err) {
reject(err);
}
else{
if (body != undefined) {
json_body = JSON.parse(body);
console.log("response inside", json_body);
resolve([json_body,turnContext]); //here we are not able to get the turncontext
} else {
reject('body undefined');
}
}
});
});
}
async flask_api(turnContext) {
var result = await this.initPromise(turnContext);
console.log("result", result);
console.log("result1", result[0]);
console.log("result2", result[1]);
}
I'm having trouble with setting a variable or at least returning it in async waterfall. I know that you can't return in async but I did a callback on my variable, jsonFinal and it goes into the function below under data.
function getIndividualMatchJSONObjHelper(matchData, matchParticipantData, indexIter) {
var individualMatchURL = 'https://na1.api.riotgames.com/lol/match/v3/matches/' + matchData.matchID[indexIter] + '?api_key=' + API_KEY;
var jsonFinal;
async.waterfall([
function(callback) {
request(individualMatchURL, function(err, response, body) {
if(!err && response.statusCode == 200) {
var json = JSON.parse(body);
for (var j = 0; j < 10; j++) {
if (matchData.championID[indexIter] == json['participants'][j].championId) {
jsonFinal = json['participants'][j];
callback(null, jsonFinal);
}
}
}
else {
console.log(err);
}
});
}
],
function(err, data) {
if(err) {
console.log(err);
}
else {
jsonFinal = data;
}
});
console.log(jsonFinal);
return jsonFinal;
}
How can I get the function to return jsonFinal properly?
You can only get the variable within the callback like with any asynchronous operation. Therefore, modify your function to also accept a callback.
function getIndividualMatchJSONObjHelper(matchData, matchParticipantData, indexIter, callback) {
var individualMatchURL = 'https://na1.api.riotgames.com/lol/match/v3/matches/' + matchData.matchID[indexIter] + '?api_key=' + API_KEY;
async.waterfall([
function (callback) {
request(individualMatchURL, function (err, response, body) {
// always trigger the callback even when you have errors or else your program will hang
if (err)
return callback(err);
if (response.statusCode != 200)
return callback(new Error('Status code was ' + response.statusCode));
var json = JSON.parse(body);
for (var j = 0; j < 10; j++) {
if (matchData.championID[indexIter] == json['participants'][j].championId) {
// match found: send back data
console.log('inside', json['participants'][j]);
return callback(null, json['participants'][j]);
}
}
// if it reaches this point, no match was found
callback();
});
}
], callback); // note: this outer 'callback' is NOT the same as the inner 'callback'
}
Then when you call your function e.g.
getIndividualMatchJSONObjHelper(data, participants, indexIter, function (err, json) {
// you can only get the JSON at this point
console.log('outside', json);
matchParticipantData.specificParticipantData[i] = json;
});
The code I wrote so far is as below.
I don't need the whole response but just part of it.
var request = require('request');
var async = require('async');
var asyncTasks = [];
var install;
async.series([
function (callback) {
setTimeout(function () {
request('URL', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body); // Show the HTML for the Google homepage.
}
});
}, 5000);
},
function (callback) {
setTimeout(function () {
request('URL', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body); // Show the HTML for the Google homepage.
}
});
}, 5000);
}
],
function (error, results) {
console.log(results);
});
One approach to do the above concurrently would be to use async.parallel - of the form of:
async.parallel([
function(){ ... },
function(){ ... }
], callback);
Another approach is to use a Promises library - BlueBird or Q are good choices.
Q.All is of the form of:
return Q.all([
promise1,
promise2
]).spread(function (resultFromPromise1, resultFromPromise2) {
// do something with the results...
});
You could use one of these approaches to parallelise the two calls. The outputs of each will give you an array containing the results of each call respectively.
Here is a simple illustration of each approach:
Using Async.js
var async = require('async');
var task = function (cb, count) {
setTimeout(function () {
cb(null, "complete: " + count);
}, 1000);
};
async.parallel([
function (cb) {
task(cb, 'one');
},
function (cb) {
task(cb, 'two');
}
], function (err, results) {
console.log(results);
//[ 'complete: one', 'complete: two' ]
});
Using Q:
var Q = require('q');
function task1(cb, count) {
var deferred = Q.defer();
setTimeout(function () {
return deferred.resolve(cb(null, count));
}, 1000);
return deferred.promise;
}
var myCb = function (err, count) {
return "complete: " + count;
};
Q.all([task1(myCb, 'one'), task1(myCb, 'two')])
.then(function (results) {
console.log(results);
//[ 'complete: one', 'complete: two' ]
});
Let me know if you are unclear.
Promises are there to help you out in such a case.
I would prefer to use 'Q' library.
I have modified your code to use Q library
var Q = require('q');
var request = require('request');
function makeCall() {
Q.all([req1(), req2()])
.spread(function (res1, res2) {
// This block is executed once all the functions( Provided in Q.all() ) are finished its execution.
// Use responses from called functions
}, function (err) {
// Error, If any
});
}
function req1() {
var defer = Q.defer();
var url = ''; // Specify URL
var options = {
method: 'get', // Method to use
url: url
}
request(options, function (err, res, body) {
if (err) {
return defer.reject(err);
}
return defer.resolve(body);
});
return defer.promise;
}
function req2() {
var defer = Q.defer();
var url = ''; // Specify URL
var options = {
method: 'get', // Method to use
url: url
}
request(options, function (err, res, body) {
if (err) {
return defer.reject(err);
}
return defer.resolve(body);
});
return defer.promise;
}
You can find docs for Q library here : Q docs
var request = require('request');
var async = require('async');
require('http').globalAgent.maxSockets = 30;
var count = 0;
var urlArr = ['url1','url2'.......] // ten thousand links to request
async.eachLimit(urlArr, 30, iterator, function(err) {
console.log('complete');
});
function iterator(url,callback) {
request(url, function(error, response, body) {
console.log(count++);
if (!error && response.statusCode === 200) {
return callback(null);
} else{
return callback(null);
}
});
}
The code above stop working after a few minutes, no error happen, the process is hoding and not exit, what's the problem with it?
var url = posts.paging.next;
for (var i = 0; i < 3; i++) {
(function(url,i) {
request(url,{json:true}, function (error, response, body) {
if (!error && response.statusCode == 200) {
url = body.paging.next;
console.log(i+"+++"+url)
}
});
})(url,i);
};//for
According to async request response (body.paging.next), İ want to change value of var url which is top of code .Please Help
NOTE: i'm trying to get all comments from facebook api , To do that , i need to get page links . Because of this , i wrote this codes , if you have any other alternative way please suggest them thanx
Assuming that you're trying to perform the requests in sequence (e.g. the next request relies on the url from the previous request), you could do something like (using the async module):
var async = require('async');
// ...
var url = posts.paging.next, i = 0;
async.whilst(
function() { return i < 3; },
function(callback) {
++i;
request(url, { json: true }, function(err, response, body) {
// TODO: handle `err`
//return callback(err);
if (!err && response.statusCode === 200) {
url = body.paging.next;
console.log(i + '+++' + url);
callback();
}
});
},
function(err) {
// done!
}
);