NodeJS using request inside a loop - node.js

I use NodeJS and request lib to make some request to an API.
I understand now that all requests are async and so it doesn't "wait" the result of the GET call, so the index of my Loop is always the same.
I was wondering if there was any simple way (without any lib) to wait for the response of the request call ?
For now, my code is this :
for (var i in entries) {
var entryId = entries[i]['id'];
var options = {
url: 'https://api.com/'+ entryId +'/get/status',
method: 'GET',
headers: {
'Authorization': auth
}
};
console.log(' ENTRY ID > '+ entryId);
request(options, function(error, response, body) {
var response = JSON.parse(body);
if (response.status.code == 200) {
var id = response.status.id;
var data = [];
data['id'] = id;
data = JSON.stringify(data);
// show first entryId of loop
console.log(' > MY ID : '+ id + ' - '+ entryId);
options = {
host: hostname,
port: 80,
path: '/path/function2',
method: 'PUT'
};
var post = http.request(options, function(json) {
var body = '';
json.on('data', function(d) {
body += d;
});
json.on('end', function() {
console.log('> DONE');
});
}).on('error', function(e) {
console.log(e);
});
post.write(data);
post.end();
}
});
}

You are looking for async/await.
Wrap your logic inside an async function, then you can await for the promise to resolve.
const request = require('request-promise')
async function foo (a) {
for (i in a)
try {
let a = await request('localhost:8080/')
// a contains your response data.
} catch (e) {
console.error(e)
}
}
foo([/*data*/])
Just use the promisified version of request module.

You also can use Promises to wait for your async code to finish.
function asyncCode(msg, cb){
setTimeout(function() {cb(msg);}, 1000);
}
var p1 = new Promises(function(resolve){
asyncCode("my asyncCode is running", resolve);
});
p1.then(function(msg) {
console.log(msg);
}).then(function() {
console.log("Hey I'm next");
});
console.log("SyncCode, Async code are waiting until I'm finished");

Related

Await function to finish calling another API

I am new to using async/await and having a couple issues.
I have the code below, which seems to not wait until the previous function is finished?
var url = require('url');
var path = require('path');
var https = require('https');
var request = require('request');
var url1 =
var url2 =
var url3 =
module.exports = async function (context, req) {
var call = await callUrl(context, url1);
context.log(call);
var call2 = await callUrl(context, url2);
context.log(call2);
var call3 = await callUrl(context, url3);
context.log(call3);
};
function callUrl (context, web) {
var requestUrl = url.parse(web);
const requestOptions = {
hostname: requestUrl.hostname,
path: requestUrl.path,
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
};
var request = https.request(requestOptions, function(res) {
var data = "";
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
var jsonData = JSON.parse(data);
return jsonData;
});
}).on('error', function(error) {
context.log("request error:", error);
return context.done();
});
request.end();
}
I am trying to get call to happen, then when it is finished call2, then when that is finished call3.
Can someone pinpoint why this does not occur? Currently, it hits all 3 pretty much asap, and each context.log is undefined presumably because the endpoints don't return anything. Each url is another azure function app API I have created.
There is nothing I am requiring to return from each call to use, I simply want them to finish before moving on the the next function.
Your callUrl method, which you call with await, needs to be either async itself or return a Promise. Why? because the work it does is itself asynchronous.
Here's your function adapted to use a Promise and return its actual value via the resolve() callback.
function callUrl (context, web) {
var requestUrl = url.parse(web);
const requestOptions = {
hostname: requestUrl.hostname,
path: requestUrl.path,
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
};
return new Promise(function (resolve,reject) {
var request = https.request( requestOptions, function( res ) {
var data = "";
res.on( 'data', function( chunk ) {
data += chunk;
} );
res.on( 'end', function() {
var jsonData = JSON.parse( data );
resolve( jsonData );
} );
} )
.on( 'error', function( error ) {
reject(error);
} );
request.end();
});
}
Notice that you use a POST operation with no body. That's a little unconventional.

nodejs - How to promisify http.request? reject got called two times

I'm trying to wrap http.request into Promise:
new Promise(function(resolve, reject) {
var req = http.request({
host: '127.0.0.1',
port: 4000,
method: 'GET',
path: '/api/v1/service'
}, function(res) {
if (res.statusCode < 200 || res.statusCode >= 300) {
// First reject
reject(new Error('statusCode=' + res.statusCode));
return;
}
var body = [];
res.on('data', function(chunk) {
body.push(chunk);
});
res.on('end', function() {
try {
body = JSON.parse(Buffer.concat(body).toString());
} catch(e) {
reject(e);
return;
}
resolve(body);
});
});
req.on('error', function(err) {
// Second reject
reject(err);
});
req.write('test');
}).then(function(data) {
console.log(data);
}).catch(function(err) {
console.log(err);
});
If I recieve errornous statusCode from remote server it will call First reject and after a bit of time Second reject. How to make properly so it calls only single reject (I think First reject is proper one in this case)? I think I need to close res myself, but there is no close() method on ClientResponse object.
UPD:
Second reject triggers very rarely - why?
Your code is almost fine. To restate a little, you want a function that wraps http.request with this form:
function httpRequest(params, postData) {
return new Promise(function(resolve, reject) {
var req = http.request(params, function(res) {
// on bad status, reject
// on response data, cumulate it
// on end, parse and resolve
});
// on request error, reject
// if there's post data, write it to the request
// important: end the request req.end()
});
}
Notice the addition of params and postData so this can be used as a general purpose request. And notice the last line req.end() -- which must always be called -- was missing from the OP code.
Applying those couple changes to the OP code...
function httpRequest(params, postData) {
return new Promise(function(resolve, reject) {
var req = http.request(params, function(res) {
// reject on bad status
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error('statusCode=' + res.statusCode));
}
// cumulate data
var body = [];
res.on('data', function(chunk) {
body.push(chunk);
});
// resolve on end
res.on('end', function() {
try {
body = JSON.parse(Buffer.concat(body).toString());
} catch(e) {
reject(e);
}
resolve(body);
});
});
// reject on request error
req.on('error', function(err) {
// This is not a "Second reject", just a different sort of failure
reject(err);
});
if (postData) {
req.write(postData);
}
// IMPORTANT
req.end();
});
}
This is untested, but it should work fine...
var params = {
host: '127.0.0.1',
port: 4000,
method: 'GET',
path: '/api/v1/service'
};
// this is a get, so there's no post data
httpRequest(params).then(function(body) {
console.log(body);
});
And these promises can be chained, too...
httpRequest(params).then(function(body) {
console.log(body);
return httpRequest(otherParams);
}).then(function(body) {
console.log(body);
// and so on
});
I know this question is old but the answer actually inspired me to write a modern version of a lightweight promisified HTTP client. Here is a new version that:
Use up to date JavaScript syntax
Validate input
Support multiple methods
Is easy to extend for HTTPS support
Will let the client decide on how to deal with response codes
Will also let the client decide on how to deal with non-JSON bodies
Code below:
function httpRequest(method, url, body = null) {
if (!['get', 'post', 'head'].includes(method)) {
throw new Error(`Invalid method: ${method}`);
}
let urlObject;
try {
urlObject = new URL(url);
} catch (error) {
throw new Error(`Invalid url ${url}`);
}
if (body && method !== 'post') {
throw new Error(`Invalid use of the body parameter while using the ${method.toUpperCase()} method.`);
}
let options = {
method: method.toUpperCase(),
hostname: urlObject.hostname,
port: urlObject.port,
path: urlObject.pathname
};
if (body) {
options.headers = {'Content-Length':Buffer.byteLength(body)};
}
return new Promise((resolve, reject) => {
const clientRequest = http.request(options, incomingMessage => {
// Response object.
let response = {
statusCode: incomingMessage.statusCode,
headers: incomingMessage.headers,
body: []
};
// Collect response body data.
incomingMessage.on('data', chunk => {
response.body.push(chunk);
});
// Resolve on end.
incomingMessage.on('end', () => {
if (response.body.length) {
response.body = response.body.join();
try {
response.body = JSON.parse(response.body);
} catch (error) {
// Silently fail if response is not JSON.
}
}
resolve(response);
});
});
// Reject on request error.
clientRequest.on('error', error => {
reject(error);
});
// Write request body if present.
if (body) {
clientRequest.write(body);
}
// Close HTTP connection.
clientRequest.end();
});
}
There are other ways as well but here you can find a simple way to make http.request as a promise or async/await type.
Here is a working sample code:
var http = require('http');
function requestAsync(name) {
return new Promise((resolve, reject) => {
var post_options = {
host: 'restcountries.eu',
port: '80',
path: `/rest/v2/name/${name}`,
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
};
let post_req = http.request(post_options, (res) => {
res.setEncoding('utf8');
res.on('data', (chunk) => {
resolve(chunk);
});
res.on("error", (err) => {
reject(err);
});
});
post_req.write('test');
post_req.end();
});
}
//Calling request function
//:1- as promise
requestAsync("india").then(countryDetails => {
console.log(countryDetails);
}).catch((err) => {
console.log(err);
});
//:2- as await
let countryDetails = await requestAsync("india");
After reading all of these and a few articles, I thought I'd post a sort of "general" solution that handles both http and https:
const http = require("http");
const https = require("https");
const url_obj = require("url");
const request = async (url_string, method = "GET", postData = null) => {
const url = url_obj.parse(url_string);
const lib = url.protocol=="https:" ? https : http;
const params = {
method:method,
host:url.host,
port: url.port || url.protocol=="https:" ? 443 : 80,
path: url.path || "/"
};
return new Promise((resolve, reject) => {
const req = lib.request(params, res => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error(`Status Code: ${res.statusCode}`));
}
const data = [];
res.on("data", chunk => {
data.push(chunk);
});
res.on("end", () => resolve(Buffer.concat(data).toString()));
});
req.on("error", reject);
if (postData) {
req.write(postData);
}
req.end();
});
}
You could use like this:
request("google.com").then(res => console.log(res)).catch(err => console.log(err))
This is heavily inspired by this article, but replaces the hacky url parsing with the built in api.
Hope this help.
const request = require('request');
async function getRequest() {
const options = {
url: 'http://example.com',
headers: {
'Authorization': 'Bearer xxx'
}
};
return new Promise((resolve, reject) => {
return request(options, (error, response, body) => {
if (!error && response.statusCode == 200) {
const json = JSON.parse(body);
return resolve(json);
} else {
return reject(error);
}
});
})
}
It's easier for you to use bluebird api, you can promisify request module and use the request function async as a promise itself, or you have the option of using the module request-promise, that makes you to not working to creating a promise but using and object that already encapsulates the module using promise, here's an example:
var rp = require('request-promise');
rp({host: '127.0.0.1',
port: 4000,
method: 'GET',
path: '/api/v1/service'})
.then(function (parsedBody) {
// GET succeeded...
})
.catch(function (err) {
// GET failed...
});

Node.js http.request in loop

Am using a function to save data. Using the http.request option. It is working fine. If I call the same function in a loop, some of the data not saving in database. Also getting the parse error message for some response.
How Can I call the function of http.request in a loop?
for (var i = 1; i <= 23; i++) {
turn_timer(i);
}
function turn_timer(nos) {
try {
var str = "num=" + nos;
var len = str.length;
win_settings.headers = {
'Content-length': len,
'Content-Type': 'application/x-www-form-urlencoded'
}
var request = http.request(win_settings, function(response) {
response.on('data', function(data) {
});
response.on('error', function(err) {
});
response.on('end', function() {
});
});
request.on('error', function(err) {
});
request.write(str + "\0");
request.end();
} catch (err) {
console.log(err);
}
}
Check the scope of your variable that stores message because async function call may overwrite it.
I believe your problem is because you are using a for loop, instead of going with a more asynchronous approach. Here is a quick attempt to solve your problem. I have omitted some of your code, as it seemed to be incomplete. I have left the important parts and added a few things based on an answer to a similar question.
var http = require('http'),
async = require('async');
function turn_timer(n, callback) {
var var str = "num=" + n,
len = str.length,
req;
win_settings.headers = {
'Content-length': len,
'Content-Type': 'application/x-www-form-urlencoded'
};
req = http.request(options, function(response) {
...
callback(null);
});
req.on('error', function(err) {
...
callback(err);
});
req.end();
}
async.timesSeries(23, function (n, next) {
turn_timer(options, function(err) {
next(err);
});
});
For more information, you can read more about async.timesSeries here: https://github.com/caolan/async#timesseriesn-callback

How can you synchronize this process using nodejs?

I need to iterate on an array, for each item I apply an operation by calling an HTTP call.
The difficulty is that i need to syncronize this process in order to call a callback after the loop (containing the array after all the operation executed by the HTTP call).
Let's consider this short example:
function customName(name, callback) {
var option = {
host:'changename.com',
path: '/'+name,
headers: { 'Content-Type': 'application/json' },
port: 80,
method:'POST'
};
var req = http.request(option, function(res) {
var output = "";
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', function() {
var obj = JSON.parse(output);
callback(obj.res);
});
});
req.on('error', function(e) {
console.error(e.message);
});
req.end();
}
function changeNames(OldNames, callback) {
var Res = [];
for (name in OldNames) {
customName(OldNames[name], function(new_name) { Res.push(new_name); });
});
callback(Res);
}
var Names = ['toto', 'tata', 'titi'];
changeNames(Names, function(Names) {
//...
});
Here the loop is over before the first HTTP call, so the Res array is empty.
How can we synchronize this execution?
I know it's not very good to synchronize treatments in nodejs. Do you think it would be better to communicate the names one by one with the client and not building an array?
You can use async.map for that. You pass it your list of names, it will run the getOriginalName function (which you mistakenly called customName, I think) for each name and gather the result, and in the end it will call a function with an array of results:
var http = require('http');
var async = require('async');
function getOriginalName(name, callback) {
var option = {
host:'changename.com',
path: '/'+name,
headers: { 'Content-Type': 'application/json' },
port: 80,
method:'POST'
};
var req = http.request(option, function(res) {
var output = "";
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', function() {
var obj = JSON.parse(output);
callback(null, obj.res);
});
});
req.on('error', function(e) {
callback(e);
});
req.end();
}
function changeNames(OldNames, callback) {
async.map(OldNames, getOriginalName, callback);
}
var Names = ['toto', 'tata', 'titi'];
changeNames(Names, function(err, NewNames) {
console.log('N', NewNames);
});

Node.js http request inside loop

I am trying to periodically load some data from external API like this:
setInterval(function() {
getData();
}, 60000);
function getData() {
if (typeof someObject.data === 'object') {
for (var prop in someObject.data) {
if (prop === 1 || prop === 2) {
var options = {
host: 'somehost.com',
path: '/somepath?param=' + prop
};
var req = http.request(options, function(res) {
// EXECUTION NEVER REACHES THIS POINT ?!?!
req.on('end', function() { alert('ended'); });
});
req.end();
}
}
}
}
If I do not do any intervals and loops, such request to the same host works perfectly. However, if I try to do something like shown above, then the request never ever calls its callback function.
What I am doing wrong here?
I think one of your conditions are bad, the following works fine for me.
var http = require('http');
setInterval(function() {
getData();
}, 1000);
function getData() {
console.log('get');
//if (typeof someObject.data === 'object') {
console.log('get 1');
//for (var prop in someObject.data) {
console.log('get 2');
//if (prop === 1 || prop === 2) {
console.log('get 3');
var options = {
host: 'google.com',
path: '/'
};
var req = http.request(options, function(res) {
console.log('http request', res.statusCode);
//req.on('end', function() {
// console.log('ended', req);
//});
});
req.end();
//}
//}
//}
}
also If I'm right, you don't need req.on('end'), the callback of the request is called when it's completed. You can also use http.get, so you don't need to call req.end
var req = http.get( options.host, function(res) {
console.log('http request', res.statusCode);
//req.on('end', function() {
// console.log('ended', req);
//});
}).on('error', function( e ) {
console.error( 'error', e );
})
see more info in the docs
hope I could help.

Resources