Nodejs loop http requests in a function - node.js

I have been able to use bluebird to run an array of http get requests in parallel within a function that returns the responses of each requests after they have all finished. However, I would like to chain the requests in sequence to run one after another without effecting the asynchronous behavior of the promises in my current function shown below.
var Promise = require('bluebird');
function buildCartWithItems(cartID,requests, cookie) {
var promises = [];
for (idx in requests) {
var request = requests[idx]; // requests is an array containing the json records of each request parameter
request["id"] = cartID;
promises.push(new Promise(function(resolve, reject) {
var options = {
host: 'example.com',
path: "/example/addToCart?"+param(request),
headers: {'Cookie': cookie, 'Connection': 'keep-alive'}
};
https.get(options, function (response) {
var body = '';
response.on('data', function(d) {
body += d;
});
response.on('end', function() {
console.log("jsonservicesPostRequest response: " + body);
var parsed = JSON.parse(body);
if (parsed["STATUS"] == "success") {
resolve(parsed.RESULT.itemGroup.id);
} else
resolve("error");
});
response.on('error', function(exception) {
console.log("auth error: " + exception);
resolve(exception);
});
});
}));
}
return Promise.all(promises);
}
Use:
var cart = 12345; // cart ID
var itemsParams = [];
for (idx in products) {
var parms = {
'guid' : prod["_id"]["GUID"],
'count': prod["deficit"],
};
itemsParams.push(parms);
}
buildCartWithItems(cart,itemsParams,newcookie).then(function(results) {
// results -> array of all of the cart.id's
console.log("Build Cart Results: " + JSON.stringify(results));
});

Related

Http requests inside a for loop from angular to node.js API returns 404

I try to send several HTTP requests continuously inside a for loop to node.js rest API.
Below is how I try to send HTTP requests from my Angular Front-end.
getOthersCompletionsByTaskID is the function that send a HTTP request.
this.tasksService.getTasks().then(res=>{
let i=0;
for(i;i< this.tasksService.tasks.length; i++){
this.completionService.getOthersCompletionsByTaskID(this.tasksService.tasks[i].TaskId).then(res =>{
let cArray: any = res;
console.log("cArray ", cArray)
})
}
this.dtTrigger.next();
this.rendMyTasksTable();
})
Below is my query execution function in node API.
function queryGetExecute(qry, params, isMultiSet, callback) {
var data = [];
var dataset = [];
var resultset = 0;
request = new Request(qry, async function (err, rowCount) {
utility.sendDbResponse(err, rowCount, dataset, callback);
});
params.forEach( param => {
request.addParameter(param.name, param.type, param.val);
});
request.on('row', async function (columns) {
utility.buildRow(columns, data);
});
request.on('doneInProc', async function (rowCount, more, rows) {
if (isMultiSet == false) {
dataset = data;
} else {
dataset.push(data);
data = [];
}
});connection.execSql(request)}
Below is the function in the repository that calls the above function.
function getCompletionsByFirebaseId(req, res){
if (req.params.firebaseId && req.params.taskId) {
var parameters = [];
parameters.push({ name: 'firebaseId', type: TYPES.NVarChar, val: req.params.firebaseId});
parameters.push({ name: 'taskId', type: TYPES.NVarChar, val: req.params.taskId});
var query = "select users.[name], completedById, [status], completionImage,completionId,exchangeTaskId from Tasks join completions on Tasks.TaskId=completions.taskId join users on users.firebaseId = completions.completedById where taskownerId=#firebaseId and Tasks.TaskId = #taskId"
dbContext.getQuery(query, parameters, false, function (error, data) {
if (data) {
req.data = data[0];
let finaldata = res.json(response(data, error));
return finaldata
}else{
return res.sendStatus(404);
}
});
}else
{
return res.sendStatus(404)
}
}
When I do requests like this, API returns 404. but the first request is becoming successful. rest requests returning 404. What am I doing wrong here?

429 - {"status":"error","message":"Api max rps reached"} while sending request to manychat sendFlow API

I'm trying to setCustomField to Manychat subscribers using its API. The link to their API is:
https://api.manychat.com/swagger#/Subscriber/post_fb_subscriber_setCustomFieldByName
Below is the code for request :
var rp = require("request-promise");
var config = require("./../campaign_config.json");
module.exports = { setWaybillForUser : async function setWaybillForUser(userid,waybill,dbRef){
var options = { method: 'POST',
uri: 'https://api.manychat.com/fb/subscriber/setCustomFieldByName',
headers:
{'Cache-Control': 'no-cache',
Accept: '*/*',
Authorization: config.Manychat_token,
'Content-Type': 'application/json' },
body:
{ subscriber_id: parseInt(userid),
field_name: 'waybill',
field_value: waybill },
json: true };
try {
const response = await rp(options);
console.log(response);
if (response.status == "success") {
return dbRef.child('datanode').child(userid).update({
"Order_Status": 3
}).then(function () {
//do nothing.
}).catch(function (error) {
console.log(error);
});
}
}
catch (err) {
console.log(err.message);
}
}
};
I'm calling this function multiple times as there are many subscribers for whom the function has to run. The code where it's being called is:
dbRef.child("subData").once('value').then(function(snapshot){
var data = snapshot.val();
var keys = Object.keys(data);
let promiseArray = [];
let orderStatusArr =[];
for(let i=0; i< keys.length; i++){
var order_status = data[keys[i]].Order_Status;
var subid = data[keys[i]].UserID;
var waybill = data[keys[i]].Waybill_Number;
if(order_status == 1){
orderStatusArr.push({
"Order_Status":order_status,
"UserID": subid,
"Waybill_Number":waybill
});
}
}
for(let i=0; i<orderStatusArr.length;i++){
var order_status = orderStatusArr[i].Order_Status;
var subid = orderStatusArr[i].UserID;
var waybill = orderStatusArr[i].Waybill_Number;
promiseArray.push(setWaybillsForUser.setWaybillForUser(subid,waybill,dbRef));
}
Promise.all(promiseArray).then(() => {
// all done here
response.send("success");
}).catch(function(err) {
console.log(err.message);
});
})
When I run the code - it logs success for some requests while throwing the above for others. At a time it's able to run the code for about 10 subscribers successfully.
From this page:
Is there any limit to a number of API calls?
There is only a request per second limit for POST methods (per page):
/fb/sending/sendFlow, /fb/sending/sendContent - 25 RPS;
/fb/subscriber/addTag, /fb/subscriber/removeTag, /fb/subscriber/setCustomField - 10 RPS.
So judging from this, and the symptoms you've posted, it looks like the second rule applies

Synchronous/sequential REST calls in loop

I'm trying to call a REST API in a "for" loop, however, the results aren't what I'm expecting.
I've attempted to wrap everything in a promise, but the order of operations is still off, executing it asynchronously rather than synchronously.
var https = require('https');
var zlib = require("zlib");
var axios = require('axios');
const cheerio = require('cheerio');
var page = 1;
var hasMore = "true";
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function getLocation(page) {
// notice that we can await a function
// that returns a promise
await delay();
var apiUrl = 'https://my.api.com/search/advanced?page=' + page +
'&pagesize=5';
https.get(apiUrl, function(response) {
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function(chunk) {
jsonString += chunk;
});
gunzip.on('end', function() {
obj = JSON.parse(jsonString);
var url = obj.items[0].owner.link;
axios(url)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
//OUTPUT LOCATION
console.log($('h3.location').text().trim());
})
.catch(console.error);
});
gunzip.on('error', function(e) {
console.log(e);
});
} else {
console.log("Error");
}
});
}
async function startGetLocation() {
var page = 1;
var hasMore = "true";
do {
//OUTPUT PAGE NUMBER
console.log(page.toString());
await getLocation(page);
page = page + 1;
} while (page < 3);
}
startGetLocation();
Based on the sample code, I would have expected the below to output:
1
New York
2
However, it's outputting:
1
2
New York
The problem is that the callback function that you passed to the https.get() function gets executed asynchronously and that the getLocation function does not wait until this part resolves.
So you could simply wrap the https.get() call and the unzipping part in a promise, wait for it to resolve and then do the axios-part.
async function getLocation(page) {
await delay();
var apiUrl = 'https://my.api.com/search/advanced?page=' + page +
'&pagesize=5';
const fetchAndUnzipPromise = new Promise((resolve, reject) => {
https.get(apiUrl, function (response) {
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function (chunk) {
jsonString += chunk;
});
gunzip.on('end', function () {
obj = JSON.parse(jsonString);
var url = obj.items[0].owner.link;
resolve(url);
});
gunzip.on('error', function (e) {
reject(e);
});
} else {
reject(new Error("Statuscode not as exepcted"));
}
});
});
return fetchAndUnzipPromise.then(url => {
return axios(url)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
//OUTPUT LOCATION
console.log($('h3.location').text().trim());
})
.catch(console.error);
})
}

Facing issue for synchronous operation in NodeJS

Below code have 2 files and having client.js which calls to server file but didn't get the synchronous output.
I tried with promise, bluebird, async_await but didn't get success.
Expected output is a sequence of the alphabet in a console.
Do not use settimout.
Understanding of files.
server.js : server file is having NodeAPI which content only routing.
Client.js : logic which tried is in this file
You need to use two console, first run server.js and second console need to run client.js and output will be print in server.js console.
Expecting output is
a
b
c
d
e
/////////////////server.js/////////////////
var express = require('express');
var bodyParser = require('body-parser')
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
app.post('/', function (req, res) {
console.log(req.body.test)
res.status(200).send('ok');
});
var server = app.listen(3000, function () {
console.log('ok');
});
//////////////client.js///////////////////
//var Promise = require('bluebird');
var request = require('request');
console.log('server file called');
var array_data = ['a', 'b', 'c', 'd', 'e'];
// var promises = [];
// for(let i = 0; i < array_data.length; i++) {
// var promise = request.post({url: 'http://localhost:3000',form : {key:array_data[i]}});
// }
// var page = 0;
// var last_page = array_data.length;
// (function loop() {
// if (page < last_page) {
// request.post({
// url: 'http://localhost:3000',
// form: 'test=' + array_data[page]
// }, function (error, response, body) {
// page++;
// loop();
// });
// }
// }());
// async function loopAsync() {
// for (let i = 0; i < array_data.length; i++) {
// await request.post({
// url: 'http://localhost:3000',
// form: 'test=' + array_data[i]
// });
// }
// }
// loopAsync();
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
await request.post({
url: 'http://localhost:3000',
form: 'test=' + array_data[page]
});
page++;
}
}
loopAsync();
[enter image description here][1]
[1]: https://i.stack.imgur.com/WVlut.png
You can use util.promisify.
Here is sample code
const reqpost = util.promisify(request.post);
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
await reqpost({
url: 'http://localhost:3000',
form: 'test=' + array_data[page]
});
page++;
} }
loopAsync();
request.post is a function that takes request options and a callback like this.
function request(letter, callback = () => {}) {
setTimeout(() => {
console.log(letter);
callback();
}, Math.random() * 1000);
}
What you're doing is calling that function without supplying a callback:
async function clientWithRequest() {
const letters = ['a', 'b', 'c'];
for(let i = 0; i < letters.length; i++) {
await request(letters[i]);
}
}
here, the requests are all fired off at the same time and will return in an indeterminate order.
What you need to do if you want to use async is make your request return a promise. Under the hood, await is really just doing request.then(somethingElse()). So if you change your request to return a promise like:
function requestPromise(letter) {
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log(letter);
resolve();
}, Math.random() * 1000);
});
}
your await code will then work as expected. Effectively it's doing.
request('a').then(() => request('b')).then(() => request('c'));
https://github.com/request/request#forms
Since you are forming the form data string yourself via Node
Is it possible that if you can just let request help you form the encoded string via their form method?
const jobs = [];
const array_data = [
'a',
'b',
'c',
'd',
'e',
];
require('request');
const request = require('request-promise');
async function loopAsync() {
for(const char of array_data) {
jobs.push(
request.post({
url: 'https://marble-scene.glitch.me/',
form: {
test: char
}
})
);
}
const output = await Promise.all(jobs);
output.forEach(char => console.log(char)); // # a b c d e
}
loopAsync();
[Edit]
Just edit the client.js code, fixed the silly syntax error inside the jobs.push block
Working prove: https://glitch.com/edit/#!/marble-scene
As an alternative to importing additional libraries. you can simply "promisify" the callback dependent request.post method.
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
await new Promise((resolve, reject) => {
request.post({
url: 'http://localhost:3000',
form: 'test=' + array_data[page]
}, function(err,httpResponse,body){
if (err) reject(err);
else resolve({httpResponse,body});
});
page++;
});
}
}
loopAsync()
As I did the R&D, request module is not returned promise so for that need to use request-promise library.
let request = require('request');
let rp = require('request-promise');
var array_data = ['a', 'b', 'c', 'd', 'e'];
//with the use of request-promise library
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
var options = {
method: 'POST',
uri: 'http://localhost:3000',
body: {
test: array_data[page]
},
json: true // Automatically stringifies the body to JSON
};
await rp(options);
page++;
}
}
loopAsync();

multiple http get calls nodejs

Thanks for looking into the code.
Here I am fetching some data using feed parser and taking out id's in navcodes array variable and wants to use these Id to make http call.Please find code below.
function processNavCode(){
var mfId = [53];
var preTitle = '';
var navCodes = [];
mfId.forEach(function(id){
var query = "http://portal.xyz.com/Rss.aspx?mf="+id;
feed(query, function(err, feeds) {
if (err) {
throw err;
}
feeds.forEach(function(feed){
var link = feed.link;
var title = feed.title;
var navCode = link.substr(link.length - 6);
if(title.split('-')[0].trim() != preTitle){
preTitle = title;
counter ++;
}
if(parseInt(navCode) != '')
navCodes.push = parseInt(navCode);
});
});
async.eachSeries(navCodes,insertbulkMFValues,function(){
console.log('I am done');
});
// insertbulkMFValues(navCode);
//Directly call insertbulkMFValues function
});
}
I have also tried to call the insertbulkMFValues directly as commented now but due to async nature of nodejs, I am getting the error of either 'Socket hang up' or 'read ECONNRESET'. I checked and used async but not able to work with that also.
var insertbulkMFValues =function(navCode,callback){
var options = {
host: 'www.quandl.com',
path: '/api/v3/datasets/AMFI/'+navCode+'.json?api_key=123456789&start_date=2013-08-30',
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
}
var req1 = https.request(options, function(response) {
var body = '';
response.setEncoding('utf8');
response.on('data', function(chunk) {
body += chunk;
});
response.on('end', function() {
if(typeof body === "string") {
var json = JSON.parse(body);
}
var mfData = json.dataset.data;
var schemeId = json.dataset.dataset_code;
var schemeName = json.dataset.name;
var isinCode = json.dataset.description;
var valueData=[];
for (var k = 0; k < mfData.length; k++) {
var myDate = new Date(mfData[k][0]);
valueData.push({
date:myDate,
NAV:parseFloat(mfData[k][1]).toFixed(2)
});
}
var query = { "navCode": schemeId };
var newData = {
createdDate: Date.now(),
navCode: schemeId,
schemeCode:count,
schemeName:schemeName,
ISINCode:isinCode,
values:valueData
};
MHistory.findOneAndUpdate(query, newData , {upsert:true}, function(err, doc){
if(err)
console.log('Errorr');
else
console.log('Success');
});
});
});
req1.on('error', function(e) {
console.log('problem with request: ' + e.message);
callback(true);
});
req1.end();
}
Thanks in advance..
J
You can directly call insertbulkMFValues for each navCode like:
if(parseInt(navCode) != '') {
insertbulkMFValues(navCode, function () {
console.log('something'}
});
}
Anything that you intend to do must be within the callback of the asynchronous function.
One option for you is to use the waterfall or parallel method of the async library to retrieve all feeds for each id and then invoke
async.eachSeries(navCodesAccumulated,insertbulkMFValues,function(){
console.log('I am done');
});
within the final result callback using the codes obtained.

Resources