request-promise loop, how to include request data sent with response? - node.js

I am trying to use request-promise in a loop and then send a response back to the client with all of the responses. The below code works, however I want to also include the request data with each response so that the request ID can be correlated with the result. Is there a built in way to do this:
promiseLoop: function (req, res) {
var ps = [];
for (var i = 0; i < 3; i++) {
// var read_match_details = {
// uri: 'https://postman-echo.com/get?foo1=bar1&foo2=bar2',
// json: true // Automatically parses the JSON string in the response
// };
var session = this.sessionInit(req, res);
if (this.isValidRequest(session)) {
var assertion = session.assertions[i];
const options = {
method: 'POST',
uri: mConfig.serviceURL,
body: assertion,
headers: {
'User-Agent': 'aggregator-service'
},
json: true
}
logger.trace(options);
ps.push(httpClient(options));
}
}
Promise.all(ps)
.then((results) => {
console.log(results); // Result of all resolve as an array
res.status(200);
res.send(results);
res.end();
}).catch(err => console.log(err)); // First rejected promise
}

Assuming httpClient() is request-promise that you refer to and the assertion value is what you're trying to pass through with this result, you could change this:
ps.push(httpClient(options));
to this:
ps.push(httpClient(options).then(result => {
return {id: assertion, result};
}));
Then, your promise would resolve to that object which contains both the result and the id and you could access each in the final array of results.
Your code doesn't show what the current result is. If it's already an object, you could also just add the id property to that object if you'd rather. This is up to you exactly how you put that final result together.
ps.push(httpClient(options).then(result => {
// add id into final result
result.id = assertion;
return result;
}));
Anyway, the general idea is that before putting the promise in the array, you use a .then() handler to slightly modify the returned result, adding in whatever data you want to add and then returning that new modified result so it becomes the resolved value of the promise chain.
To make sure you process all responses, even if some have an error, you can use the newer [Promise.allSettled()][1] instead of Promise.all() and then look through which responses succeeded or failed in processing the results. Or, you can catch any errors, turn them into resolved promises, but give them a sential value (often null) that you can see in processing the final results:
ps.push(httpClient(options).then(result => {
// add id into final result
result.id = assertion;
return result;
}).catch(err => {
console.log(err);
// got an error, but don't want Promise.all() to stop
// so turn the rejected promise into a resolved promise
// that resolves to an object with an error in it
// Processing code can look for an `.err` property.
return {err: err};
}));
Then, later in your processing code:
Promise.all(ps)
.then((results) => {
console.log(results); // Result of all resolve as an array
// filter out error responses
let successResults = results.filter(item => !item.err);
res.send(successResults );
}).catch(err => console.log(err)); // First rejected promise

Promise.allSettled will not stop at error. It make sure you process all responses, even if some have an error.
const request = require('request-promise');
const urls = ["http://", "http://"];
const promises = urls.map(url => request(url));
Promise.allSettled(promises)
.then((data) => {
// data = [promise1,promise2]
})
.catch((err) => {
console.log(JSON.stringify(err, null, 4));
});

Related

Mongoose promise never gets to .then()

I am using q and I have multiple mongoose .exec() promises that never gets to the .then() part of the code, so never allow the q to resolve. Can't figure out why it never comes back.
var defer = q.defer();
var promises = [];
console.log('Exams:', exams.length);
for (var e=0; e<exams.length; e++) {
console.log('Exams:', exams[e]._id);
var newPromise = Pupilexam.find({ _exam: exams[e]._id }).populate('_user').exec()
.then((pupils) => {
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
resolve(exams[e]);
})
.catch((err) => {
reject(err);
});
console.log(typeof newPromise);
promises.push(newPromise);
console.log("Promised pushed");
}
q.all(promises).then(function(data){
console.log("q'd all");
defer.resolve(res.status(200).json(exams));
});
return defer;
The Pupilexam.find().exec() never reaches the .then() so the promises never resolve and the defer never resolves. Why would the mongoose find not get to the .then()? What have I missed?
*** UPDATE ***
Even using the built in promises, we get the same issue. The Pupilexams.find() call never comes back.
var promises = [];
for (var e=0; e<exams.length; e++) {
console.log('e:', e);
console.log('Exam', exams[e]._id);
var newPromise = Pupilexam.find({ _exam: exams[e]._id }).populate('_user').exec()
.then((pupils) => {
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
})
.catch(handleError(res));
promises.push(newPromise);
}
Promise.all(promises).then((exams) => {
console.log(values);
res.status(200).json(exams)
});
With this method I also get a headers error on the call UnhandledPromiseRejectionWarning: Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
** ADDITIONAL CODE REQUESTED **
function handleError(res, statusCode) {
statusCode = statusCode || 500;
return function(err) {
console.log(err.message);
res.status(statusCode).send(err);
};
}
To answer the updated question regarding the Cannot set headers after they are sent to the client error. Looks like you send a response to the client inside your handleError function. Now, if more than one Pupilexam.find call fails, handleError would be invoked twice, resulting in the mentioned error.
You should move the catch-handler down to the Promise.all call:
const promises = [];
for (const exam of exams) {
const newPromise = Pupilexam
.find({ _exam: exam._id }).populate('_user').exec()
.then((pupils) => {
exam.pupils = pupils;
});
promises.push(newPromise);
}
Promise.all(promises)
.then((exams) => {
res.status(200).json(exams);
})
.catch(handleError(res));
I guess that you are indeed returning your promise but you are returning an empty json.
There are 2 problems with your approach:
You are not returning from your then: should return pupils and it is returning undefined
You are logging values that I don't know what it is
.then((pupils) => {
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
// you should return something // return pupils
})
promises.push(newPromise);
Promise.all(promises).then((exams) => {
// ['undefined', 'undefined', ...]
console.log(values);
res.status(200).json(exams)
});
Looks like the answer was that on these two lines the exams[e] is not in scope, because by the time the promise comes back the loop has moved on, so e is wrong and gets too high so it was erroring.
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
Only discovered that when I read #eol's message about the catch and decided to catch it properly and output.
it is Look from your code.
//(async) function
for (var e of exams) {
try {
const pupils = await Pupilexam.find({ _exam: exams[e]._id
}).populate('_user').exec().lean()
e.pupils = pupils
}catch((err){
//handleError
};
}
res.status(200).json({data: exams})
maybe that will show you how match are you wrong

Unable to get async/await working with Axios and array.map()

I'm trying to build a function that maps over a given array and then runs an axios call on each item. Once that is done, I want to return the mapped array for use in another function.
Here is the code:
require('dotenv').config();
const axios = require('axios');
const consola = require('consola');
function getLeagues(listItems) {
const today = new Date();
const season = today.getMonth() >= 6 ? today.getFullYear() : today.getFullYear() - 1; // this gets the game ready for the new season every July 1st
// we will end up directly returning listItems.map once the issue is solved
const selectedLeagues = listItems.map(async (item) => {
const countryName = item.country;
const leagueName = item.league;
try {
const response = await axios({
method: 'GET',
url: `https://api-football-v1.p.rapidapi.com/v2/leagues/country/${countryName}/${season}`,
headers: {
'content-type': 'application/octet-stream',
'x-rapidapi-host': 'api-football-v1.p.rapidapi.com',
'x-rapidapi-key': process.env.FOOTY_API_KEY,
},
});
const leagueData = response.data.api.leagues
.filter((league) => league.name === leagueName)
.map((data) => {
return {
leagueId: data.league_id,
name: data.name,
seasonStart: data.season_start,
seasonEnd: data.season_end,
country: data.country,
};
})
.pop(); // we use pop() because filter() and map() return arrays and we don't want an array of 1 object, just that object
consola.ready({
// this displays all of the data as needed
// this also runs after the below consola block
message: `leagueData: ${JSON.stringify(leagueData, null, 2)}`,
badge: true,
});
return leagueData;
} catch (error) {
throw new Error(error);
}
});
consola.ready({
// this displays an array with an empty object, not an array with above leagueData object
// this also runs before the above consola block
message: `selectedLeagues: ${JSON.stringify(selectedLeagues, null, 2)}`,
badge: true,
});
return selectedLeagues;
}
module.exports = getLeagues;
I'm not sure why the selectedLeagues array is being returned before the leagueData object is even ready. I thought async/await held everything. Instead, in my console, I am getting:
selectedLeagues: [
{}
]
leagueData: {
"leagueId": 753,
"name": "Liga 3",
"seasonStart": "2019-07-19",
"seasonEnd": "2020-05-16",
"country": "Germany"
}
What am I doing wrong?
You have to wrap your listItems.map in a promise all function because map on its own isn't compatible with async.
// Now magically you can add async to your map function...
Promise.all(listItems.map(async item => {
// Then put your try catch here so that it only wraps around
// the results of the function you're awaiting...
let response
try {
response = await axios()
} catch (err) {
return err;
}
// Anything else you want to do with the response...
return response
})).then(results => {
// All the resolved promises returned from the map function.
console.log(results)
})
When you use the await keyword inside an async function the rest of the code will just wait for the result of the awaited function, the try catch part is to catch any error you might get that's out of your control which is why you only try catch around the awaited function.
If you wrap too much of your code inside a try catch you won't be able to diagnose and handle the error properly.
You could put a try catch around the whole of your code if you wanted but the problem would be that the whole code would error out whenever you have any kind of small problem.
You can also do this with a for of loop which might look a bit cleaner...
for await (let item of listItems) {
// try catch await axios etc...
}
You can use async together with a Promise,
const arr = [1, 2, 3];
const asyncRes = await Promise.all(arr.map(async (i) => {
await sleep(10);
return i + 1;
}));
console.log(asyncRes);
// 2,3,4
What's happening in your .map() is async, not what's outside of it. The .map() kicks off but it doesn't block the consola.ready({ at the bottom.

Axios.all, how to configure axios wait time to mitigate hung up?

My application uses an internal webservice for fetching data, i have a job which creates approx 500 requests which getsfired async to complete the fetch operation.
I make use of Axios, by creating an array of axios promises and then resolving them using using Axios.all();
It works fine until some 200 requests but post that i get socket hung up, however on the server side i see the requests are being processed.
How to configure axios to set custom time out, or is it a better idea to splice my promises array and then run them as multiple batches ?
Source code
let getAxiosPromiseArray = (urlList) => {
var axiosArrayofPromise = [];
return new Promise ( (resolve, reject) => {
try {
urlList.forEach ( (URL) => {
axiosArrayofPromise.push(axios.get(URL));
});
resolve(axiosArrayofPromise);
}
catch (err) {
reject("There is a problem getting Axios array of promises " + err);
}
})
}
async function processAxiosPromises (PromiseArray) {
try {
var results = []
results = await axios.all(PromiseArray);
return results;
}
catch(err) {
throw("There was a problem resolving promises array (Axios) " + err);
}
}
getallID().then ( (urlList) => {
return getAxiosPromiseArray(urlList);
}).then( (AxiosPromises) => {
return processAxiosPromises(AxiosPromises);
}).then ((resultData) => {
console.log(resultData);
});
Error
There was a problem resolving promises array (Axios) Error: socket hang up
First, that pair of functions getAxiosPromiseArray() and processAxiosPromises() needs fixing.
Your new Promise() construction is unnecessary. You can simply return Promise.all(arrayofPromise) (or axios.all(...) if you must) and do away with the other function.
Renaming the remaining function to something meaningful, you would end up with eg :
let getData = (urlList) => {
return Promise.all(urlList.map(URL => axios.get(URL)))
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message; // augment the error message ...
throw error; // ... and re-throw the errror.
});
};
And call as follows :
getallID().then(getData)
.then(resultData => {
console.log(resultData);
}).catch(error => {
console.error(error);
});
That will put you on solid ground but, on its own, is unlikely to fix a concurrency problem (if that's what it is), for which the simplest approach is to use Bluebird's Promise.map with the concurrency option.
The caller code can remain the same, just change getData(), as follows:
let getData = (urlList) => {
let concurrency = 10; // play with this value to find a reliable concurrency limit
return Promise.map(urlList, URL => axios.get(URL), {'concurrency': concurrency})
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message;
throw error;
});
};
// where `Promise` is Bluebird.
const axios = require('axios');
const axiosThrottle = require('axios-throttle');
//pass axios object and value of the delay between requests in ms
axiosThrottle.init(axios,200)
const options = {
method: 'GET',
};
const urlList = [
'https://jsonplaceholder.typicode.com/todos/1',
'https://jsonplaceholder.typicode.com/todos/2',
'https://jsonplaceholder.typicode.com/todos/3',
'https://jsonplaceholder.typicode.com/todos/4',
'https://jsonplaceholder.typicode.com/todos/5',
'https://jsonplaceholder.typicode.com/todos/6',
'https://jsonplaceholder.typicode.com/todos/7',
'https://jsonplaceholder.typicode.com/todos/8',
'https://jsonplaceholder.typicode.com/todos/9',
'https://jsonplaceholder.typicode.com/todos/10'
];
const promises = [];
const responseInterceptor = response => {
console.log(response.data);
return response;
};
//add interceptor to work with each response seperately when it is resolved
axios.interceptors.response.use(responseInterceptor, error => {
return Promise.reject(error);
});
for (let index = 0; index < urlList.length; index++) {
options.url = urlList[index];
promises.push(axiosThrottle.getRequestPromise(options, index));
}
//run when all promises are resolved
axios.all(promises).then(responses => {
console.log(responses.length);
});
https://github.com/arekgotfryd/axios-throttle

NodeJs delay each promise within Promise.all()

I'm trying to update a tool that was created a while ago which uses nodejs (I am not a JS developer, so I'm trying to piece the code together) and am getting stuck at the last hurdle.
The new functionality will take in a swagger .json definition, compare the endpoints against the matching API Gateway on the AWS Service, using the 'aws-sdk' SDK for JS and then updates the Gateway accordingly.
The code runs fine on a small definition file (about 15 endpoints) but as soon as I give it a bigger one, I start getting tons of TooManyRequestsException errors.
I understand that this is due to my calls to the API Gateway service being too quick and a delay / pause is needed. This is where I am stuck
I have tried adding;
a delay() to each promise being returned
running a setTimeout() in each promise
adding a delay to the Promise.all and Promise.mapSeries
Currently my code loops through each endpoint within the definition and then adds the response of each promise to a promise array:
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath));
Once the loop is finished I run this:
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
I have tried the same with a mapSeries (no luck).
It looks like the functions within the (getMethodResponse promise) are run immediately and hence, no matter what type of delay I add they all still just execute. My suspicious is that the I need to make (getMethodResponse) return a function and then use mapSeries but I cant get this to work either.
Code I tried:
Wrapped the getMethodResponse in this:
return function(value){}
Then added this after the loop (and within the loop - no difference):
Promise.mapSeries(function (promises) {
return 'a'();
}).then(function (results) {
console.log('result', results);
});
Also tried many other suggestions:
Here
Here
Any suggestions please?
EDIT
As request, some additional code to try pin-point the issue.
The code currently working with a small set of endpoints (within the Swagger file):
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
//Working with small set
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
I have since tried adding this in place of the return Promise.all():
Promise.map(promises, function() {
// Promise.map awaits for returned promises as well.
console.log('X');
},{concurrency: 5})
.then(function() {
return console.log("y");
});
Results of this spits out something like this (it's the same for each endpoint, there are many):
Error: TooManyRequestsException: Too Many Requests
X
Error: TooManyRequestsException: Too Many Requests
X
Error: TooManyRequestsException: Too Many Requests
The AWS SDK is being called 3 times within each promise, the functions of which are (get initiated from the getMethodResponse() function):
apigateway.getRestApisAsync()
return apigateway.getResourcesAsync(resourceParams)
apigateway.getMethodAsync(params, function (err, data) {}
The typical AWS SDK documentation state that this is typical behaviour for when too many consecutive calls are made (too fast). I've had a similar issue in the past which was resolved by simply adding a .delay(500) into the code being called;
Something like:
return apigateway.updateModelAsync(updateModelParams)
.tap(() => logger.verbose(`Updated model ${updatedModel.name}`))
.tap(() => bar.tick())
.delay(500)
EDIT #2
I thought in the name of thorough-ness, to include my entire .js file.
'use strict';
const AWS = require('aws-sdk');
let apigateway, lambda;
const Promise = require('bluebird');
const R = require('ramda');
const logger = require('../logger');
const config = require('../config/default');
const helpers = require('../library/helpers');
const winston = require('winston');
const request = require('request');
const _ = require('lodash');
const region = 'ap-southeast-2';
const methodLib = require('../aws/methods');
const emitter = require('../library/emitter');
emitter.on('updateRegion', (region) => {
region = region;
AWS.config.update({ region: region });
apigateway = new AWS.APIGateway({ apiVersion: '2015-07-09' });
Promise.promisifyAll(apigateway);
});
function getSwaggerFromHttp(externalUrl) {
return new Promise((resolve, reject) => {
request.get({
url: externalUrl,
header: {
"content-type": "application/json"
}
}, (err, res, body) => {
if (err) {
winston.error(err);
reject(err);
}
let result = JSON.parse(body);
resolve(result);
})
});
}
/*
Deletes a method response
*/
function deleteMethodResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteMethodResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Method response ${statusCode} deleted for path: ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Method Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Deletes an integration response
*/
function deleteIntegrationResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteIntegrationResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Integration response ${statusCode} deleted for path ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Integration Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Get Resource
*/
function getMethodResponse(httpMethod, statusCode, apiName, resourcePath) {
let params = {
httpMethod: httpMethod.toUpperCase(),
resourceId: '',
restApiId: ''
}
return getResourceDetails(apiName, resourcePath)
.error((e) => {
logger.unimportant('Error: ' + e.stack)
})
.then((result) => {
//Only run the comparrison of models if the resourceId (from the url passed in) is found within the AWS Gateway
if (result) {
params.resourceId = result.resourceId
params.restApiId = result.apiId
var awsMethodResponses = [];
try {
apigateway.getMethodAsync(params, function (err, data) {
if (err) {
if (err.statusCode == 404) {
return console.log(`Method ${params.httpMethod} not found on resource path: ${resourcePath} (resourceId: ${params.resourceId})`); // an error occurred
}
console.log(err, err.stack); // an error occurred
}
else {
if (data) {
_.each(data.methodResponses, function (value, key) {
if (key >= 200 && key <= 204) {
awsMethodResponses.push(key)
}
});
awsMethodResponses = _.pull(awsMethodResponses, statusCode); //List of items not found within the Gateway - to be removed.
_.each(awsMethodResponses, function (value, key) {
if (data.methodResponses[value].responseModels) {
var existingModel = data.methodResponses[value].responseModels['application/json']; //Check if there is currently a model attached to the resource / method about to be deleted
methodLib.updateResponseAssociation(params.httpMethod, params.resourceId, params.restApiId, statusCode, existingModel); //Associate this model to the same resource / method, under the new response status
}
deleteMethodResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
deleteIntegrationResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
})
}
}
})
.catch(err => {
console.log(`Error: ${err}`);
});
}
catch (e) {
console.log(`getMethodAsync failed, Error: ${e}`);
}
}
})
};
function getResourceDetails(apiName, resourcePath) {
let resourceExpr = new RegExp(resourcePath + '$', 'i');
let result = {
apiId: '',
resourceId: '',
path: ''
}
return helpers.apiByName(apiName, AWS.config.region)
.delay(1200)
.then(apiId => {
result.apiId = apiId;
let resourceParams = {
restApiId: apiId,
limit: config.awsGetResourceLimit,
};
return apigateway.getResourcesAsync(resourceParams)
})
.then(R.prop('items'))
.filter(R.pipe(R.prop('path'), R.test(resourceExpr)))
.tap(helpers.handleNotFound('resource'))
.then(R.head)
.then([R.prop('path'), R.prop('id')])
.then(returnedObj => {
if (returnedObj.id) {
result.path = returnedObj.path;
result.resourceId = returnedObj.id;
logger.unimportant(`ApiId: ${result.apiId} | ResourceId: ${result.resourceId} | Path: ${result.path}`);
return result;
}
})
.catch(err => {
console.log(`Error: ${err} on API: ${apiName} Resource: ${resourcePath}`);
});
};
function delay(t) {
return new Promise(function(resolve) {
setTimeout(resolve, t)
});
}
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
You apparently have a misunderstanding about what Promise.all() and Promise.map() do.
All Promise.all() does is keep track of a whole array of promises to tell you when the async operations they represent are all done (or one returns an error). When you pass it an array of promises (as you are doing), ALL those async operations have already been started in parallel. So, if you're trying to limit how many async operations are in flight at the same time, it's already too late at that point. So, Promise.all() by itself won't help you control how many are running at once in any way.
I've also noticed since, that it seems this line promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath)) is actually executing promises and not simply adding them to the array. Seems like the last Promise.all() doesn't actually do much.
Yep, when you execute promises.push(getMethodResponse()), you are calling getMethodResponse() immediately right then. That starts the async operation immediately. That function then returns a promise and Promise.all() will monitor that promise (along with all the other ones you put in the array) to tell you when they are all done. That's all Promise.all() does. It monitors operations you've already started. To keep the max number of requests in flight at the same time below some threshold, you have to NOT START the async operations all at once like you are doing. Promise.all() does not do that for you.
For Bluebird's Promise.map() to help you at all, you have to pass it an array of DATA, not promises. When you pass it an array of promises that represent async operations that you've already started, it can do no more than Promise.all() can do. But, if you pass it an array of data and a callback function that can then initiate an async operation for each element of data in the array, THEN it can help you when you use the concurrency option.
Your code is pretty complex so I will illustrate with a simple web scraper that wants to read a large list of URLs, but for memory considerations, only process 20 at a time.
const rp = require('request-promise');
let urls = [...]; // large array of URLs to process
Promise.map(urls, function(url) {
return rp(url).then(function(data) {
// process scraped data here
return someValue;
});
}, {concurrency: 20}).then(function(results) {
// process array of results here
}).catch(function(err) {
// error here
});
In this example, hopefully you can see that an array of data items are being passed into Promise.map() (not an array of promises). This, then allows Promise.map() to manage how/when the array is processed and, in this case, it will use the concurrency: 20 setting to make sure that no more than 20 requests are in flight at the same time.
Your effort to use Promise.map() was passing an array of promises, which does not help you since the promises represent async operations that have already been started:
Promise.map(promises, function() {
...
});
Then, in addition, you really need to figure out what exactly causes the TooManyRequestsException error by either reading documentation on the target API that exhibits this or by doing a whole bunch of testing because there can be a variety of things that might cause this and without knowing exactly what you need to control, it just takes a lot of wild guesses to try to figure out what might work. The most common things that an API might detect are:
Simultaneous requests from the same account or source.
Requests per unit of time from the same account or source (such as request per second).
The concurrency operation in Promise.map() will easily help you with the first option, but will not necessarily help you with the second option as you can limit to a low number of simultaneous requests and still exceed a requests per second limit. The second needs some actual time control. Inserting delay() statements will sometimes work, but even that is not a very direct method of managing it and will either lead to inconsistent control (something that works sometimes, but not other times) or sub-optimal control (limiting yourself to something far below what you can actually use).
To manage to a request per second limit, you need some actual time control with a rate limiting library or actual rate limiting logic in your own code.
Here's an example of a scheme for limiting the number of requests per second you are making: How to Manage Requests to Stay Below Rate Limiting.

Function with async request in Node js

I have a loop, which iterates over array and in every iteration I have to do a http request, like this:
var httpsRequest = require('request')
var getData = function(id) {
var result;
httpsRequest({
url: 'https://link/'+id,
}, (error, resp, body) => {
if(resp.statusCode == 200) {
result = JSON.parse(body);
}
});
//here I would like to wait for a result
}
var data = [];
for(row in rows) {
data.push(getData(row.ID))
}
resp.send(JSON.stringify(data)) //I send data back to the client
I cannot do the rest of the for loop in callback, I have to wait for a result which will be returned from a function getData and move to the next iteration.
How to handle this?
PS I know I could use callback function but what if after the last iteration program will send the response (last line above) before the last getData execution finish?
Regards
As stated in the answer by Johannes, the use of promises is a good idea. Since you're using request I'd like to propose an alternative method by using request-promise which is a promisified version of 'request' using bluebird.
The requests will in this case return a promise, and by using .map() you can create an array of promises that you can await using Promise.all(). When all promises are resolved, the response can be sent! This also differs from the use of .reduce(), which only will start to execute the next request as soon as the previous one is done. By using an array of promises, you can start all the requests at the same time.
var httpsRequest = require('request-promise')
var getData = function(id) {
return httpsRequest({
url: 'https://link/' + id,
}, (error, resp, body) => {
if(resp.statusCode == 200) {
return JSON.parse(body);
} else {
//Throw error, this will be caught in the .catch()
throw error;
}
});
}
var promises = rows.map(function(row){
return getData(row.ID)
});
Promise.all(promises)
.then(function(results){
//All requests are done!
//The variable results will be an array of all the results in the same order as they were requested
resp.send(JSON.stringify(results));
})
.catch(function(error){
//Handle the error thrown in the 'getData' function
});
If you need to wait for each iteration to be done before starting another one, you can use Promises and reduce. If you only want to wait for all requests to be finished it's better to use map + Promise.all as explained in Daniel Bs answer.
// i asume rows is an array as you wrote you iterate over one.
const results = [];
rows.reduce((previous, row) => {
return previous.then(() => getData(row.ID).then(result => results.push(result)) // do whatever you want with the result
);
}, Promise.resolve())
.then(() => resp.send(JSON.stringify(results)));
const getData = (id) => {
return new Promise((resolve, reject)=> {
httpsRequest({
url: 'https://link/'+id,
}, (error, resp, body) => {
if(error) return reject(error);
if(resp.statusCode == 200) {
return resolve(JSON.parse(body));
}
return resolve(); // if you want to pass non 200 through. You may want to do sth different here
});
});
};

Resources