I'm trying to both be able to handle a paginated API, as well as do retries if throttled for too many requests. The pagination is handled by recursing if 'nextToken' is present in the response object. I'm hoping to be able to catching a Throttling Exception, and effectively start the whole request over by recursing without passing the token. This is my current code:
function getAllExecHist(execArn) {
var sfn = new AWS.StepFunctions();
sfn = Promise.promisifyAll(sfn);
execHists = [];
return new Promise(function(resolve, reject) {
function getExecHist(nextToken) {
params = {};
params.executionArn = execArn;
if (nextToken !== undefined) {
params.nextToken = nextToken;
}
sfn.getExecutionHistoryAsync(params)
.then(function(results) {
execHists = execHists.concat(results.events);
if (!results.nextToken) {
resolve(execHists);
}
else {
getExecHist(results.nextToken);
}
})
.catch(function(e) {
console.log('caught this: ', e);
console.log('retrying');
return new Promise(function(res, rej) {
console.log('Sleeping');
setTimeout(function() {
execHists = [];
res(getExecHist());
}, random(100,10000));
});
})
}
getExecHist();
});
}
The recursion was handling pagination without issue, but since adding the catch, it simply never returns. Any ideas what I'm doing wrong / how to fix?
The AWS SDK supports promises and you can configure Bluebird as it's promise library.
const Promise = require('bluebird');
const AWS = require('aws');
AWS.config.setPromisesDependency(Promise);
const sfn = new AWS.StepFunctions();
Use Promise.delay() instead of setTimeout.
Try and avoid creating new promises if functions are already returning them. Only wrap a promise in new Promise if you have a lot of synchronous code that might throw an error or needs to resolve the promise early.
The following also avoids the extra function and nested scope by passing values between function calls.
function getExecHist(execArn, execHists, nextToken) {
let params = {};
params.executionArn = execArn;
if ( nextToken !== undefined ) params.nextToken = nextToken;
if ( execHists === undefined ) execHists = [];
return sfn.getExecutionHistory(params).promise()
.then(results => {
execHists = execHists.concat(results.events);
if (!results.nextToken) return execHists;
return getExecHist(execArn, execHists, results.nextToken);
})
.catch(e => {
console.log('caught this: ', e);
console.log('retrying');
return Promise.delay(random(100,10000))
.then(() => getExecHist(execArn));
})
}
Eventually you should be specific about what errors you retry on and include a count or time limit too.
Also note that this is the wrong way to retry a rate limit issue as this starts again from the beginning. A rate limit retry should continue from where it left off, otherwise you are just adding to your rate limit problems.
Related
I am using q and I have multiple mongoose .exec() promises that never gets to the .then() part of the code, so never allow the q to resolve. Can't figure out why it never comes back.
var defer = q.defer();
var promises = [];
console.log('Exams:', exams.length);
for (var e=0; e<exams.length; e++) {
console.log('Exams:', exams[e]._id);
var newPromise = Pupilexam.find({ _exam: exams[e]._id }).populate('_user').exec()
.then((pupils) => {
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
resolve(exams[e]);
})
.catch((err) => {
reject(err);
});
console.log(typeof newPromise);
promises.push(newPromise);
console.log("Promised pushed");
}
q.all(promises).then(function(data){
console.log("q'd all");
defer.resolve(res.status(200).json(exams));
});
return defer;
The Pupilexam.find().exec() never reaches the .then() so the promises never resolve and the defer never resolves. Why would the mongoose find not get to the .then()? What have I missed?
*** UPDATE ***
Even using the built in promises, we get the same issue. The Pupilexams.find() call never comes back.
var promises = [];
for (var e=0; e<exams.length; e++) {
console.log('e:', e);
console.log('Exam', exams[e]._id);
var newPromise = Pupilexam.find({ _exam: exams[e]._id }).populate('_user').exec()
.then((pupils) => {
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
})
.catch(handleError(res));
promises.push(newPromise);
}
Promise.all(promises).then((exams) => {
console.log(values);
res.status(200).json(exams)
});
With this method I also get a headers error on the call UnhandledPromiseRejectionWarning: Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
** ADDITIONAL CODE REQUESTED **
function handleError(res, statusCode) {
statusCode = statusCode || 500;
return function(err) {
console.log(err.message);
res.status(statusCode).send(err);
};
}
To answer the updated question regarding the Cannot set headers after they are sent to the client error. Looks like you send a response to the client inside your handleError function. Now, if more than one Pupilexam.find call fails, handleError would be invoked twice, resulting in the mentioned error.
You should move the catch-handler down to the Promise.all call:
const promises = [];
for (const exam of exams) {
const newPromise = Pupilexam
.find({ _exam: exam._id }).populate('_user').exec()
.then((pupils) => {
exam.pupils = pupils;
});
promises.push(newPromise);
}
Promise.all(promises)
.then((exams) => {
res.status(200).json(exams);
})
.catch(handleError(res));
I guess that you are indeed returning your promise but you are returning an empty json.
There are 2 problems with your approach:
You are not returning from your then: should return pupils and it is returning undefined
You are logging values that I don't know what it is
.then((pupils) => {
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
// you should return something // return pupils
})
promises.push(newPromise);
Promise.all(promises).then((exams) => {
// ['undefined', 'undefined', ...]
console.log(values);
res.status(200).json(exams)
});
Looks like the answer was that on these two lines the exams[e] is not in scope, because by the time the promise comes back the loop has moved on, so e is wrong and gets too high so it was erroring.
console.log("Adding pupils", exams[e]._id);
exams[e].pupils = pupils;
Only discovered that when I read #eol's message about the catch and decided to catch it properly and output.
it is Look from your code.
//(async) function
for (var e of exams) {
try {
const pupils = await Pupilexam.find({ _exam: exams[e]._id
}).populate('_user').exec().lean()
e.pupils = pupils
}catch((err){
//handleError
};
}
res.status(200).json({data: exams})
maybe that will show you how match are you wrong
I would like to have a configuration file with variables set with data I fetch from an API.
I think I must use async and await features to do so, otherwise my variable would stay undefined.
But I don't know how to integrate this and keep the node exports.myVariable = myData available within an async function ?
Below is the code I tried to write to do so (all in the same file) :
const fetchAPI = function(jsonQuery) {
return new Promise(function (resolve, reject) {
var reqOptions = {
headers: apiHeaders,
json:jsonQuery,
}
request.post(apiURL, function (error, res, body) {
if (!error && res.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
}
var wallsData = {}
const fetchWalls = async function (){
var jsonQuery = [{ "recordType": "page","query": "pageTemplate = 1011"}]
let body = await utils.fetchAPI(jsonQuery)
let pageList = await body[0].dataHashes
for(i=0;i<pageList.length;i++){
var page = pageList[i]
wallsData[page.title.fr] = [page.difficultyList,page.wallType]
}
return wallsData
throw new Error("WOOPS")
}
try{
const wallsData = fetchWalls()
console.log(wallsData)
exports.wallsData = wallsData
}catch(err){
console.log(err)
}
The output of console.log(wallsData) shows Promise { <pending> }, therefore it is not resolved and the configuration file keep being executed without the data in wallsData...
What do I miss ?
Thanks,
Cheers
A promise is a special object that either succeeds with a result or fails with a rejection. The async-await-syntax is syntactic sugar to help to deal with promises.
If you define a function as aync it always will return a promise.
Even a function like that reads like
const foo = async() => {
return "hello";
}
returns a promise of a string, not only a string. And you need to wait until it's been resolved or rejected.
It's analogue to:
const foo = async() => {
return Promise.resolve("Hello");
}
or:
const foo = async() => {
return new Promise(resolve => resolve("Hello"));
}
Your fetchWalls similarly is a promise that will remain pending for a time. You'll have to make sure it either succeeds or fails by setting up the then or catch handlers in your outer scope:
fetchWalls()
.then(console.log)
.catch(console.error);
The outer scope is never async, so you cannot use await there. You can only use await inside other async functions.
I would also not use your try-catch for that outer scope promise handling. I think you are confusing the try-catch approach that is intended to be used within async functions, as there it helps to avoid nesting and reads like synchronous code:
E.g. you could do inside your fetchWalls defintion:
const fetchWalls = async function (){
var jsonQuery = [{ "recordType": "page","query": "pageTemplate = 1011"}]
try {
let body = await utils.fetchAPI(jsonQuery)
} catch(e) {
// e is the reason of the promise rejection if you want to decide what to do based on it. If you would not catch it, the rejection would chain through to the first error handler.
}
...
}
Can you change the statements like,
try{
const wallsData = fetchWalls();
wallsData.then((result) => {
console.log(result);
});
exports.wallsData = wallsData; // when importing in other file this returns as promise and we should use async/await to handle this.
}catch(err){
console.log(err)
}
My application uses an internal webservice for fetching data, i have a job which creates approx 500 requests which getsfired async to complete the fetch operation.
I make use of Axios, by creating an array of axios promises and then resolving them using using Axios.all();
It works fine until some 200 requests but post that i get socket hung up, however on the server side i see the requests are being processed.
How to configure axios to set custom time out, or is it a better idea to splice my promises array and then run them as multiple batches ?
Source code
let getAxiosPromiseArray = (urlList) => {
var axiosArrayofPromise = [];
return new Promise ( (resolve, reject) => {
try {
urlList.forEach ( (URL) => {
axiosArrayofPromise.push(axios.get(URL));
});
resolve(axiosArrayofPromise);
}
catch (err) {
reject("There is a problem getting Axios array of promises " + err);
}
})
}
async function processAxiosPromises (PromiseArray) {
try {
var results = []
results = await axios.all(PromiseArray);
return results;
}
catch(err) {
throw("There was a problem resolving promises array (Axios) " + err);
}
}
getallID().then ( (urlList) => {
return getAxiosPromiseArray(urlList);
}).then( (AxiosPromises) => {
return processAxiosPromises(AxiosPromises);
}).then ((resultData) => {
console.log(resultData);
});
Error
There was a problem resolving promises array (Axios) Error: socket hang up
First, that pair of functions getAxiosPromiseArray() and processAxiosPromises() needs fixing.
Your new Promise() construction is unnecessary. You can simply return Promise.all(arrayofPromise) (or axios.all(...) if you must) and do away with the other function.
Renaming the remaining function to something meaningful, you would end up with eg :
let getData = (urlList) => {
return Promise.all(urlList.map(URL => axios.get(URL)))
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message; // augment the error message ...
throw error; // ... and re-throw the errror.
});
};
And call as follows :
getallID().then(getData)
.then(resultData => {
console.log(resultData);
}).catch(error => {
console.error(error);
});
That will put you on solid ground but, on its own, is unlikely to fix a concurrency problem (if that's what it is), for which the simplest approach is to use Bluebird's Promise.map with the concurrency option.
The caller code can remain the same, just change getData(), as follows:
let getData = (urlList) => {
let concurrency = 10; // play with this value to find a reliable concurrency limit
return Promise.map(urlList, URL => axios.get(URL), {'concurrency': concurrency})
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message;
throw error;
});
};
// where `Promise` is Bluebird.
const axios = require('axios');
const axiosThrottle = require('axios-throttle');
//pass axios object and value of the delay between requests in ms
axiosThrottle.init(axios,200)
const options = {
method: 'GET',
};
const urlList = [
'https://jsonplaceholder.typicode.com/todos/1',
'https://jsonplaceholder.typicode.com/todos/2',
'https://jsonplaceholder.typicode.com/todos/3',
'https://jsonplaceholder.typicode.com/todos/4',
'https://jsonplaceholder.typicode.com/todos/5',
'https://jsonplaceholder.typicode.com/todos/6',
'https://jsonplaceholder.typicode.com/todos/7',
'https://jsonplaceholder.typicode.com/todos/8',
'https://jsonplaceholder.typicode.com/todos/9',
'https://jsonplaceholder.typicode.com/todos/10'
];
const promises = [];
const responseInterceptor = response => {
console.log(response.data);
return response;
};
//add interceptor to work with each response seperately when it is resolved
axios.interceptors.response.use(responseInterceptor, error => {
return Promise.reject(error);
});
for (let index = 0; index < urlList.length; index++) {
options.url = urlList[index];
promises.push(axiosThrottle.getRequestPromise(options, index));
}
//run when all promises are resolved
axios.all(promises).then(responses => {
console.log(responses.length);
});
https://github.com/arekgotfryd/axios-throttle
I'm trying to execute a for loop within a promise with no success, i think my issue has to do with where to call resolve but i'm not sure
/*
* Get conversations of user
* #param user {String}
*/
function getConversations(user){
return new Promise(function(resolve, reject){
var conversations = user.Conversations
var newConversations = []
for(var conversation of conversations) {
helperGetConvo(conversation.ConversID).then(function(convo){
newConversations.push(createConversationObject({messages:[], name:convo.conversationName, users:["broulaye", "doumbia"], Id:convo.conversationID}))
}).catch(function(reason) {
console.log("failure when finding conversation 2: " + reason)
})
}
resolve(newConversations)
})
}
function helperGetConvo(convoId) {
return new Promise (function(resolve, reject){
query.findConversation(convoId).then(function(convers) {
if(convers) {
console.log("conversation was found: " + convers)
}
else {
console.log("conversation was not found: " + convers)
}
resolve(convers)
}).catch(function(reason) {
console.log("failure when finding conversation: " + reason)
})
})
}
when I execute my code like this the getConversations function only returns an empty array. but when i change the getConversations function like so:
function getConversations(user){
return new Promise(function(resolve, reject){
var conversations = user.Conversations
var newConversations = []
for(var conversation of conversations) {
helperGetConvo(conversation.ConversID).then(function(convo){
newConversations.push(createConversationObject({messages:[], name:convo.conversationName, users:["broulaye", "doumbia"], Id:convo.conversationID}))
resolve(newConversations)
}).catch(function(reason) {
console.log("failure when finding conversation 2: " + reason)
})
}
})
}
i do get an output, however it does not get through the whole forloop i believe because from my understanding resolve work like a return statement.
someone help plz
You need to use Promise.all
function getConversations(user){
var conversations = user.Conversations
var promises = conversations.map(c=>helperGetConvo(c.ConversID))
return Promise.all(promises)
.then(data=>{
let newConversations = data.map(convo=>{
return createConversationObject({messages:[], name:convo.conversationName, users:["broulaye", "doumbia"], Id:convo.conversationID})
})
return newConversations
})
.catch(reason=>{
console.log("failure when finding conversation: " + reason)
})
}
Use the function like so
getConversations(user).then(newConversations=>{
//your code
})
One way is to collect the promises in an array using map instead of for-in. Then use Promise.all() to wait for all of them to resolve (or one to reject).
Something like:
return Promise.all(conversations.map(conversation => {
return helperGetConvo(...).then().catch();
}
Remember all promises MUST either resolve or reject. If you don't follow this rule you get into trouble.
You can loop a promise within a helper function that I found when I was trying to address a similar issue. I use this method to loop promises as it will not fall over at the first rejected promise. Instead I can handle the resolve or reject and return the final result once the loop has finished. The Promise in the code snippet below is using bluebird, http://bluebirdjs.com/docs/getting-started.html
function promiseWhile(condition, action) {
return new Promise((resolve, reject) => {
var loop = () => {
if (!condition()) return resolve();
return Promise.cast(action())
.then(loop)
.catch(reject);
};
process.nextTick(loop);
return resolve;
})
}
I modified your code examples provided with some dummy data and got it to work with the helper function. As a result I believe your getConversations function would look like this:
function getConversations(user) {
var conversations = user.Conversations;
var newConversations = [];
var stop = conversations.length;
var index = 0
//loop promise
return promiseWhile(() => {
// Condition for stopping
return index < stop;
}, () => {
// Action to run, should return a promise
return new Promise((resolve, reject) => {
helperGetConvo(conversations[index].ConversID)
.then(function(convo) {
newConversations.push(createConversationObject({
messages: [],
name: convo.conversationName,
users: ['broulaye', 'doumbia'],
Id: convo.conversationID
}));
index++;
resolve();
})
.catch((error) => {
console.log('failure when finding conversation: ' + error);
index++;
resolve();
});
})
})
//This will execute when loop ends
.then(() => {
return newConversations;
});
}
Hope this helps.
The problem is that when you call resolve you are resolving the entire promise. The for loop does not wait for each helperGetConvo() call to finish before moving on to the next one. Whichever of those promises hits the then statement first will call resolve and that's what your outer promise will resolve to.
You can read more on promises at: Understanding promises in node.js.
If you want to wait for a group of promises to finish, use Promise.all. It takes in a list of promises, and will only resolve if all promises complete successfully.
function getConversations(user) {
return new Promise(function (resolve, reject) {
var conversations = user.Conversations;
var newConversations = [];
//create a list of promises
var promises = [];
for (var conversation of conversations) {
// push each promise into our array
promises.push(
helperGetConvo(conversation.ConversID).then(function (convo) {
newConversations.push(createConversationObject({
messages: [],
name: convo.conversationName,
users: ['broulaye', 'doumbia'],
Id: convo.conversationID
}));
}).catch(function (reason) {
console.log('failure when finding conversation 2: ' + reason);
})
);
}
// wait for all promises to complete
// when then do, resolve the newConversations variable
// which will now have all of the conversation objects that we wanted to create
Promise.all(promises).then(() => resolve(newConversations)).catch(reject);
});
}
You could also use async/await to clean this up. Async/await provides some nice syntactic sugar for removing the need to do return new Promise(...). This next code snippet is not the best way to use async/await because the for loop will process everything synchronously (one conversation at a time). This blog post has been hugely helpful for my understanding of using async/await in iterative problems: https://blog.lavrton.com/javascript-loops-how-to-handle-async-await-6252dd3c795.
async function getConversations(user) {
var conversations = user.Conversations;
var newConversations = [];
// process each converstaion in sequence
for (var conversation of conversations) {
// instead of doing .then() we can use await
// convo will have the result from the helperGetConvo
// we put it in a try/catch because output
// we still want to have the error if something fails
try {
var convo = await helperGetConvo(conversation.ConversID);
newConversations.push(createConversationObject({
messages: [],
name: convo.conversationName,
users: ['broulaye', 'doumbia'],
Id: convo.conversationID
}));
} catch(reason) {
console.log('failure when finding conversation 2: ' + reason);
}
}
// return
return newConversations;
}
Async functions return promises. So you can call this function by doing getConversations(user).then(...). But I think async/await makes your code look much cleaner. There are definitely further optimizations that you can do, but hopefully this gets you started.
I'm trying to update a tool that was created a while ago which uses nodejs (I am not a JS developer, so I'm trying to piece the code together) and am getting stuck at the last hurdle.
The new functionality will take in a swagger .json definition, compare the endpoints against the matching API Gateway on the AWS Service, using the 'aws-sdk' SDK for JS and then updates the Gateway accordingly.
The code runs fine on a small definition file (about 15 endpoints) but as soon as I give it a bigger one, I start getting tons of TooManyRequestsException errors.
I understand that this is due to my calls to the API Gateway service being too quick and a delay / pause is needed. This is where I am stuck
I have tried adding;
a delay() to each promise being returned
running a setTimeout() in each promise
adding a delay to the Promise.all and Promise.mapSeries
Currently my code loops through each endpoint within the definition and then adds the response of each promise to a promise array:
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath));
Once the loop is finished I run this:
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
I have tried the same with a mapSeries (no luck).
It looks like the functions within the (getMethodResponse promise) are run immediately and hence, no matter what type of delay I add they all still just execute. My suspicious is that the I need to make (getMethodResponse) return a function and then use mapSeries but I cant get this to work either.
Code I tried:
Wrapped the getMethodResponse in this:
return function(value){}
Then added this after the loop (and within the loop - no difference):
Promise.mapSeries(function (promises) {
return 'a'();
}).then(function (results) {
console.log('result', results);
});
Also tried many other suggestions:
Here
Here
Any suggestions please?
EDIT
As request, some additional code to try pin-point the issue.
The code currently working with a small set of endpoints (within the Swagger file):
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
//Working with small set
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
I have since tried adding this in place of the return Promise.all():
Promise.map(promises, function() {
// Promise.map awaits for returned promises as well.
console.log('X');
},{concurrency: 5})
.then(function() {
return console.log("y");
});
Results of this spits out something like this (it's the same for each endpoint, there are many):
Error: TooManyRequestsException: Too Many Requests
X
Error: TooManyRequestsException: Too Many Requests
X
Error: TooManyRequestsException: Too Many Requests
The AWS SDK is being called 3 times within each promise, the functions of which are (get initiated from the getMethodResponse() function):
apigateway.getRestApisAsync()
return apigateway.getResourcesAsync(resourceParams)
apigateway.getMethodAsync(params, function (err, data) {}
The typical AWS SDK documentation state that this is typical behaviour for when too many consecutive calls are made (too fast). I've had a similar issue in the past which was resolved by simply adding a .delay(500) into the code being called;
Something like:
return apigateway.updateModelAsync(updateModelParams)
.tap(() => logger.verbose(`Updated model ${updatedModel.name}`))
.tap(() => bar.tick())
.delay(500)
EDIT #2
I thought in the name of thorough-ness, to include my entire .js file.
'use strict';
const AWS = require('aws-sdk');
let apigateway, lambda;
const Promise = require('bluebird');
const R = require('ramda');
const logger = require('../logger');
const config = require('../config/default');
const helpers = require('../library/helpers');
const winston = require('winston');
const request = require('request');
const _ = require('lodash');
const region = 'ap-southeast-2';
const methodLib = require('../aws/methods');
const emitter = require('../library/emitter');
emitter.on('updateRegion', (region) => {
region = region;
AWS.config.update({ region: region });
apigateway = new AWS.APIGateway({ apiVersion: '2015-07-09' });
Promise.promisifyAll(apigateway);
});
function getSwaggerFromHttp(externalUrl) {
return new Promise((resolve, reject) => {
request.get({
url: externalUrl,
header: {
"content-type": "application/json"
}
}, (err, res, body) => {
if (err) {
winston.error(err);
reject(err);
}
let result = JSON.parse(body);
resolve(result);
})
});
}
/*
Deletes a method response
*/
function deleteMethodResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteMethodResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Method response ${statusCode} deleted for path: ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Method Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Deletes an integration response
*/
function deleteIntegrationResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteIntegrationResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Integration response ${statusCode} deleted for path ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Integration Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Get Resource
*/
function getMethodResponse(httpMethod, statusCode, apiName, resourcePath) {
let params = {
httpMethod: httpMethod.toUpperCase(),
resourceId: '',
restApiId: ''
}
return getResourceDetails(apiName, resourcePath)
.error((e) => {
logger.unimportant('Error: ' + e.stack)
})
.then((result) => {
//Only run the comparrison of models if the resourceId (from the url passed in) is found within the AWS Gateway
if (result) {
params.resourceId = result.resourceId
params.restApiId = result.apiId
var awsMethodResponses = [];
try {
apigateway.getMethodAsync(params, function (err, data) {
if (err) {
if (err.statusCode == 404) {
return console.log(`Method ${params.httpMethod} not found on resource path: ${resourcePath} (resourceId: ${params.resourceId})`); // an error occurred
}
console.log(err, err.stack); // an error occurred
}
else {
if (data) {
_.each(data.methodResponses, function (value, key) {
if (key >= 200 && key <= 204) {
awsMethodResponses.push(key)
}
});
awsMethodResponses = _.pull(awsMethodResponses, statusCode); //List of items not found within the Gateway - to be removed.
_.each(awsMethodResponses, function (value, key) {
if (data.methodResponses[value].responseModels) {
var existingModel = data.methodResponses[value].responseModels['application/json']; //Check if there is currently a model attached to the resource / method about to be deleted
methodLib.updateResponseAssociation(params.httpMethod, params.resourceId, params.restApiId, statusCode, existingModel); //Associate this model to the same resource / method, under the new response status
}
deleteMethodResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
deleteIntegrationResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
})
}
}
})
.catch(err => {
console.log(`Error: ${err}`);
});
}
catch (e) {
console.log(`getMethodAsync failed, Error: ${e}`);
}
}
})
};
function getResourceDetails(apiName, resourcePath) {
let resourceExpr = new RegExp(resourcePath + '$', 'i');
let result = {
apiId: '',
resourceId: '',
path: ''
}
return helpers.apiByName(apiName, AWS.config.region)
.delay(1200)
.then(apiId => {
result.apiId = apiId;
let resourceParams = {
restApiId: apiId,
limit: config.awsGetResourceLimit,
};
return apigateway.getResourcesAsync(resourceParams)
})
.then(R.prop('items'))
.filter(R.pipe(R.prop('path'), R.test(resourceExpr)))
.tap(helpers.handleNotFound('resource'))
.then(R.head)
.then([R.prop('path'), R.prop('id')])
.then(returnedObj => {
if (returnedObj.id) {
result.path = returnedObj.path;
result.resourceId = returnedObj.id;
logger.unimportant(`ApiId: ${result.apiId} | ResourceId: ${result.resourceId} | Path: ${result.path}`);
return result;
}
})
.catch(err => {
console.log(`Error: ${err} on API: ${apiName} Resource: ${resourcePath}`);
});
};
function delay(t) {
return new Promise(function(resolve) {
setTimeout(resolve, t)
});
}
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
You apparently have a misunderstanding about what Promise.all() and Promise.map() do.
All Promise.all() does is keep track of a whole array of promises to tell you when the async operations they represent are all done (or one returns an error). When you pass it an array of promises (as you are doing), ALL those async operations have already been started in parallel. So, if you're trying to limit how many async operations are in flight at the same time, it's already too late at that point. So, Promise.all() by itself won't help you control how many are running at once in any way.
I've also noticed since, that it seems this line promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath)) is actually executing promises and not simply adding them to the array. Seems like the last Promise.all() doesn't actually do much.
Yep, when you execute promises.push(getMethodResponse()), you are calling getMethodResponse() immediately right then. That starts the async operation immediately. That function then returns a promise and Promise.all() will monitor that promise (along with all the other ones you put in the array) to tell you when they are all done. That's all Promise.all() does. It monitors operations you've already started. To keep the max number of requests in flight at the same time below some threshold, you have to NOT START the async operations all at once like you are doing. Promise.all() does not do that for you.
For Bluebird's Promise.map() to help you at all, you have to pass it an array of DATA, not promises. When you pass it an array of promises that represent async operations that you've already started, it can do no more than Promise.all() can do. But, if you pass it an array of data and a callback function that can then initiate an async operation for each element of data in the array, THEN it can help you when you use the concurrency option.
Your code is pretty complex so I will illustrate with a simple web scraper that wants to read a large list of URLs, but for memory considerations, only process 20 at a time.
const rp = require('request-promise');
let urls = [...]; // large array of URLs to process
Promise.map(urls, function(url) {
return rp(url).then(function(data) {
// process scraped data here
return someValue;
});
}, {concurrency: 20}).then(function(results) {
// process array of results here
}).catch(function(err) {
// error here
});
In this example, hopefully you can see that an array of data items are being passed into Promise.map() (not an array of promises). This, then allows Promise.map() to manage how/when the array is processed and, in this case, it will use the concurrency: 20 setting to make sure that no more than 20 requests are in flight at the same time.
Your effort to use Promise.map() was passing an array of promises, which does not help you since the promises represent async operations that have already been started:
Promise.map(promises, function() {
...
});
Then, in addition, you really need to figure out what exactly causes the TooManyRequestsException error by either reading documentation on the target API that exhibits this or by doing a whole bunch of testing because there can be a variety of things that might cause this and without knowing exactly what you need to control, it just takes a lot of wild guesses to try to figure out what might work. The most common things that an API might detect are:
Simultaneous requests from the same account or source.
Requests per unit of time from the same account or source (such as request per second).
The concurrency operation in Promise.map() will easily help you with the first option, but will not necessarily help you with the second option as you can limit to a low number of simultaneous requests and still exceed a requests per second limit. The second needs some actual time control. Inserting delay() statements will sometimes work, but even that is not a very direct method of managing it and will either lead to inconsistent control (something that works sometimes, but not other times) or sub-optimal control (limiting yourself to something far below what you can actually use).
To manage to a request per second limit, you need some actual time control with a rate limiting library or actual rate limiting logic in your own code.
Here's an example of a scheme for limiting the number of requests per second you are making: How to Manage Requests to Stay Below Rate Limiting.