Testing a callback nested in a promise - node.js

tl;dr I need to test that my method adds a row to a spreadsheet on successful load of a Google spreadsheet.
saveDataToGoogleSpreadSheet(conversationData){
return new Promise((resolve, reject) => {
Spreadsheet.load(this.getGoogleAPISettings(), (err, spreadsheet) => {
if (err) {
return reject(err);
}
return spreadsheet.receive((receivedError, rows, info) => {
if (receivedError) {
return reject(receivedError);
}
const rowData = this.getSpreadsheetRowData(conversationData, info.totalRows);
spreadsheet.add(rowData);
return spreadsheet.send(sendError => (sendError ? reject(sendError) : resolve()));
});
});
});
}
I tested the case one and case two of the function (the two first errors) but I couldn't do it for the last one, the case of success where we an add a row to a spreadsheet.
I need some help with the structure of the test, or a hint on how could my test be.
Edit: how the previous tests were made
it('should add a row to a Google Spreadsheet', (done) => {
nock('https://spreadsheets.google.com')
.post('/feeds/cells/1ZOd7Sysc-JNa-D5AHb7ZJkwBRMBGaeKpzIwEl7B8RbQ/1/private/full/batch')
.replyWithError({ message: 'abcde' });
api.saveDataToGoogleSpreadSheet({ data: 'some data' })
.then(() => done(new Error('should not have made the call')))
.catch((err) => {
expect(err).to.equal('Error Reading Spreadsheet');
done();
});
}).timeout(4000);

It is hard to tell what is the problem with the test from the little code you have, and no background on what the various objects are and how they come to be, so I will just assume that Spreadsheet is an object that is created through a library you require, and other than that, all other objects are created by the module. i.e. I assume you somewhere have a line resembling something like this:
const Spreadsheet = require('some-google-docs-spreadsheet-lib');
That means one problem is finding out how to control the Spreadsheet object so we can stub out its behavior.
Just to start you out, you might get some good pointers on general code and test structure for easy testing from these two answers, as they cover the two most relevant techniques: dependency injection and exploiting link seams.
Mocking Redis Constructor with Sinon
How to test an ES6 class that needs jquery?
For all I know, you might already utilize one of these techniques, as you say you have been able to test the two error situations. But maybe you have not really been unit testing and done the actual network calls to the service instead (which is more of an integration test)? Anyway, I'll assume no more than what I wrote above and show you how to do the testing using proxyquire:
const assert = require('assert');
const dummy = ()=> {};
const SpreadSheetStubLibrary = { load: dummy };
const MyClassToTest = proxyquire('../src/my-module', {
'some-google-docs-spreadsheet-lib': SpreadSheetStubLibrary
})
const config = {};
const conversationData = {};
let stubSetup;
let spreadsheet;
let myObj;
function setupStubs() {
stubSetup = stubSpreadsheetLoadFunction();
spreadsheet = stubSetup.spreadsheet;
SpreadSheetStubLibrary.load = stubSetup.load;
myObj = new MyClassToTest(config);
conversationData = {};
};
function createSpreadsheetStubObj(){
return {
receive: sinon.stub(),
add: sinon.stub(),
send: sinon.stub()
}
}
function stubSpreadsheetLoadFunction(){
const spreadsheet = createSpreadsheetStubObj();
return {
load: (settings, cb) => cb(null, spreadsheet),
spreadSheetStubObj: spreadsheet
};
}
it('should add a row to the spreadsheet on successful load', () => {
// Arrange
setupStubs();
const rowData = { foo: 1, bar: 2};
spreadsheet.receive.yields(); // calls any callback given
myObj.getSpreadsheetRowData = () => rowData; // see lines below
// if you want to use the real getSpreadsheetRowData, uncomment these lines
//const rows = []; // not used in the method?
//const info = { totalRows : 100 };
//spreadsheet.receive.yields(null, rows, info);
// Act
return myObj.saveDataToGoogleSpreadSheet(conversationData).then(()=>{
// Assert
assert(spreadsheet.add.calledOnce);
assert(spreadsheet.add.calledWith(rowData));
});
});
it('should add a row to the spreadsheet on successful load', () => {
// reuse the above
});
See the Sinon docs for the stub API.
Disclosure: I am part of the Sinon maintainer team.

Related

Mocking with Sinon against a service module

I have got myself to the stage of unit testing, and to be honest, with all the different examples online I have got myself confused. I have a good understanding of Mocha & Chai, but Sinon is a different story.
So I have what I think is a pretty straight forward setup. I have a POST route that calls a controller. This controller is like so (removed some basic validation code)
const { createUser } = require('../services/user.service');
const apiResponse = require('../helpers/apiResponse');
const postUser = async (req, res) => {
const user = {
account_id: req.body.id,
status: req.body.status,
created_at: new Date(),
updated_at: new Date(),
};
const result = await createUser(user);
return apiResponse.successResponseWithData(res, 'User added.', result.affectedRows);
} catch (err) {
return apiResponse.errorResponse(res, err);
}
};
module.exports = {
postUser,
};
So all it really does is validate, and then creates a user object with the req and pass that to a service class. This services class does nothing more than pass the data to a database class.
const { addUserToDb } = require('../database/user.db');
const createUser = async (user) => {
try {
const createdUser = await addUserToDb(user);
return createdUser;
} catch (err) {
throw new Error(err);
}
};
module.exports = {
createUser,
};
I wont show the database class because what I want to focus on first is the controller, and then I can hopefully do the rest myself.
So from what I understand, I should be testing functions. If a function makes an external call, I should spy, mock, stub that call? I should only spy, mock or stub this functions dependencies, if one of the dependencies
has its own dependency (like the service module above having a database call dependency), this should be performed in another test? Sorry, just a few questions to help me understand.
Anyways, so I have created a user.controller.test.js file. I have not got far with it, but this is what I have so far
const chai = require('chai');
const sinon = require('sinon');
const { expect } = chai;
const faker = require('faker');
const controller = require('../controllers/user.controller');
const service = require('../services/user.service');
const flushPromises = () => new Promise(setImmediate);
describe('user.controller', () => {
describe('postUser', () => {
beforeEach(() => {
//I see a lot of code use a beforeEach, what should I be doing here?
});
it('should create a user when account_id and status params are provided', async () => {
const req = {
body: { account_id: faker.datatype.uuid(), status: 'true' },
};
const stubValue = {
id: faker.datatype.id(),
account_id: faker.datatype.uuid(),
status: 'true',
created_at: faker.date.past(),
updated_at: faker.date.past(),
};
});
});
});
If I am being totally honest I am pretty lost as to what I should be testing here. From my understanding, I need to mock the service module I think.
Could someone kindly provide some insight as to what I should be doing in this test?
Many thanks
Update
Thank you for your detailed response, I have managed to get a spy working which is a step forward. So I want to do a test on my service module, createUser method.
You can see that my createUser method takes a user Object as a parameter and passes this to a database module where it is inserted into the database and then the user object returned.
So when testing my service class, I need to mock this call to my database module.
const chai = require('chai');
const sinon = require('sinon');
const { expect } = chai;
const faker = require('faker');
const service = require('../services/user.service');
const database = require('../database/user.db');
describe('user.service', () => {
describe('createUser', () => {
it('should create a user when user object is provided', async () => {
const user = {
id: faker.datatype.string(),
status: 'true',
created_at: faker.date.past(),
updated_at: faker.date.past(),
};
const expectedUser = {
id: user.id,
status: user.status,
created_at: user.created_at,
updated_at: user.updated_at,
};
const mockedDatabase = sinon.mock(database);
mockedDatabase.expects('addUserToDb').once().withArgs(expectedUser);
await service.createUser(user);
mockedDatabase.verify();
mockedDatabase.restore();
});
});
});
When I test this, I seem to be getting this response, and it still seems to be inserting the record into my database.
ExpectationError: Expected addUserToDb({
id: 'yX7AX\\J&gf',
status: 'true',
created_at: 2020-06-03T03:10:23.472Z,
updated_at: 2020-05-24T14:44:14.749Z
}, '[...]') once (never called)
at Object.fail (node_modules\sinon\lib\sinon\mock-expectation.js:314:25)
Do you have any idea what I am doing wrong?
Thanks
before I try, I would like to suggest to drop the try/catch blocks everywhere, I will assume you're using expressJs in your Node application, and for such, take a look at express-promise-router as using that Router (instead the default one) will automatically catch anything it was thrown and you just need to focus on the code...
taking your example, you would write:
const { addUserToDb } = require('../database/user.db');
const createUser = async (user) => addUserToDb(user);
module.exports = {
createUser,
};
and
const { createUser } = require('../services/user.service');
const apiResponse = require('../helpers/apiResponse');
const postUser = async (req, res) => {
const { id: account_id, status } = res.body;
const result = await createUser({ account_id, status }); // set the date in the fn
return apiResponse.successResponseWithData(res, 'User added.', result.affectedRows);
};
module.exports = {
postUser,
};
if there's an error and in some place on the route an error is thrown, you will get a nice message back in the response with the error
regarding the code it self, seems a lot cleaner to read - keep in mind that code is for humans, the machine does not even care how you name your variables 😊
Now, regarding the tests ... I do tend to split things into 3 parts
unit tests: the functions itself, single one, like validation, helpers, etc
integration tests: when you call your API endpoint what should be returned
GUI tests (or end-to-end/e2e): applied when a GUI exists, will skip this for now
so in your case, the first thing to make sure of is what are you testing... and taking that, start from the small blocks (unit tests) and move up to the blocks that make sure all is glued together (e2e)
So all it really does is validate, and then creates a user object with the req and pass that to a service class. This services class does nothing more than pass the data to a database class.
Seems a great way to start, so it "validates" ... let's test our validation, let's pass null, undefined, string when all you want is int and so on, until we get a pretty good idea that whatever it passes, we will reply correctly with and without an error
Note I tend to use OpenAPI specs, which makes things easier for me as it provides 2 things
documentation of the endpoints
validation of the endpoints with a nice error message to the consumer
and yes, I always test some validation just to make sure it's working as expected, even though I trust the tool 100% 😜
So from what I understand, I should be testing functions.
well, an application is a group of functions, so all good there 💪
If a function makes an external call, I should spy, mock, stub that call?
I'll try to explain as best as I can what spies, stubs and mocks in Sinon are, please be gentle 🙏
Spies
they tell us information about functions calls, like, number of times called, arguments, return value, and more - they have two types, anonymous spies or spies that wrap methods in our code
function testMyCallback(callback) { callback(); }
describe('testMyCallback fn', function() {
it('should call the callback', function() {
const callbackSpy = sinon.spy(); // anonymous spy - no arguments
testMyCallback(callbackSpy);
expect(callbackSpy).to.have.been.calledOnce;
});
});
const user = {
setNname: function(name) {
this.name = name;
}
}
describe('setname fn', function() {
it('should be called with name', function() {
const setNameSpy = sinon.spy(user, 'setName'); // wrap method spy
user.setName('Katie');
expect(setNameSpy).to.have.been.calledOnce;
expect(setNameSpy).to.have.been.valledWith('Katie');
setNameSpy.restore(); // to remove the Spy and prevent future errors
});
});
Stubs
are power-spies, as they have all the functionality of Spies, but they replace the target function, they have methods that can return a specific value or throw a specific exception and a bit more
they are great to be used with your question regarding external calls, as they replace calls (so you can mock the call behavior and never use the original call)
the simplest of the examples is:
function isAdult(age) {
return age > 21;
}
describe('Sinon Stub Example', () => {
it('should pass', (done) => {
const isAdult = sinon.stub().returns('something');
isAdult(0).should.eql('something');
isAdult(0).should.not.eql(false);
done();
});
});
we've STUB'ed our function, and explicitly said it's a "function" that returns a string something... and for now on, we will never need to go to the function itself, as we have STUB it, we've replaced the real behavior with our own
another example of using STUBs when calling our API application in our integration tests
describe('when we stub our API call', () => {
beforeEach(() => {
this.get = sinon.stub(request, 'get'); // stub "request.get" function
});
afterEach(() => {
request.get.restore(); // remove our power-spy
});
describe('GET /api/v1/accounts', () => {
const responseObject = {
status: 200,
headers: {
'content-type': 'application/json'
}
};
const responseBody = {
status: 'success',
data: [
{
accountId: 1,
status: 'active'
},
{
accountId: 2,
status: 'disabled'
}
]
};
it('should return all accounts', (done) => {
// the 3 objects of our callback (err, res, body)
this.get.yields(null, responseObject, JSON.stringify(responseBody));
request.get(`${base}/api/v1/movies`, (err, res, body) => {
expect(res.statusCode).to.be.eql(200);
expect(res.headers['content-type']).to.contain('application/json');
body = JSON.parse(body);
expect(body).to.be.an('array').that.includes(2);
done();
});
});
});
});
you can also stub axios, but you will need a new library, either moxios, or proxyquire or more...
Mocks
are a bit similar to Stubs (our Power-Spies) but they can be used to replace whole objects and alter their behavior, they are mostly used when you need to stub more than one function from a single object - if all you need is to replace a single function, a stub is easier to use
Mocks can make things oversimplify and you could break your application without even knowing, so be aware...
a normally use is, for example
function setupNewAccount(info, callback) {
const account = {
account_id: info.id,
status: info.status,
created_at: new Date(),
updated_at: new Date()
};
try { Database.save(account, callback); }
catch (err) { callback(err); }
}
describe('setupNewAccount', function() {
it('', function() {
const account = { account_id: 1, status: 'active' };
const expectedAccount = {
account_id: account.id, status: account.status
};
const database = sinon.mock(Database);
database.expectes('save').once().withArgs(expectedAccount);
setupNewAccount(account, function() {});
database.verify();
database.restore();
});
});
something that we will keep forgetting is the .restore() part, and for that, there's a package (one more...) called sinon-test that will auto cleanup at the end of a test
I just hope it helped you with some of your questions and it's a bit clearer now 😏
BTW, for stubbing HTTP requests, I use nock as I think it's much easier to read and use than Sinon, especially for anyone that is reading code for the first time and has no experience in either Sinon or Nock...

How to make a function wait for data to appear in the DB? NodeJS

I am facing a peculiar situation.
I have a backend system (nodejs) which is being called by FE (pretty standard :) ). This endpoint (nodejs) needs to call another system (external) and get the data it produces and return them to the FE. Until now it all might seem pretty usual but here comes the catch.
The external system has async processing and therefore responds to my request immediately but is still processing data (saves them in a DB) and I have to get those data from DB and return them to the FE.
And here goes the question: what is the best (efficient) way of doing it? It usually takes a couple of seconds only and I am very hesitant of making a loop inside the function and for the data to appear in the DB.
Another way would be to have the external system call an endpoint at the end of the processing (if possible - would need to check that with the partner) and wait in the original function until that endpoint is called (not sure exactly how to implement that - so if there is any documentation, article, tutorial, ... would appreciate it very much if you could share guys)
thx for the ideas!
I can give you an example that checks the Database and waits for a while if it can't find a record. And I made a fake database connection for example to work.
// Mocking starts
ObjectID = () => {};
const db = {
collection: {
find: () => {
return new Promise((resolve, reject) => {
// Mock like no record found
setTimeout(() => { console.log('No record found!'); resolve(false) }, 1500);
});
}
}
}
// Mocking ends
const STANDBY_TIME = 1000; // 1 sec
const RETRY = 5; // Retry 5 times
const test = async () => {
let haveFound = false;
let i = 0;
while (i < RETRY && !haveFound) {
// Check the database
haveFound = await checkDb();
// If no record found, increment the loop count
i++
}
}
const checkDb = () => {
return new Promise((resolve) => {
setTimeout(async () => {
record = await db.collection.find({ _id: ObjectID("12345") });
// Check whether you've found or not the record
if (record) return resolve(true);
resolve(false);
}, STANDBY_TIME);
});
}
test();

TypeError: firestoreService.snapshot_ is not a function

I've been using firebase functions test to do some testing on my functions. I have some code that is supposed to post a thing to firestore, basically in the same way that the examples show to do in the realtime database examples:
exports.addMessage = functions.https.onRequest((req, res) => {
const original = req.query.text;
admin.firestore()
.collection('messages')
.add({ original })
.then(documentReference => res.send(documentReference))
.catch(error => res.send(error));
});
For my test, I've spoofed some basic functionality using sinon, mocha and chai. Here is my current test, which is failing with the error message: TypeError: firestoreService.snapshot_ is not a function
describe('addMessage', () => {
// add message should add a message to the database
let oldDatabase;
before(() => {
// Save the old database method so it can be restored after the test.
oldDatabase = admin.firestore;
});
after(() => {
// Restoring admin.database() to the original method.
admin.firestore = oldDatabase;
});
it('should return the correct data', (done) => {
// create stubs
const refStub = sinon.stub();
// create a fake request object
const req = {
query : {
text: 'fly you fools!'
}
};
const snap = test.firestore.makeDocumentSnapshot({ original: req.query.text }, 'messages/1234');
// create a fake document reference
const fakeDocRef = snap._ref;
// create a fake response object
const res = {
send: returnedDocRef => {
// test the result
assert.equal(returnedDocRef, fakeDocRef);
done();
}
};
// spoof firestore
const adminStub = sinon.stub(admin, 'firestore').get(() => () => {
return {
collection: () => {
return {
add: (data) => {
const secondSnap = test.firestore.makeDocumentSnapshot(data, 'messages/1234');
const anotherFakeDocRef = secondSnap._ref;
return Promise.resolve(anotherFakeDocRef);
}
}
}
}
});
// call the function to execute the test above
myFunctions.addMessage(req, res);
});
});
My question is how the heck do I fix this?
I previously had a test that was just passing the first snap and fakeDocRef, and my test was passing fine, but as soon as I resolve the promise with the new fake document reference, it fails...
Any help would be appreciated! Thanks!
There are three different types of the calls, that are different:
Operating on the Collections.
Operating on the Documents.
Operating on the results of the query.
They have to be used consistently.
Please refer a documentation to see the difference operation on the collection and the document.

NodeJs delay each promise within Promise.all()

I'm trying to update a tool that was created a while ago which uses nodejs (I am not a JS developer, so I'm trying to piece the code together) and am getting stuck at the last hurdle.
The new functionality will take in a swagger .json definition, compare the endpoints against the matching API Gateway on the AWS Service, using the 'aws-sdk' SDK for JS and then updates the Gateway accordingly.
The code runs fine on a small definition file (about 15 endpoints) but as soon as I give it a bigger one, I start getting tons of TooManyRequestsException errors.
I understand that this is due to my calls to the API Gateway service being too quick and a delay / pause is needed. This is where I am stuck
I have tried adding;
a delay() to each promise being returned
running a setTimeout() in each promise
adding a delay to the Promise.all and Promise.mapSeries
Currently my code loops through each endpoint within the definition and then adds the response of each promise to a promise array:
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath));
Once the loop is finished I run this:
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
I have tried the same with a mapSeries (no luck).
It looks like the functions within the (getMethodResponse promise) are run immediately and hence, no matter what type of delay I add they all still just execute. My suspicious is that the I need to make (getMethodResponse) return a function and then use mapSeries but I cant get this to work either.
Code I tried:
Wrapped the getMethodResponse in this:
return function(value){}
Then added this after the loop (and within the loop - no difference):
Promise.mapSeries(function (promises) {
return 'a'();
}).then(function (results) {
console.log('result', results);
});
Also tried many other suggestions:
Here
Here
Any suggestions please?
EDIT
As request, some additional code to try pin-point the issue.
The code currently working with a small set of endpoints (within the Swagger file):
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
//Working with small set
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
I have since tried adding this in place of the return Promise.all():
Promise.map(promises, function() {
// Promise.map awaits for returned promises as well.
console.log('X');
},{concurrency: 5})
.then(function() {
return console.log("y");
});
Results of this spits out something like this (it's the same for each endpoint, there are many):
Error: TooManyRequestsException: Too Many Requests
X
Error: TooManyRequestsException: Too Many Requests
X
Error: TooManyRequestsException: Too Many Requests
The AWS SDK is being called 3 times within each promise, the functions of which are (get initiated from the getMethodResponse() function):
apigateway.getRestApisAsync()
return apigateway.getResourcesAsync(resourceParams)
apigateway.getMethodAsync(params, function (err, data) {}
The typical AWS SDK documentation state that this is typical behaviour for when too many consecutive calls are made (too fast). I've had a similar issue in the past which was resolved by simply adding a .delay(500) into the code being called;
Something like:
return apigateway.updateModelAsync(updateModelParams)
.tap(() => logger.verbose(`Updated model ${updatedModel.name}`))
.tap(() => bar.tick())
.delay(500)
EDIT #2
I thought in the name of thorough-ness, to include my entire .js file.
'use strict';
const AWS = require('aws-sdk');
let apigateway, lambda;
const Promise = require('bluebird');
const R = require('ramda');
const logger = require('../logger');
const config = require('../config/default');
const helpers = require('../library/helpers');
const winston = require('winston');
const request = require('request');
const _ = require('lodash');
const region = 'ap-southeast-2';
const methodLib = require('../aws/methods');
const emitter = require('../library/emitter');
emitter.on('updateRegion', (region) => {
region = region;
AWS.config.update({ region: region });
apigateway = new AWS.APIGateway({ apiVersion: '2015-07-09' });
Promise.promisifyAll(apigateway);
});
function getSwaggerFromHttp(externalUrl) {
return new Promise((resolve, reject) => {
request.get({
url: externalUrl,
header: {
"content-type": "application/json"
}
}, (err, res, body) => {
if (err) {
winston.error(err);
reject(err);
}
let result = JSON.parse(body);
resolve(result);
})
});
}
/*
Deletes a method response
*/
function deleteMethodResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteMethodResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Method response ${statusCode} deleted for path: ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Method Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Deletes an integration response
*/
function deleteIntegrationResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteIntegrationResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Integration response ${statusCode} deleted for path ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Integration Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Get Resource
*/
function getMethodResponse(httpMethod, statusCode, apiName, resourcePath) {
let params = {
httpMethod: httpMethod.toUpperCase(),
resourceId: '',
restApiId: ''
}
return getResourceDetails(apiName, resourcePath)
.error((e) => {
logger.unimportant('Error: ' + e.stack)
})
.then((result) => {
//Only run the comparrison of models if the resourceId (from the url passed in) is found within the AWS Gateway
if (result) {
params.resourceId = result.resourceId
params.restApiId = result.apiId
var awsMethodResponses = [];
try {
apigateway.getMethodAsync(params, function (err, data) {
if (err) {
if (err.statusCode == 404) {
return console.log(`Method ${params.httpMethod} not found on resource path: ${resourcePath} (resourceId: ${params.resourceId})`); // an error occurred
}
console.log(err, err.stack); // an error occurred
}
else {
if (data) {
_.each(data.methodResponses, function (value, key) {
if (key >= 200 && key <= 204) {
awsMethodResponses.push(key)
}
});
awsMethodResponses = _.pull(awsMethodResponses, statusCode); //List of items not found within the Gateway - to be removed.
_.each(awsMethodResponses, function (value, key) {
if (data.methodResponses[value].responseModels) {
var existingModel = data.methodResponses[value].responseModels['application/json']; //Check if there is currently a model attached to the resource / method about to be deleted
methodLib.updateResponseAssociation(params.httpMethod, params.resourceId, params.restApiId, statusCode, existingModel); //Associate this model to the same resource / method, under the new response status
}
deleteMethodResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
deleteIntegrationResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
})
}
}
})
.catch(err => {
console.log(`Error: ${err}`);
});
}
catch (e) {
console.log(`getMethodAsync failed, Error: ${e}`);
}
}
})
};
function getResourceDetails(apiName, resourcePath) {
let resourceExpr = new RegExp(resourcePath + '$', 'i');
let result = {
apiId: '',
resourceId: '',
path: ''
}
return helpers.apiByName(apiName, AWS.config.region)
.delay(1200)
.then(apiId => {
result.apiId = apiId;
let resourceParams = {
restApiId: apiId,
limit: config.awsGetResourceLimit,
};
return apigateway.getResourcesAsync(resourceParams)
})
.then(R.prop('items'))
.filter(R.pipe(R.prop('path'), R.test(resourceExpr)))
.tap(helpers.handleNotFound('resource'))
.then(R.head)
.then([R.prop('path'), R.prop('id')])
.then(returnedObj => {
if (returnedObj.id) {
result.path = returnedObj.path;
result.resourceId = returnedObj.id;
logger.unimportant(`ApiId: ${result.apiId} | ResourceId: ${result.resourceId} | Path: ${result.path}`);
return result;
}
})
.catch(err => {
console.log(`Error: ${err} on API: ${apiName} Resource: ${resourcePath}`);
});
};
function delay(t) {
return new Promise(function(resolve) {
setTimeout(resolve, t)
});
}
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
You apparently have a misunderstanding about what Promise.all() and Promise.map() do.
All Promise.all() does is keep track of a whole array of promises to tell you when the async operations they represent are all done (or one returns an error). When you pass it an array of promises (as you are doing), ALL those async operations have already been started in parallel. So, if you're trying to limit how many async operations are in flight at the same time, it's already too late at that point. So, Promise.all() by itself won't help you control how many are running at once in any way.
I've also noticed since, that it seems this line promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath)) is actually executing promises and not simply adding them to the array. Seems like the last Promise.all() doesn't actually do much.
Yep, when you execute promises.push(getMethodResponse()), you are calling getMethodResponse() immediately right then. That starts the async operation immediately. That function then returns a promise and Promise.all() will monitor that promise (along with all the other ones you put in the array) to tell you when they are all done. That's all Promise.all() does. It monitors operations you've already started. To keep the max number of requests in flight at the same time below some threshold, you have to NOT START the async operations all at once like you are doing. Promise.all() does not do that for you.
For Bluebird's Promise.map() to help you at all, you have to pass it an array of DATA, not promises. When you pass it an array of promises that represent async operations that you've already started, it can do no more than Promise.all() can do. But, if you pass it an array of data and a callback function that can then initiate an async operation for each element of data in the array, THEN it can help you when you use the concurrency option.
Your code is pretty complex so I will illustrate with a simple web scraper that wants to read a large list of URLs, but for memory considerations, only process 20 at a time.
const rp = require('request-promise');
let urls = [...]; // large array of URLs to process
Promise.map(urls, function(url) {
return rp(url).then(function(data) {
// process scraped data here
return someValue;
});
}, {concurrency: 20}).then(function(results) {
// process array of results here
}).catch(function(err) {
// error here
});
In this example, hopefully you can see that an array of data items are being passed into Promise.map() (not an array of promises). This, then allows Promise.map() to manage how/when the array is processed and, in this case, it will use the concurrency: 20 setting to make sure that no more than 20 requests are in flight at the same time.
Your effort to use Promise.map() was passing an array of promises, which does not help you since the promises represent async operations that have already been started:
Promise.map(promises, function() {
...
});
Then, in addition, you really need to figure out what exactly causes the TooManyRequestsException error by either reading documentation on the target API that exhibits this or by doing a whole bunch of testing because there can be a variety of things that might cause this and without knowing exactly what you need to control, it just takes a lot of wild guesses to try to figure out what might work. The most common things that an API might detect are:
Simultaneous requests from the same account or source.
Requests per unit of time from the same account or source (such as request per second).
The concurrency operation in Promise.map() will easily help you with the first option, but will not necessarily help you with the second option as you can limit to a low number of simultaneous requests and still exceed a requests per second limit. The second needs some actual time control. Inserting delay() statements will sometimes work, but even that is not a very direct method of managing it and will either lead to inconsistent control (something that works sometimes, but not other times) or sub-optimal control (limiting yourself to something far below what you can actually use).
To manage to a request per second limit, you need some actual time control with a rate limiting library or actual rate limiting logic in your own code.
Here's an example of a scheme for limiting the number of requests per second you are making: How to Manage Requests to Stay Below Rate Limiting.

Nodejs - Mocha, Chai multiple async testing

Complete NodeJS testing noob here. Trying to individually test functions that are called through my API (meaning, rather than make an http request to a specific endpoint, which usually invokes several functions, which in turn make requests to different third party APIs, I want to test the functions themselves separately). The way they're called is I've built a class for each data source (data source = third party API), each class contains the same functions with the same exact signatures - getData and convertData, and return a callback with the results.
I've also created a module that creates many user mocks, since each user context returns different data (meaning, a user object is fed into getData, which uses certain user properties in order to determine what data should be returned).
The way I wanted to test this was to create numerous mocks, then run the functions for each. This is what I've got so far:
// Data sources to iterate over. Each is a class instance acquired through "require".
var dataSources = [
source1,
source2,
source3,
source4
];
describe('getData', function() {
this.timeout(10000);
describe('per data source,', function() {
context('standard call', function() {
// Associative array to hold the data returned, a key for each data source.
var finalResults = {};
// Iterate over all data sources
_.forEach(dataSources, function(dataSource) {
// Generate user mocks
var users = userMocks(10);
// Iterate over all users.
_.forEach(users, function (user) {
// Call each data source with each of the users.
// Numbers of calls to make - (users * data-sources), so in this case - 10*4.
dataSource.getData(user, function (err, data) {
if (err) return done(err);
// Convert the data returned to my format
dataSource.convertData(data, function (err, processedData) {
if (err) return done(err);
// Populate finalResults with converted data from each source
if (finalResults[dataSource.sourceName]) {
finalResults[dataSource.sourceName] = finalResults[dataSource.sourceName].concat(processedData);
} else {
finalResults[dataSource.sourceName] = processedData;
}
});
});
});
});
it('should return something', function(done) {
_.forEach(finalResults.keys, function(key) {
expect(finalResults[key]).to.not.be.empty;
expect(finalResults[key].length).to.be.greaterThan(0);
});
setTimeout(function() {
done();
}, 10000);
})
});
});
});
});`
This works (or at least the test passes when the query is valid, which is what I wanted), but it's cumbersome and (so very) far from elegant or effective, specifically the usage of timeout rather than using promises, async of some sort, or maybe a different alternative I'm not yet familiar with.
Since most of the resources I found (http://alanhollis.com/node-js-testing-a-node-js-api-with-mocha-async-and-should/, https://developmentnow.com/2015/02/05/make-your-node-js-api-bulletproof-how-to-test-with-mocha-chai-and-supertest/, https://justinbellamy.com/testing-async-code-with-mocha/, just to name a few) discuss direct API testing rather than specific async functions, I would love to get some input/best practices tips from more experienced Noders.
You need to know when bunch of asynchronous operations complete. Elegant way to test that is to use promises and promise aggregation:
Promise.all([ promise1, promise2, promise3 ]).then(function(results) {
// all my promises are fulfilled here, and results is an array of results
});
Wrap your dataSources into a promises using bluebird. You don't need to modify tested code self, bluebird provides convenience method:
var Promise = require('bluebird')
var dataSources = [
source1,
source2,
source3,
source4
].map(Promise.promisifyAll);
Use newly promisified functions to create promise for each call:
context('standard call', function() {
var finalResults = {};
var promiseOfResults = datasources.map(function(dataSource) {
var users = userMocks(10);
// Promise.all will take an array of promises and return a promise that is fulfilled then all of promises are
return Promise.all( users.map(function(user) {
// *Async functions are generated by bluebird, via Promise.promisifyAll
return dataSource.getDataAsync(user)
.then(dataSource.convertDataAsync)
.then(function(processedData) {
if (finalResults[dataSource.sourceName]) {
finalResults[dataSource.sourceName] = finalResults[dataSource.sourceName].concat(processedData);
} else {
finalResults[dataSource.sourceName] = processedData;
}
});
});
});
// promiseOfResults consists now of array of agregated promises
it('should return something', function(done) {
// Promise.all agregates all od your 'datasource' promises and is fulfilled when all of them are
// You don't need the promise result here, since you agegated finalResults yourself
return Promise.all( promiseOfResults ).then(function() {
_.forEach(finalResults.keys, function(key) {
expect(finalResults[key]).to.not.be.empty;
expect(finalResults[key].length).to.be.greaterThan(0);
});
done();
});
});
Rest of your test should use same Promise.all( promiseOfResults ), unless you need new set of results.

Resources