async/promise with multiple api calls - node.js

I kind of understand the differences between callbacks, promises and async await, but I'm not quite sure how to apply this to my problem.
I stripped away a lot of the code, but basically I have an app running with an endpoint that needs to execute 3 functions (maybe I need a 4th?) and also send a "res.end()" within 3 seconds. The 3 functions are dependent on each other. Do I need to chain these functions? My main() seems completely wrong.
deleteRule() has to run and finish first
createRule() must run after deleteRule() has completed
orderRule() must run after createRule() has completed
router.post('/', function (req, res) {
async function deleteRule() {
axios.delete(DeleteURL, {auth: auth, httpsAgent: agent})
.then(response => {
let deleteRes = response.data
console.log(deleteRes)
})
}
const list = './jsons/list.json'
async function createRule() {
fs.readFile(list, 'utf-8', function(err, data) {
if (err) throw err
var thestuff = JSON.parse(data)
axios.post(CreateURL, thestuff, {auth: auth, httpsAgent: agent})
.then(response => {
let createRes = response.data
console.log(createRes)
})
})
}
async function orderRule() {
axios.put(usOrderURL, theOrder, {auth: auth, httpsAgent: agent})
.then(response => {
let orderRes = response.data
console.log(orderRes)
})
}
async function main() {
const deleteListResult = await deleteRule();
const createListResult = await createRule();
const orderListResult = await orderRule();
}
main();
// res.end must finish in 3 seconds from the initial post on the first line
res.end()
})

The then() calls return promises, but you don't do anything with them. You should either await them, or return them.
Since you declared your functions as async, make use of await in them -- that is the whole point of the async keyword:
async function deleteRule() {
let response = await axios.delete(DeleteURL, {auth: auth, httpsAgent: agent});
console.log(response.data);
return response.data;
}
Make a similar change to the other two rule-functions.

import fs from 'fs-extra'
const list = './jsons/list.json'
async function deleteRule() {
const response = await axios.delete(DeleteURL, {auth: auth, httpsAgent: agent})
const deleteRes = response.data
console.log(deleteRes)
return deleteRes
}
async function createRule() {
const data = await fs.readFile(list, 'utf-8')
const theStuff = JSON.parse(data)
const response = await axios.post(CreateURL, theStuff, {auth: auth, httpsAgent: agent})
const createRes = response.data
console.log(createRes)
return createRes
}
async function orderRule() {
const response = await axios.put(usOrderURL, theOrder, {auth: auth, httpsAgent: agent})
const orderRes = response.data
console.log(orderRes)
return orderRes
}
router.post('/', async function (req, res) {
const deleteListResult = await deleteRule();
const createListResult = await createRule();
const orderListResult = await orderRule();
// res.end must finish in 3 seconds from the initial post on the first line
res.end()
})

Related

Mocking function to unit test Serverless Lambda

I am really struggling to understand unit testing within a Serverless Application. So I obviously have my handler, and I have a single Lambda function
const responses = require('../utils/jsonResponse');
const someConnector = require('../services/connectToService/connectToService');
module.exports = async (event) => {
const connectionParams = {
//some env variables
};
try {
const token = await someConnector.connectToService(connectionParams);
return responses.status(token, 200);
} catch (e) {
return responses.status(
`Issue connecting to service - ${e.message}`,
500,
);
}
};
So this Lambda function is pretty straight forward, gets some environment variables, and awaits a response from a service. It then returns the response.
So I have already done integration tests for this which is fine, but now I wanted to do a Unit test. I wanted to test this function in isolation, so essentially I want to mock connectToService to return my own responses.
So I came up with the following
require('dotenv').config();
const { expect } = require('chai');
const sinon = require('sinon');
let sandbox = require("sinon").createSandbox();
const LambdaTester = require('lambda-tester');
const handler = require('../../../handler');
const msConnector = require('../../../services/connectToService/connectToService');
describe('Testing handler', async (done) => {
describe('endpoint someEndpoint returns 200', () => {
it('Should resolve with 200', async () => {
before(() => {
sandbox = sinon.createSandbox();
sandbox.stub(msConnector, 'connectToService').resolves('some-token');
});
afterEach(() => {
sandbox.restore();
});
await LambdaTester(handler.someEndpoint)
.expectResult((result) => {
console.log(result);
expect(result.statusCode).to.equal(200);
});
});
});
done();
});
msConnector is the filename of the service, connectToService is the function name. What I want to do is not invoke this function, but return some-token when my Lambda calls it.
However, I have the console.log, and what I get from that is the real token, not some-token.
This tells me that the mocked function is really being called and executed and returning the real value.
So how can I mock this to make sure it returns some-token?
Thanks
Service function
const { DOMParser } = require('#xmldom/xmldom');
const axios = require('axios');
const { loginRequest } = require('./xml/login');
const connectToService = async (connectionParams) => {
//this injects config details into XML
const xmlRequest = loginRequest(
connectionParams.username,
connectionParams.password,
connectionParams.url,
);
const config = {
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': xmlRequest.length,
},
};
const token = await axios
.post(connectionParams.msHost, xmlRequest, config)
.then((res) => {
const dom = new DOMParser().parseFromString(res.data, 'text/xml');
if (
dom.documentElement
.getElementsByTagName('wsse:secToken')
.item(0)
) {
return dom.documentElement
.getElementsByTagName('wsse:secToken')
.item(0).firstChild.nodeValue;
}
throw new Error('Invalid Username/Password');
})
.catch((err) => {
throw new Error(`Error making connection - ${err.message}`);
});
return token;
};
module.exports = {
connectToService,
};
The function connectToService may be not same copy between you mocked and called.
Because you overwrote a new object by module.exports = .... This causes you probably get different object for each require.
Try to do the below approach sharing the same object for all require.
const { DOMParser } = require('#xmldom/xmldom');
const axios = require('axios');
const { loginRequest } = require('./xml/login');
const connectToService = async (connectionParams) => {
//this injects config details into XML
const xmlRequest = loginRequest(
connectionParams.username,
connectionParams.password,
connectionParams.url,
);
const config = {
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': xmlRequest.length,
},
};
const token = await axios
.post(connectionParams.msHost, xmlRequest, config)
.then((res) => {
const dom = new DOMParser().parseFromString(res.data, 'text/xml');
if (
dom.documentElement
.getElementsByTagName('wsse:secToken')
.item(0)
) {
return dom.documentElement
.getElementsByTagName('wsse:secToken')
.item(0).firstChild.nodeValue;
}
throw new Error('Invalid Username/Password');
})
.catch((err) => {
throw new Error(`Error making connection - ${err.message}`);
});
return token;
};
module.exports.connectToService = connectToService;

How to get Express Node route to wait for function before rendering

I am trying to get a route to wait for an async function in another module to return before render runs, but no matter what I do, res.render always runs first.
This is my current code, which actually just freezes and never loads:
router.get('/', function(req, res, next) {
try {
const cities = spreadsheet.getData()
} catch(err) {
console.log(err)
}
res.render('index', { cities: cities})
})
and the function it is waiting for is this:
exports.getData = function () {
parsedData = [];
accessSpreadsheet().then(function(data) {
console.log(parsedData)
return parsedData;
});
};
const accessSpreadsheet = async() => {
await doc.useServiceAccountAuth({
client_email: process.env.GOOGLE_SERVICE_ACCOUNT_EMAIL,
private_key: process.env.GOOGLE_PRIVATE_KEY,
});
const loadedDoc = await doc.loadInfo();
sheet = await doc.sheetsByIndex[0];
const cells = await sheet.loadCells(allCells[cellsIndex]);
const data = await parseData();
const moreCells = await checkNextCells()
return;
}
The render runs first, and the parsedData prints in the console. I also tried making the route async, and I tried res.render inside a callback. Is there any way to make this work?
Since accessSpreadSheet is an async function, you either need to await or return the promise (as suggested by Patrick Roberts in the comment), in getData function, and similarly in the router.
Using async await you can update your code as below (not tested)
exports.getData = async function () {
parsedData = [];
parsedData = await accessSpreadSheet();
return parsedData;
};
const accessSpreadsheet = async() => {
await doc.useServiceAccountAuth({
client_email: process.env.GOOGLE_SERVICE_ACCOUNT_EMAIL,
private_key: process.env.GOOGLE_PRIVATE_KEY,
});
const loadedDoc = await doc.loadInfo();
sheet = await doc.sheetsByIndex[0];
const cells = await sheet.loadCells(allCells[cellsIndex]);
const data = await parseData();
const moreCells = await checkNextCells()
return;
}
And in the router
router.get('/', async function(req, res, next) {
let cities;
try {
cities = await spreadsheet.getData()
} catch(err) {
console.log(err)
}
res.render('index', { cities: cities})
})
In your router, spreadsheet.getData() being async, you need to handle it with async/wait, which will require your callback to be async.
router.get('/', async function(req, res, next) {
try {
const cities = await spreadsheet.getData();
res.render('index', { cities: cities}) // moved this to inside a try block
} catch(err) {
console.log(err)
// you need to handle the exception here, return error message etc
}
})
In getData, you need to return a promise that will be resolved when called in router.
exports.getData = async function () {
let parsedData = [];
try {
parsedData = await accessSpreadsheet();
} catch (exc) {
// handle exception here
}
return parsedData;
};
finally in accessSpreadsheet(), I do not see where you return the data parsed.
const accessSpreadsheet = async() => {
try{
await doc.useServiceAccountAuth({
client_email: process.env.GOOGLE_SERVICE_ACCOUNT_EMAIL,
private_key: process.env.GOOGLE_PRIVATE_KEY,
});
let loadedDoc = await doc.loadInfo();
let sheet = await doc.sheetsByIndex[0];
let cells = await sheet.loadCells(allCells[cellsIndex]); // cellsIndex is not defined
let data = await parseData(); // are you to pass the sheet? do not know how parsedData() works
let moreCells = await checkNextCells()
return data; // assuming data is what is meant to be returned
} catch(exc) {
// probably return null or re-raise exception
}
}
It is important to always use try/catch or then/catch when dealing with async code in NodeJS.
Hope it sheds some light!

Koa.js always get Not Found 404

I develop in Koa and I use Firebase to messaging, because of a real-time database. When I want to get messages from firebase I get Not found, but in console.log() it shows me.
This is my function to getConversation(Messages)
async getConversation(conversationName, callback) {
var ref = await admin.database().ref(`messages/${conversationName}`)
await ref.on('value', (snapshot, prevChildKey) => {
var newPost = snapshot.val()
let values = Object.values(newPost)
callback(values)
})
}
Then I call it in another controller like this
async getMessages(ctx) {
const id = ctx.params.id
const nameOfConversation = await ctx.db.Conversation.findById(id)
await firebaseIndex.fbController.getConversation(nameOfConversation.name, response => {
console.log(response)
ctx.body = response //TODO
})
}
At the last, I call it in routes.
router.get('/getConversation/:id', middlewares.isAuthenticate, controllers.userConversation.getMessages)
I always get body Not found.
Do anybody know how I can solve it?
I solved it.
async getMessages(ctx) {
const id = ctx.params.id
const nameOfConversation = await ctx.db.Conversation.findById(id)
ctx.body = await new Promise((resolve, reject) => {
firebaseIndex.fbController.getConversation(nameOfConversation.name, async response => {
resolve(response)
})
})
}
ctx.body has to have a Promise.

How to wait for a url callback before send HTTP response in koa?

I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});

Write a loop around async await that will read and write a files in parallel?

I'm using fs and phantomJS
const phantom = require('phantom');
const fs = require('fs');
I have 4 routes (urls) that get opened from phantom JS. When opened, the page content is read and then node.fs will write out that content into it's own html files.
const routes = [
'about',
'home',
'todo',
'lazy',
]
Question:
How do I loop over this async function for every value in const routes in parallel.
(async function() {
const instance = await phantom.create();
const page = await instance.createPage();
const status = await page.open(`http://localhost:3000/${routes}`);
const content = await page.property('content');
await fsPromise(`${routes}.html`, content);
await instance.exit();
}());
const fsPromise = (file, str) => {
return new Promise((resolve, reject) => {
fs.writeFile(file, str, function (err) {
if (err) return reject(err);
resolve(`${routes} > ${routes}.html`);
});
})
};
It took me a while to get this actually up and running in an environment that supports await and async. It turns out Node v7.5.0 supports them - way simpler than fighting with babel! The only other thorn in this investigation was that request-promise, which I was using to test, doesn't seem to fail gracefully when the promise isn't built properly. I saw a lot of errors like this when I tried to use await with it:
return await request.get(options).map(json => json.full_name + ' ' + json.stargazers_count);
^^^^^^^
SyntaxError: Unexpected identifier
In the end though, I realized that your promise function doesn't actually use async/await (which is why mine errored), so the premise should be the same. Here's the test that I got working — it's very similar to yours. The key is in the synchronous for() iteration:
var request = require('request-promise')
var headers = { 'User-Agent': 'YOUR_GITHUB_USERID' }
var repos = [
'brandonscript/usergrid-nodejs',
'facebook/react',
'moment/moment',
'nodejs/node',
'lodash/lodash'
]
function requestPromise(options) {
return new Promise((resolve, reject) => {
request.get(options).then(json => resolve(json.full_name + ' ' + json.stargazers_count))
})
}
(async function() {
for (let repo of repos) {
let options = {
url: 'https://api.github.com/repos/' + repo,
headers: headers,
qs: {}, // or you can put client_id / client secret here
json: true
};
let info = await requestPromise(options)
console.log(info)
}
})()
And while I can't test it, I'm pretty sure this will work:
const routes = [
'about',
'home',
'todo',
'lazy',
]
(async function() {
for (let route of routes) {
const instance = await phantom.create();
const page = await instance.createPage();
const status = await page.open(`http://localhost:3000/${route}`);
const content = await page.property('content');
await fsPromise(`${route}.html`, content);
await instance.exit();
}
}())
Since you're using ES7 syntax, you should also be able to get the fsPromise() function to perform without declaring a promise:
async const fsPromise = (file, str) => {
return await fs.writeFile(file, str)
}

Resources