I am being challenged trying to make an async call inside an event.
Here's the code from Nodemailer - I've added the line where I need to make an async call:
let transporter = nodemailer.createTransport({
SES: new aws.SES({
apiVersion: '2010-12-01'
}),
sendingRate: 1 // max 1 messages/second
});
// Push next messages to Nodemailer
transporter.on('idle', () => {
while (transporter.isIdle()) {
// I need to make an async db call to get the next email in queue
const mail = await getNextFromQueue()
transporter.sendMail(mail);
}
});
I found this post which suggest switching things around which makes sense however I have been unable to apply it correctly to this.
Update - The answer was to mock sendMail using Sinon.
You can just mark your callback as async and use await inside of it.
The fact that it's an event handler callback makes no difference since at the end it's just a plain-old Function.
Node snippet
'use strict'
const EventEmitter = require('events')
const myEmitter = new EventEmitter()
const getDogs = () => {
return new Promise(resolve => {
setTimeout(() => {
resolve(['Woof', 'Woof', 'Woof'])
}, 500)
})
}
myEmitter.on('event', async () => {
const dogs = await getDogs()
console.log(dogs)
})
myEmitter.emit('event')
Alternative scenario
If you still can't get it to work it might be because transporter.on is not the same as EventEmitter.on - meaning it's a custom function provided by transporter.
It could assume internally that the callback function provided is not a Promise - keep in mind that labelling a function as async forces the function to always implicitly return a Promise.
If that's the case you might want to wrap the async function in an IIFE.
// ..rest of code from above
myEmitter.on('event', () => {
// wrap into an IIFE to make sure that the callback
// itself is not transformed into a Promise
(async function() {
const dogs = await getDogs()
console.log(dogs)
})()
})
myEmitter.emit('event')
I had a similar scenario and if I were you I would have done the following.
let transporter = nodemailer.createTransport({
SES: new aws.SES({
apiVersion: '2010-12-01'
}),
sendingRate: 1 // max 1 messages/second
});
const sendMail = async () => {
while (transporter.isIdle()) {
// I need to make an async db call to get the next email in queue
const mail = await getNextFromQueue()
transporter.sendMail(mail);
}
}
// Push next messages to Nodemailer
transporter.on('idle', sendMail);
Related
I'm trying to learn async/await and your feedback would help a lot.
I'm simply using fs.readFile() as a specific example of functions that has not been modernized with Promises and async/await.
(I'm aware of fs.readFileSync() but I want to learn the concepts.)
Is the pattern below an ok pattern? Are there any issues with it?
const fs = require('fs');
const util = require('util');
//promisify converts fs.readFile to a Promised version
const readFilePr = util.promisify(fs.readFile); //returns a Promise which can then be used in async await
async function getFileAsync(filename) {
try {
const contents = await readFilePr(filename, 'utf-8'); //put the resolved results of readFilePr into contents
console.log('✔️ ', filename, 'is successfully read: ', contents);
}
catch (err){ //if readFilePr returns errors, we catch it here
console.error('⛔ We could not read', filename)
console.error('⛔ This is the error: ', err);
}
}
getFileAsync('abc.txt');
import from fs/promises instead, like this:
const { readFile } = require('fs/promises')
This version returns the promise you are wanting to use and then you don't need to wrap readFile in a promise manually.
Here is some more ways on using async/await
EDITED: as #jfriend00 pointed in comments, of course you have to use standard NodeJS features with built in methods like fs.readFile. So I changed fs method in the code below to something custom, where you can define your own promise.
// Create your async function manually
const asyncFn = data => {
// Instead of result, return promise
return new Promise((resolve, reject) => {
// Here we have two methods: resolve and reject.
// To end promise with success, use resolve
// or reject in opposite
//
// Here we do some task that can take time.
// For example purpose we will emulate it with
// setTimeout delay of 3 sec.
setTimeout(() => {
// After some processing time we done
// and can resolve promise
resolve(`Task completed! Result is ${data * data}`);
}, 3000);
});
}
// Create function from which we will
// call our asyncFn in chain way
const myFunct = () => {
console.log(`myFunct: started...`);
// We will call rf with chain methods
asyncFn(2)
// chain error handler
.catch(error => console.log(error))
// chain result handler
.then(data => console.log(`myFunct: log from chain call: ${data}`));
// Chain call will continue execution
// here without pause
console.log(`myFunct: Continue process while chain task still working.`);
}
// Create ASYNC function to use it
// with await
const myFunct2 = async () => {
console.log(`myFunct2: started...`);
// Read file and wait for result
const data = await asyncFn(3);
// Use your result inline after promise resolved
console.log(`myFunct2: log from async call: ${data}`);
console.log(`myFunct2: continue process after async task completed.`);
}
// Run myFunct
myFunct();
myFunct2();
I wrote this code in lib/helper.js:
var myfunction = async function(x,y) {
....
return [variableA, variableB]
}
exports.myfunction = myfunction;
Then I tried to use it in another file :
var helper = require('./helper.js');
var start = function(a,b){
....
const result = await helper.myfunction('test','test');
}
exports.start = start;
I got an error:
await is only valid in async function
What is the issue?
The error is not refering to myfunction but to start.
async function start() {
....
const result = await helper.myfunction('test', 'test');
}
// My function
const myfunction = async function(x, y) {
return [
x,
y,
];
}
// Start function
const start = async function(a, b) {
const result = await myfunction('test', 'test');
console.log(result);
}
// Call start
start();
I use the opportunity of this question to advise you about an known anti pattern using await which is : return await.
WRONG
async function myfunction() {
console.log('Inside of myfunction');
}
// Here we wait for the myfunction to finish
// and then returns a promise that'll be waited for aswell
// It's useless to wait the myfunction to finish before to return
// we can simply returns a promise that will be resolved later
// useless async here
async function start() {
// useless await here
return await myfunction();
}
// Call start
(async() => {
console.log('before start');
await start();
console.log('after start');
})();
CORRECT
async function myfunction() {
console.log('Inside of myfunction');
}
// Here we wait for the myfunction to finish
// and then returns a promise that'll be waited for aswell
// It's useless to wait the myfunction to finish before to return
// we can simply returns a promise that will be resolved later
// Also point that we don't use async keyword on the function because
// we can simply returns the promise returned by myfunction
function start() {
return myfunction();
}
// Call start
(async() => {
console.log('before start');
await start();
console.log('after start');
})();
Also, know that there is a special case where return await is correct and important : (using try/catch)
Are there performance concerns with `return await`?
To use await, its executing context needs to be async in nature
As it said, you need to define the nature of your executing context where you are willing to await a task before anything.
Just put async before the fn declaration in which your async task will execute.
var start = async function(a, b) {
// Your async task will execute with await
await foo()
console.log('I will execute after foo get either resolved/rejected')
}
Explanation:
In your question, you are importing a method which is asynchronous in nature and will execute in parallel. But where you are trying to execute that async method is inside a different execution context which you need to define async to use await.
var helper = require('./helper.js');
var start = async function(a,b){
....
const result = await helper.myfunction('test','test');
}
exports.start = start;
Wondering what's going under the hood
await consumes promise/future / task-returning methods/functions and async marks a method/function as capable of using await.
Also if you are familiar with promises, await is actually doing the same process of promise/resolve. Creating a chain of promise and executes your next task in resolve callback.
For more info you can refer to MDN DOCS.
When I got this error, it turned out I had a call to the map function inside my "async" function, so this error message was actually referring to the map function not being marked as "async". I got around this issue by taking the "await" call out of the map function and coming up with some other way of getting the expected behavior.
var myfunction = async function(x,y) {
....
someArray.map(someVariable => { // <- This was the function giving the error
return await someFunction(someVariable);
});
}
I had the same problem and the following block of code was giving the same error message:
repositories.forEach( repo => {
const commits = await getCommits(repo);
displayCommit(commits);
});
The problem is that the method getCommits() was async but I was passing it the argument repo which was also produced by a Promise. So, I had to add the word async to it like this: async(repo) and it started working:
repositories.forEach( async(repo) => {
const commits = await getCommits(repo);
displayCommit(commits);
});
If you are writing a Chrome Extension and you get this error for your code at root, you can fix it using the following "workaround":
async function run() {
// Your async code here
const beers = await fetch("https://api.punkapi.com/v2/beers");
}
run();
Basically you have to wrap your async code in an async function and then call the function without awaiting it.
The current implementation of async / await only supports the await keyword inside of async functions Change your start function signature so you can use await inside start.
var start = async function(a, b) {
}
For those interested, the proposal for top-level await is currently in Stage 2: https://github.com/tc39/proposal-top-level-await
async/await is the mechanism of handling promise, two ways we can do it
functionWhichReturnsPromise()
.then(result => {
console.log(result);
})
.cathc(err => {
console.log(result);
});
or we can use await to wait for the promise to full-filed it first, which means either it is rejected or resolved.
Now if we want to use await (waiting for a promise to fulfil) inside a function, it's mandatory that the container function must be an async function because we are waiting for a promise to fulfiled asynchronously || make sense right?.
async function getRecipesAw(){
const IDs = await getIds; // returns promise
const recipe = await getRecipe(IDs[2]); // returns promise
return recipe; // returning a promise
}
getRecipesAw().then(result=>{
console.log(result);
}).catch(error=>{
console.log(error);
});
If you have called async function inside foreach update it to for loop
Found the code below in this nice article: HTTP requests in Node using Axios
const axios = require('axios')
const getBreeds = async () => {
try {
return await axios.get('https://dog.ceo/api/breeds/list/all')
} catch (error) {
console.error(error)
}
}
const countBreeds = async () => {
const breeds = await getBreeds()
if (breeds.data.message) {
console.log(`Got ${Object.entries(breeds.data.message).length} breeds`)
}
}
countBreeds()
Or using Promise:
const axios = require('axios')
const getBreeds = () => {
try {
return axios.get('https://dog.ceo/api/breeds/list/all')
} catch (error) {
console.error(error)
}
}
const countBreeds = async () => {
const breeds = getBreeds()
.then(response => {
if (response.data.message) {
console.log(
`Got ${Object.entries(response.data.message).length} breeds`
)
}
})
.catch(error => {
console.log(error)
})
}
countBreeds()
In later nodejs (>=14), top await is allowed with { "type": "module" } specified in package.json or with file extension .mjs.
https://www.stefanjudis.com/today-i-learned/top-level-await-is-available-in-node-js-modules/
This in one file works..
Looks like await only is applied to the local function which has to be async..
I also am struggling now with a more complex structure and in between different files. That's why I made this small test code.
edit: i forgot to say that I'm working with node.js.. sry. I don't have a clear question. Just thought it could be helpful with the discussion..
function helper(callback){
function doA(){
var array = ["a ","b ","c "];
var alphabet = "";
return new Promise(function (resolve, reject) {
array.forEach(function(key,index){
alphabet += key;
if (index == array.length - 1){
resolve(alphabet);
};
});
});
};
function doB(){
var a = "well done!";
return a;
};
async function make() {
var alphabet = await doA();
var appreciate = doB();
callback(alphabet+appreciate);
};
make();
};
helper(function(message){
console.log(message);
});
A common problem in Express:
The warning can refer to the function, or where you call it.
Express items tend to look like this:
app.post('/foo', ensureLoggedIn("/join"), (req, res) => {
const facts = await db.lookup(something)
res.redirect('/')
})
Notice the => arrow function syntax for the function.
The problem is NOT actually in the db.lookup call, but right here in the Express item.
Needs to be:
app.post('/foo', ensureLoggedIn("/join"), async function (req, res) {
const facts = await db.lookup(something)
res.redirect('/')
})
Basically, nix the => and add async function .
"await is only valid in async function"
But why? 'await' explicitly turns an async call into a synchronous call, and therefore the caller cannot be async (or asyncable) - at least, not because of the call being made at 'await'.
Yes, await / async was a great concept, but the implementation is completely broken.
For whatever reason, the await keyword has been implemented such that it can only be used within an async method. This is in fact a bug, though you will not see it referred to as such anywhere but right here. The fix for this bug would be to implement the await keyword such that it can only be used TO CALL an async function, regardless of whether the calling function is itself synchronous or asynchronous.
Due to this bug, if you use await to call a real asynchronous function somewhere in your code, then ALL of your functions must be marked as async and ALL of your function calls must use await.
This essentially means that you must add the overhead of promises to all of the functions in your entire application, most of which are not and never will be asynchronous.
If you actually think about it, using await in a function should require the function containing the await keyword TO NOT BE ASYNC - this is because the await keyword is going to pause processing in the function where the await keyword is found. If processing in that function is paused, then it is definitely NOT asynchronous.
So, to the developers of javascript and ECMAScript - please fix the await/async implementation as follows...
await can only be used to CALL async functions.
await can appear in any kind of function, synchronous or asynchronous.
Change the error message from "await is only valid in async function" to "await can only be used to call async functions".
I'm building a Slackbot that makes a call to an Express app, which then needs to 1) fetch some other data from the Slack API, and 2) insert resulting data in my database. I think I have the flow right finally using async await, but the operation is timing out because the original call from the Slackbot needs to receive a response within some fixed time I can't control. It would be fine for my purposes to ping the bot with a response immediately, and then execute the rest of the logic asynchronously. But I'm wondering the best way to set this up.
My Express route looks like:
const express = require('express');
const router = express.Router();
const knex = require('../../db/knex.js');
const slack = require('../../services/slack_helpers');
// POST api/slack/foo
router.post('/foo', async (req, res) => {
let [body, images] = await slack.grab_context(req);
knex('texts')
.insert({ body: body,
image_ids: images })
.then(text => { res.send('worked!'); }) // This sends a response back to the original Slackbot call
.catch(err => { res.send(err); })
});
module.exports = router;
And then the slack_helpers module looks like:
const { WebClient } = require('#slack/web-api');
const Slack = new WebClient(process.env.SLACKBOT_TOKEN);
async function grab_context(req) {
try {
const context = await Slack.conversations.history({ // This is the part that takes too long
channel: req.body.channel_id,
latest: req.headers['X-Slack-Request-Timestamp'],
inclusive: true,
limit: 5
});
} catch (error) {
return [error.toString(), 'error'];
}
return await parse_context(context);
};
function parse_context(context) {
var body = [];
context.messages.forEach(message => {
body.push(message.text);
});
body = body.join(' \n');
return [body, ''];
}
module.exports = {
grab_context
};
I'm still getting my head around asynchronous programming, so I may be missing something obvious. I think basically something like res.send perhaps needs to come before the grab_context call? But again, not sure the best flow here.
Update
I've also tried this pattern in the API route, but still getting a timeout:
slack.grab_context(req).then((body, images) => {
knex ...
})
Your timeout may not be coming from where you think. From what I see, it is coming from grab_context. Consider the following simplified version of grab_context
async function grab_context_simple() {
try {
const context = { hello: 'world' }
} catch (error) {
return [error.toString(), 'error']
}
return context
}
grab_context_simple() /* => Promise {
<rejected> ReferenceError: context is not defined
...
} */
You are trying to return context outside of the try block where it was defined, so grab_context will reject with a ReferenceError. It's very likely that this error is being swallowed at the moment, so it would seem like it is timing out.
The fix is to move a single line in grab_context
async function grab_context(req) {
try {
const context = await Slack.conversations.history({
channel: req.body.channel_id,
latest: req.headers['X-Slack-Request-Timestamp'],
inclusive: true,
limit: 5
});
return await parse_context(context); // <- moved this
} catch (error) {
return [error.toString(), 'error'];
}
};
I'm wondering the best way to set this up.
You could add a higher level try/catch block to handle errors that arise from the /foo route. You could also improve readability by staying consistent between async/await and promise chains. Below is how you could use async/await with knex, as well as the aforementioned try/catch block
const express = require('express');
const router = express.Router();
const knex = require('../../db/knex.js');
const slack = require('../../services/slack_helpers');
const insertInto = table => payload => knex(table).insert(payload)
const onFooRequest = async (req, res) => {
try {
let [body, images] = await slack.grab_context(req);
const text = await insertInto('texts')({
body: body,
image_ids: images,
});
res.send('worked!');
} catch (err) {
res.send(err);
}
}
router.post('/foo', onFooRequest);
module.exports = router;
Simple call to ec2 Describing Security groups and returning the security group ID. Using Async / await, but when logging the return value, I get undefined. I fully admit I'm coming from Python and I've tried my hardest to wrap my brain around async calls. I thought I had it nailed, but I'm obviously missing something.
'use strict';
// Load Modules
const AWS = require('aws-sdk')
//Set the region
AWS.config.update({region: 'us-west-2'});
// Call AWS Resources
const ec2 = new AWS.EC2();
// Get Security Group ID From Event
const getSgIdFromEvent = async (event) => {
var ec2params = { Filters: [{Name: 'tag:t_whitelist',Values[event['site']]}]};
await ec2.describeSecurityGroups(ec2params, function (err, response) {
if (err) {return console.error(err.message)}
else {
var sgId = response.SecurityGroups[0].GroupId;
return sgId;
};
});
};
// MAIN FUNCTION
exports.handler = (event, context) => {
getSgIdFromEvent(event)
.then(sgId => {console.log(sgId)});
}
"sgId" should return the security group ID. It does print out fine in the original function before the return.
Typically if it is an async call you want you handle it similar to this way without using a callback
// Load Modules
const AWS = require('aws-sdk')
//Set the region
AWS.config.update({ region: 'us-west-2' });
// Call AWS Resources
const ec2 = new AWS.EC2();
// Get Security Group ID From Event
const getSgIdFromEvent = async (event) => {
var ec2params = { Filters: [{ Name: 'tag:t_whitelist', Values[event['site']]}] };
try {
const securityGroupsDesc = await ec2.describeSecurityGroups(ec2params).promise();
const sgId = securityGroupsDesc.SecurityGroups[0].GroupId;
//do something with the returned result
return sgId;
}
catch (error) {
console.log('handle error');
// throw error;
}
});
};
// MAIN FUNCTION
exports.handler = (event, context) => {
getSgIdFromEvent(event)
.then(sgId => { console.log(sgId) });
}
however if it doesn't support async you just use the callback to handle the returned data or error without using async function.However Reading into AWS docs you can find that the function ec2.describeSecurityGroups() returns an AWS Request
which has a method promise() that needs to be invoked to send the request and get a promise returned.Note that the try catch here is not needed but good to have in case error occurs during the process.
As I said in the comment, chance are that describeSecurityGroups doesn't return a Promise. Try transforming it explictly in a Promise instead:
const promiseResponse = await new Promise((res, rej) => {
ec2.describeSecurityGroups(ec2params, function (err, response) {
if (err) {return rej(err.message)}
else {
var sgId = response.SecurityGroups[0].GroupId;
res(sgId);
};
})
});
// promiseResponse is now equal to sgId inside the callback
return promiseResponse; // this will work because the function is async
Note: You can drop the else keyword
Here is the code that worked using async / await. Thanks to #Cristian Traina I realized ec2.describeSecurityGroups wasn't returning a promise, it was returning an AWS.Event.
// Get Security Group ID From Event
const getSgIdFromEvent = async (event) => {
console.log('Getting Security Group ID')
var params = { Filters: [{Name: 'tag:t_whitelist', Values
[event['site']]}]};
const describeSG = await ec2.describeSecurityGroups(params).promise();
return describeSG.SecurityGroups[0].GroupId;
};
// Get Ingress Rules from Security Group
const getSgIngressRules = async (sgId) => {
console.log(`Getting SG Ingress rules for ${sgId}`)
var params = { GroupIds: [ sgId]};
try{
const ingressRules = await ec2.describeSecurityGroups(params).promise();
return ingressRules;
}
catch (error) {
console.log("Something went wrong getting Ingress Ruls");
}
};
// MAIN FUNCTION
exports.handler = (event, context) => {
getSgIdFromEvent(event)
.then(sgId => {return getSgIngressRules(sgId);})
.then(ingressRules => {console.log(ingressRules);});
}
I submitted this as the answer now since the getSgIdFromEvent function I have, is only 8 lines and still using the async/await like I was desiring.
What I was missing was the .promise() on the end of the function and returning that promise.
Thanks for all the responses!
I use Supertest to test my Express apps, but I'm running into a challenge when I want my handlers to do asynchronous processing after a request is sent. Take this code, for example:
const request = require('supertest');
const express = require('express');
const app = express();
app.get('/user', async (req, res) => {
res.status(200).json({ success: true });
await someAsyncTaskThatHappensAfterTheResponse();
});
describe('A Simple Test', () => {
it('should get a valid response', () => {
return request(app)
.get('/user')
.expect(200)
.then(response => {
// Test stuff here.
});
});
});
If the someAsyncTaskThatHappensAfterTheResponse() call throws an error, then the test here is subject to a race condition where it may or may not failed based on that error. Even aside from error handling, it's also difficult to check for side effects if they happen after the response is set. Imagine that you wanted to trigger database updates after sending a response. You wouldn't be able to tell from your test when you should expect that the updates have completely. Is there any way to use Supertest to wait until the handler function has finished executing?
This can not be done easily because supertest acts like a client and you do not have access to the actual req/res objects in express (see https://stackoverflow.com/a/26811414/387094).
As a complete hacky workaround, here is what worked for me.
Create a file which house a callback/promise. For instance, my file test-hack.js looks like so:
let callback = null
export const callbackPromise = () => new Promise((resolve) => {
callback = resolve
})
export default function callWhenComplete () {
if (callback) callback('hack complete')
}
When all processing is complete, call the callback callWhenComplete function. For instance, my middleware looks like so.
import callWhenComplete from './test-hack'
export default function middlewareIpnMyo () {
return async function route (req, res, next) {
res.status(200)
res.send()
// async logic logic
callWhenComplete()
}
}
And finally in your test, await for the callbackPromise like so:
import { callbackPromise } from 'test-hack'
describe('POST /someHack', () => {
it.only('should handle a post request', async () => {
const response = await request
.post('/someHack')
.send({soMuch: 'hackery'})
.expect(200)
const result = await callbackPromise()
// anything below this is executed after callWhenComplete() is
// executed from the route
})
})
Inspired by #travis-stevens, here is a slightly different solution that uses setInterval so you can be sure the promise is set up before you make your supertest call. This also allows tracking requests by id in case you want to use the library for many tests without collisions.
const backgroundResult = {};
export function backgroundListener(id, ms = 1000) {
backgroundResult[id] = false;
return new Promise(resolve => {
// set up interval
const interval = setInterval(isComplete, ms);
// completion logic
function isComplete() {
if (false !== backgroundResult[id]) {
resolve(backgroundResult[id]);
delete backgroundResult[id];
clearInterval(interval);
}
}
});
}
export function backgroundComplete(id, result = true) {
if (id in backgroundResult) {
backgroundResult[id] = result;
}
}
Make a call to get the listener promise BEFORE your supertest.request() call (in this case, using agent).
it('should respond with a 200 but background error for failed async', async function() {
const agent = supertest.agent(app);
const trackingId = 'jds934894d34kdkd';
const bgListener = background.backgroundListener(trackingId);
// post something but include tracking id
await agent
.post('/v1/user')
.field('testTrackingId', trackingId)
.field('name', 'Bob Smith')
.expect(200);
// execute the promise which waits for the completion function to run
const backgroundError = await bgListener;
// should have received an error
assert.equal(backgroundError instanceof Error, true);
});
Your controller should expect the tracking id and pass it to the complete function at the end of controller backgrounded processing. Passing an error as the second value is one way to check the result later, but you can just pass false or whatever you like.
// if background task(s) were successful, promise in test will return true
backgroundComplete(testTrackingId);
// if not successful, promise in test will return this error object
backgroundComplete(testTrackingId, new Error('Failed'));
If anyone has any comments or improvements, that would be appreciated :)