how can await cause a for loop to exit - node.js

I've been having an issue with awaiting a function in a for loop in Nodejs
Initially I believed that I was missing something in the stream docs but then rewrote my code to not use streams and still have the same issue
It seems as though using await in a for loop is causing the loop to wait once till the promise resolves then exits the loop
here's an example of my issue: it uses streams as its simpler but I can post a longer example without them if needed
const { PassThrough } = require("stream");
const waitTillStreamEnd = (stream) => {
console.log("waiting for end");
const prom = new Promise((res) => {
stream.on("end", res);
});
return prom.then((e) => {
console.log(e);
return e;
});
};
(async () => {
for (let i = 0; i < 10; i++) {
const stream = new PassThrough();
setTimeout(() => stream.end("test"), 3000);
console.log("timeout set");
const res = await waitTillStreamEnd(stream);
console.log("after await");
}
})();
my output from this is:
timeout set
waiting for end
Nothing after the promise resolves is executed, removing the await causes the loop to run 10 times as I'd expect but it only ones once with the await statement there
Any help would be appreciated

It looks like the promise that you set up in waitTillStreamEnd is not being resolved, which means your loop is not going to get past the first execution.
I note that in waitTillStreamEnd you are returning a promise.then()
However, you should just return the promise, so that it can be properly resolved by the await call in your loop.
Also, with this code:
then((e) => {
console.log(e);
return e;
});
Looks like you are trying to do some error handling with this, but it's not working.
A better place to do error handling would be by providing a second reject callback when setting up your promise, and then listening for the stream error event. This would then cause the error to be propagated through to your for loop for proper handling via a try catch block.
Something along these lines, just typed this on the fly, so there may be some formatting issues :
const { PassThrough } = require("stream");
const waitTillStreamEnd = (stream) => {
console.log("waiting for end");
const prom = new Promise((resolve, reject) => {
stream.on("end", resolve);
stream.on("error", err=>{
reject(err) });
});
return prom;
};
(async () => {
try {
for (let i = 0; i < 10; i++) {
const stream = new PassThrough();
setTimeout(() => stream.end("test"), 3000);
console.log("timeout set");
const res = await waitTillStreamEnd(stream);
console.log("after await");
}
} catch(err) {
console.error(err);
}
})();
Good luck !

Related

How to properly deal with errors while using NodeJS's readline module

I am attempting to read and process a file line by line. I would like to use try / catch async pattern to do this. Below is an example lifted directly from NodeJS docs on how to use readline module.
const { once } = require('events');
const { createReadStream } = require('fs');
const { createInterface } = require('readline');
(async function processLineByLine() {
try {
const rl = createInterface({
input: createReadStream('big-file.txt'),
crlfDelay: Infinity
});
rl.on('line', (line) => {
// Process the line.
});
await once(rl, 'close');
console.log('File processed.');
} catch (err) {
console.error(err);
}
})();
The await once part is throwing me through a loop I think. What I want to do if I encounter an error while parsing the line:
rl.on('line', (line) => {
try {
// Process the line. maybe error from parsing?
JSON.parse(line)
} catch ( error ) {
throw new Error("error while attempting to process json.")
}
});
Is have access to that newly thrown error in the outer try / catch block like:
console.log('File processed.');
} catch (err) {
console.error(err);
// should see "error while attempting to process json."
}
})();
So far the firebase function crashes without ever reaching the outer try / catch block. I've tried adding error event listeners to the readline stream like:
rl.on("error", () => { // throw error here })
with no success.
try/catch only catch synchrone errors. So it won't catch anything from inside rl.on(). Doing await once() just await the stream rl before to execute the console.log('File processed.'); but the try{}catch(e){} has already been executed so any err can't be catch.
rl.on('error', () => {} will only catch the error from the rl stream itself, so even if an error occur at createReadStream('big-file.txt') it won't be catch (and as it is async the final catch(e) won t catch it neither).
To catch any error which occured in rl.on('line ....', one of the solutions is to reject the error. A rejected error into an async/await func will be catch() like in an synchrone flow.
an Exemple
async function processLineByLine() {
try{
async function run() {
const rs = createReadStream(__filename)
// otherwise createReadStream() err are not handled
rs.on('error', () => {
console.log('HandleReadStreanErr')
})
const rl = createInterface({
input: rs,
crlfDelay: Infinity
})
return new Promise((resolve, reject) => {
rl.on('line', (line) => {
try {
throw new Error("error while attempting to process json.")
resolve(console.log(line.toString()))
} catch(e) {
reject(e)
}
})
})
// handle specificaly the rl stream error
rl.on('error', () => console.log('errr rl stream'))
await once(rl, 'close');
console.log('File processed.');
}
// await the overall execution for the catch() to wait
await run()
} catch(e) {
// only rejected err reach here or the one happening synchronously
console.error('eeeeeee')
}
}
processLineByLine()
I personally like to handle each err close to where they occur. But some persons like to handle them at a single place with an err handler. In this case we can wrap the overall execution with a promise
async function processLineByLine() {
try{
async function run() {
return new Promise(async (resolve, reject) => {
const rs = createReadStream('khg.jk')
rs.on('error', () => {
reject('HandleReadStreanErr')
})
const rl = createInterface({
input: rs,
crlfDelay: Infinity
})
rl.on('line', (line) => {
try {
// uncomment following err
// throw new Error("error while attempting to process json.")
resolve(console.log(line.toString()))
} catch(e) {
reject(e)
}
})
rl.on('error', () => reject('errr rl stream'))
await once(rl, 'close');
console.log('File processed.');
})
}
await run()
} catch(e) {
console.error('set error handler: ', e)
}
}
processLineByLine()

async function resolve with Promise but returns undefined

I'm reading data from db with using await so I used Promise but the function seems to return nothing
async function read() {
return new Promise((resolve, reject) => {
const db = new DB();
db
.read()
.then(result => {
resolve(result);
}).catch(() => {
reject('db-error');
});
});
}
(async () => {
const data = await read();
console.log(data); // undefined
})();
How can I make read() return result?
You are making it more complicated than it has to be. If you are already using an API that returns a promise then there is no need to use the promise constructor yourself.
And declaring a function as async is only necessary if you are using await in it to deal with promises.
So either do:
function read() {
const db = new DB();
return db
.read()
.catch(() => {
return 'db-error';
});
}
Or
async function read() {
const db = new DB();
try {
return await db.read();
} catch(error) {
return 'db-error';
}
}
If you are still not getting the value you want then you are not using the database API correctly and you have to read its documentation to figure out how to get back the right data.
The awesome guys who write the MDN Web Docs say that the result of await will be undefined if the promise that is being waited on is rejected: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#handling_rejected_promises
Check out the following scenario.
This is a simple function that returns a Promise:
function asyncFunc(waitTime) {
return new Promise((resolve, reject) => {
setTimeout(() => {
// say we prefer people who do things in 3 seconds or less
if (waitTime <= 3000) {
resolve('Promise resolved! You\'re fast! :)');
} else {
reject('Promise rejected! You\'re slow! :(');
}
}, waitTime);
});
}
Let's test the function using a method similar to yours:
async function testAsyncFunc(waitTime) {
try {
const result = await asyncFunc(waitTime);
console.log(result);
} catch(error) {
console.error(error.message);
}
}
testAsyncFunc(3000); // Returns `Promise resolved! You're fast! :)`, as expected
testAsyncFunc(3001); // Returns `undefined` instead of `Promise rejected! You're slow! :(`
But since we want the actual rejection error of the asynchronous operation instead of undefined, the solution is to chain catch to the await statement to catch any rejection errors immediately you call the asynchronous function and then throw the error so it can be caught by any catch error handler you may want to use, like so:
async function testAsyncFunc(waitTime) {
try {
const result = await asyncFunc(waitTime)
.catch(error => {
// throw the rejection error so it can be handled by the catch block below
throw new Error(error);
});
// if no errors
console.log(result);
} catch(error) {
console.error(error.message);
}
}
testAsyncFunc(3001); // Returns the expected result: `Promise rejected! You're slow! :(`

nodejs async await inside createReadStream

I am reading a CSV file line by line and inserting/updating in MongoDB. The expected output will be
1. console.log(row);
2. console.log(cursor);
3.console.log("stream");
But getting output like
1. console.log(row);
console.log(row); console.log(row); console.log(row); console.log(row); ............ ............
2. console.log(cursor);
3.console.log("stream");
Please let me know what i am missing here.
const csv = require('csv-parser');
const fs = require('fs');
var mongodb = require("mongodb");
var client = mongodb.MongoClient;
var url = "mongodb://localhost:27017/";
var collection;
client.connect(url,{ useUnifiedTopology: true }, function (err, client) {
var db = client.db("UKCompanies");
collection = db.collection("company");
startRead();
});
var cursor={};
async function insertRec(row){
console.log(row);
cursor = await collection.update({CompanyNumber:23}, row, {upsert: true});
if(cursor){
console.log(cursor);
}else{
console.log('not exist')
}
console.log("stream");
}
async function startRead() {
fs.createReadStream('./data/inside/6.csv')
.pipe(csv())
.on('data', async (row) => {
await insertRec(row);
})
.on('end', () => {
console.log('CSV file successfully processed');
});
}
In your startRead() function, the await insertRec() does not stop more data events from flowing while the insertRec() is processing. So, if you don't want the next data event to run until the insertRec() is done, you need to pause, then resume the stream.
async function startRead() {
const stream = fs.createReadStream('./data/inside/6.csv')
.pipe(csv())
.on('data', async (row) => {
try {
stream.pause();
await insertRec(row);
} finally {
stream.resume();
}
})
.on('end', () => {
console.log('CSV file successfully processed');
});
}
FYI, you also need some error handling if insertRec() fails.
That is expected behavior in this case because your on data listener triggers the insertRec asynchronously as and when data is available in stream. So that is why your first line of insert method is getting executed kind of in parallel. If you want to control this behavior you can use highWaterMark (https://nodejs.org/api/stream.html#stream_readable_readablehighwatermark) property while creating the read stream. This way you will get 1 record at a time but I am not sure what your use case is.
something like this
fs.createReadStream(`somefile.csv`, {
"highWaterMark": 1
})
Also you are not awaiting your startRead method. I would wrap it inside the promise and resolve it in end listener else you will not know when the processing got finished. Something like
function startRead() {
return new Promise((resolve, reject) => {
fs.createReadStream(`somepath`)
.pipe(csv())
.on("data", async row => {
await insertRec(row);
})
.on("error", err => {
reject(err);
})
.on("end", () => {
console.log("CSV file successfully processed");
resolve();
});
});
}
From Node 10+ ReadableStream got property Symbol.asyncIterator and is's allow processing stream using for-await-of
async function startRead() {
const readStream = fs.createReadStream('./data/inside/6.csv');
for await (const row of readStream.pipe(csv())) {
await insertRec(row);
}
console.log('CSV file successfully processed');
}

Wait until all promises finished

How can I know when all promises inside the for loop finished executing? Is there a better way to do it?
for (let index = 0; index < array.length; index++) {
request(array[index], function (error, response, body) {
promise1().then(result1 => {
if (result1 !== 0) {
promise2().then(result2 => {
promise3(result2);
}
} else {
promise3(result1);
}
});
});
}
console.log('all promises finished');
This is made easier by transforming the code to use async/await to map the items into promises, then using the standard Promise.all() function to wait for all of the promises to resolve:
// promisified request()
const requestP = item =>
new Promise((resolve, reject) => {
request(item, (error, response, body) => {
if (error) return reject(error);
resolve({ response, body });
});
});
const processItem = async item => {
const { response, body } = await requestP(item);
const result1 = await promise1(/* this probably uses response or body? */);
if (result1 !== 0) {
const result2 = await promise2();
return await promise3(result2);
} else {
return await promise3(result1);
}
};
const promises = array.map(processItem);
Promise.all(promises).then(() => {
console.log("all promises finished");
});
You can keep adding the promises you create in a list inside the for loop.
Once you are done with your loop, you can register a Promise.all method.
let list_of_promises = [];
for(...) {
list_of_promises.push(
new Promise(function(resolve, reject)) {
...
}
)
}
Promise.all(list_of_promises).then(function(data)) {
...
}.catch(function(err) {
...
});
If all your promises get resolved, .then method will be called.
Note: Even if one of your promise fails, it will go inside the .catch method.

Stream Promise results to client as they resolve in Promise.all

In Node, I am using bluebird's Promise.all to execute Promises asynchronously. I don't want to wait for all Promises to resolve before I send results back to the client; rather, I'd like to stream the result of each Promise as soon as it resolves. Is this possible? My research indicates that it is not, but I thought it worth reaching out to the community.
Here is the code (with no attempt to implement streaming):
async bulkExecution(req, res) {
try {
const { assets } = req.body;
let thePromises = _.map(assets, (asset) => {
return onePromise(asset);
});
// I want to stream each Promise result to client as it resolves
let results = await Promise.all(thePromises);
return res.status(200).send(results);
} catch (err) {
return res.status(500).send(err);
}
}
I think you'd have to iterate over all the promises and do a .then() on each promise and write a partial response then close the response after all the promises complete. Something like this should work. The order of the responses may be different than the order of the promise array. For instance, my example below will return the second promise first.
let res = {
write: (data) => {console.log(data)},
end: () => {}
};
let promise1 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve('foo');
}, 500);
});
let promise2 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve('foo');
}, 300);
});
let promises = [promise1, promise2];
let count = 0;
promises.forEach((promise) => {
promise.then((data) => {
res.write(data);
count++;
if (count >= promises.length){
res.end();
}
})
})
Modified to write in JSON format.
let res = {
write: (data) => {
console.log(data)
},
end: () => {}
};
let promise1 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve({foo:"bar"});
}, 500);
});
let promise2 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve({bar:"foo"});
}, 300);
});
let promises = [promise1, promise2];
let count = 0;
res.write("[");
promises.forEach((promise) => {
promise.then((data) => {
res.write(JSON.stringify(data));
count++;
if (count >= promises.length) {
res.write("]");
res.end();
} else {
res.write(",");
}
})
})
Streaming out JSON like this isn't common, but could be used to reduce the amount of memory the server uses. That's because you don't need to create the full response in memory before sending the response. The client will still need enough memory to read the entire object into memory.
Try doing .then( data => data.json) on the promise like so
let thePromises = _.map(assets, (asset) => {
return onePromise(asset).then(data => data.json);
});

Resources