How to wait for a stream to finish piping? (Nodejs) - node.js

Question

I have a for loop array of promises, so I used Promise.all to go through them and called then afterwards.
let promises = [];
promises.push(promise1);
promises.push(promise2);
promises.push(promise3);
Promise.all(promises).then((responses) => {
for (let i = 0; i < promises.length; i++) {
if (promise.property === something) {
//do something
} else {
let file = fs.createWriteStream('./hello.pdf');
let stream = responses[i].pipe(file);
/*
I WANT THE PIPING AND THE FOLLOWING CODE
TO RUN BEFORE NEXT ITERATION OF FOR LOOP
*/
stream.on('finish', () => {
//extract the text out of the pdf
extract(filePath, {splitPages: false}, (err, text) => {
if (err) {
console.log(err);
} else {
arrayOfDocuments[i].text_contents = text;
}
});
});
}
}
promise1, promise2, and promise3 are some http requests, and if one of them is an application/pdf, then I write it to a stream and parse the text out of it. But this code runs the next iteration before parsing the test out of the pdf. Is there a way to make the code wait until the piping to the stream and extracting are finished before moving on to the next iteration?

Answer 1

Without async/await, it's quite nasty. With async/await, just do this:
Promise.all(promises).then(async (responses) => {
for (...) {
await new Promise(fulfill => stream.on("finish", fulfill));
//extract the text out of the PDF
}
})

Answer 2

Something like the following would also work. I use this pattern fairly often:
let promises = [];
promises.push(promise1);
promises.push(promise2);
promises.push(promise3);
function doNext(){
if(!promises.length) return;
promises.shift().then((resolved) =>{
if(resolved.property === something){
...
doNext();
}else{
let file = fs.createWriteStream('./hello.pdf');
let stream = resolved.pipe(file);
stream.on('finish', () =>{
...
doNext();
});
}
})
}
doNext();
or break up the handler to a controller and Promisified handler:
function streamOrNot(obj){
return new Promise(resolve, reject){
if(obj.property === something){
resolve();
return;
}
let file = fs.createWriteStream...;
stream.on('finish', () =>{
...
resolve();
});
}
}
function doNext(){
if(!promises.length) return;
return promises.shift().then(streamOrNot).then(doNext);
}
doNext()

Answer 3

Use await with stream.pipeline() instead of stream.pipe():
import * as StreamPromises from "stream/promises";
...
await StreamPromises.pipeline(sourceStream, destinationStream);

Answer 4

You can write the else part inside a self invoked function. So that the handling of stream will happen in parallel
(function(i) {
let file = fs.createWriteStream('./hello.pdf');
let stream = responses[i].pipe(file);
/*
I WANT THE PIPING AND THE FOLLOWING CODE
TO RUN BEFORE NEXT ITERATION OF FOR LOOP
*/
stream.on('finish', () => {
//extract the text out of the pdf
extract(filePath, {splitPages: false}, (err, text) => {
if (err) {
console.log(err);
}
else {
arrayOfDocuments[i].text_contents = text;
}
});
});
})(i)
Else you can handle the streaming part as part of the original/individual promise itself.
As of now you are creating the promise and adding it to array, instead of that you add promise.then to the array(which is also a promise). And inside the handler to then you do your streaming stuff.