Nodejs Stream chaining promises write only the first one - node.js

Question

i'm trying to make a Stream that is writing to multiples files using an async Generator.
Using reduce promises are executed sequentially so the desired output is :
$ wc -l *
42 f1.csv
98 f2.csv
78 f3.csv
but i'm getting the following result :
$ wc -l *
42 f1.csv // only the first one is good
0 f2.csv // nothing in
0 f3.csv // same
AsyncGenerator() is a ElasticSearch Scroll Search (Doc)
I send him different query on each promise.
const { Readable } = require('stream');
const fs = require('fs');
const util = require('util');
const stream = require('stream');
const pipeline = util.promisify(stream.pipeline);
async function* Query(params) {
var response = await ESClient.search(params);
while (true) {
const sourceHits = response.hits.hits;
if (sourceHits.length === 0) {
break;
}
for (const hit of sourceHits) {
yield hit;
}
if (!response._scroll_id) {
break;
}
response = await ESClient.scroll({
scrollId: response._scroll_id,
scroll: params.scroll
});
}
}
async function* AsyncGenerator() {
const params = {
index: 'apache-logs',
scroll: '5s',
_source: ["foo"],
size: 2048,
body: { query } // ElasticSearch DSL Query
};
for await (const res of Query(params)) {
yield res;
}
}
async function WriteToFile(file) {
return new Promise(async (resolve, reject) => {
const input = Readable.from(AsyncGenerator());
const output = fs.createWriteStream(file + '.csv');
await pipeline(
input,
json2csv,
output
);
resolve();
});
}
const lists = ["f1", "f2", "f3"];
const last = lists.reduce(async (previous, next) => {
await previous;
return WriteToFile(next);
}, Promise.resolve());
last.then(_ => {
console.log("DONE");
}).catch((e) => {
console.log(e);
});
(Lorem ipsum lalalaalalal (sry stackoverflow is asking to provide more details but i dont have lol))

Answer 1

Hi I am not sure because my acquaintances are poor in NodeJs. I think you are only calling one time to the function which is creating your .csv
const lists = ["f1", "f2", "f3"];
const last = lists.reduce(async (previous, next) => {
await previous;
return WriteToFile(next);
}, Promise.resolve());