How to stream to a Promise? - node.js

Question

When I POST data to my app, I write it to a file and share it with the other instances via PUT. I want to return from POST with the status of each stream (the file and the PUTs).
putResults is an array that's part of the enclosing class, meant to hold the results of each request.
How do I collect the responses? I could return an array of Promises from createWriteStreams but then how could I req.pipe to them? Can you stream to a Promise?
post(req, res, next) {
let listeners = this.getWriteStreams();
let c = listeners.length;
for (i = 0; i < c; i++) {
req.pipe(listeners[i]);
}
/* What do I put here to return after all requests finish? */
}
put(req, res, next) {
var fstream = fs.createWriteStream(this.path);
req.pipe(fstream);
req.on('end', () => {
fstream.close();
res.status(201).send("OK");
});
req.on('error', (err) => {
res.status(500).send(err);
});
}
createWriteStreams() {
let listeners = [];
// We always want to save to a file
listeners.push(fs.createWriteStream(this.path).on('close', ()=>{
this.putResults.push({ host: hutil.myHost, status: 201 });
}));
// If there are other servers in current env, send to them, too!
let otherGuys = hostutil.otherServers();
if (otherGuys.length > 0) {
for (i = 0; i < otherGuys.length; i++) {
let opts = {
hostname: hutil.fq(otherGuys[i]),
port: this.port,
path: this.endpoint,
method: 'PUT',
};
let req = https.request(opts, res => {
this.putResults.push({ host: opts.hostname, status: res.statusCode});
});
req.on('error', (e) => {
this.putResults.push({ host: opts.hostname, status: e });
});
listeners.push(req);
}
}
return listeners;
}

Answer 1

Well, in case anyone ever has this question, the key point of knowledge is that an input stream can be passed around as if it has multiple custom spigots on it - opening one doesn't seem to prematurely spray data all over the other places you're placing the hose!
So, since you cannot stream to a Promise, you still stream to the streams, and you can apparently take your time setting them up. Here's my solution: pass the request to the streams wrapped in the promises.
function post(req, res, next) {
let promises = this.streamPromises(req);
Promise.allSettled(promises).then((results) => {
// Remove the Promise container junk - results come in 2 completely different object structures. Great design, jeez. :-\
let noContainer = results.map(item => item.value).filter(i => i != undefined);
noContainer.push(...results.map(item => item.reason).filter(i => i != undefined));
res.status(200).json(noContainer);
}).catch(err => {
log.warn(`POST request for ${this.filename} failed, at least in part: ${err}`)
res.status(200).json({ host: hutil.myHost, status: err });
});
}
function put(req, res, next) {
var fstream = fs.createWriteStream(this.fqFile);
req.pipe(fstream);
req.on('end', () => {
fstream.close();
log.info(`${req.transID} Saved data to ${this.fqFile} sent by ${req.referer}`);
res.status(201).send("OK");
});
req.on('error', (err) => {
log.warn(`${req.transID} Error receiving/saving PUT file ${this.fqFile} sent by ${req.referer}`);
res.status(500).send(err);
});
}
function streamPromises(postReq) {
let listeners = [];
listeners.push(this.streamLocalFrom(postReq)); // add file first
// Send to other servers in the environment
let otherGuys = hosts.peerServers();
if (otherGuys.length > 0) {
for (i = 0; i < otherGuys.length; i++) {
let opts = {
hostname: hosts.fq(otherGuys[i]),
thatHost: otherGuys[i], // ducked this into the object to avoid parsing fq hostname
port: appconfig.port, // we are all listening here
path: this.endpoint,
method: 'PUT',
timeout: 1000,
ca: fs.readFileSync(appconfig.authorityFile)
};
let p = new Promise((resolve, reject) => {
let req = https.request(opts, res => {
log.info(`${this.filename}: Response from ${opts.hostname}:${opts.port}: ${res.statusCode}`);
// let hn = opts.hostname.match(/(.*?)\./)[1] || opts.hostname;
resolve({ host: opts.thatHost, status: res.statusCode });
});
req.on('error', (e) => {
log.warn(`Error piping ${this.filename} to ${opts.hostname}:${opts.port}: ${e}`);
reject({ host: opts.thatHost, status: e });
});
postReq.pipe(req);
});
listeners.push(p);
}
}
return listeners;
}
function streamLocalFrom(postReq) {
return new Promise((resolve, reject) => {
let fileError = false;
let fstream = fs.createWriteStream(this.fqFile);
fstream.on('close', (err) => {
if (!fileError) {
log.info(`Saved data to file at ${this.fqFile}`);
resolve({ host: me, status: 201 });
}
});
fstream.on('error', (err) => {
log.warn(`Could not save ${this.fqFile} because ${err}`);
reject({ host: hutil.myHost, status: 500 });
fileError = true;
fstream.close();
});
postReq.pipe(fstream);
});
}