return new Promise((resolve, reject) => someReadStream
.pipe(decoder)
.on('data', data => somePromise(data))
.on('end', () => resolve(true))
.on('error', e => reject(e)));
This stream can't catch the error that thrown by somePromise.
How to catch the a Promise's error in a stream?
Also is there any way to make the stream return the result by the 'somePromise'?
You can catch somePromise like:
.on('data', data => somePromise(data).catch(e => reject(e)))
If you want to return somePromise result:
return new Promise((resolve, reject) => {
let results = [];
someReadStream
.pipe(decoder)
.on('data', data => results.push(somePromise(data)))
.on('end', () => resolve(Promise.all(results).catch(e => reject(e))))
.on('error', e => reject(e)));
}
The stream will not catch the error - so either you accumulate, like Andriy2 suggested (but keep in mind that you may run into memory consumption issues if you run through big ammounts of data) or you use the scramjet framework like this:
const {DataStream} = require("scramjet");
return someReadStream
.pipe(decoder)
.pipe(new DataStream())
.consume((data) => somePromise(data))
);
Scramjet is pretty lightweight so you'll add just 3 dependent modules in total.
DataStream.prototype.consume returns a Promise which makes your code even easier. :)
-- EDIT --
In a recent version a catch method was introduced so you can even handle the error to your liking - even dismiss (by resolving the catch block):
dataStream
.catch(err => {
if (!(err instanceOf DismissableError))
throw err;
logger.error("A dismissable error occured", err);
});
Related
ok i saw this example of reading a stream and returning a promise using new Promise.
function readStream(stream, encoding = "utf8") {
stream.setEncoding(encoding);
return new Promise((resolve, reject) => {
let data = "";
stream.on("data", chunk => data += chunk);
stream.on("end", () => resolve(data));
stream.on("error", error => reject(error));
});
}
const text = await readStream(process.stdin);
My question is why "new Promise" ? can i do it in the 2nd version like
function readStream(stream, encoding = "utf8") {
stream.setEncoding(encoding);
let data = "";
stream.on("data", chunk => data += chunk);
stream.on("end", () => Promise.resolve(data));
stream.on("error", error => Promise.reject(error));
}
const text = await readStream(process.stdin);
Haven't tried it yet, but basically want to avoid the new keyword.
some updates on the 2nd version, since async functions always return a Promise.
A function/method will return a Promise under the following circumstances:
You explicitly created and returned a Promise from it's body.
You returned a Promise that exists outside the method.
You marked it as async.
const readStream = async (stream, encoding = "utf8") => {
stream.setEncoding(encoding);
let data = "";
stream.on("data", chunk => data += chunk);
stream.on("end", () => Promise.resolve(data));
stream.on("error", error => Promise.reject(error));
}
const text = await readStream(process.stdin);
How's this 3rd version ?
If you want readStream to return a promise, you'll have to ... return a promise for readStream (returning a promise in some callback is not doing that).
What the first code is doing, is promisifying the stream API. And that's exactly how it should be done.
The second version of the code is based on a misunderstanding: it seems to hope that returning a promise in the callback passed to the stream.on method, will somehow make readStream return that promise. But when the on callback is called, readStream has already returned. Since readStream has no return statement, it already returned undefined and not a promise.
As a side note, when the stream API calls the callback you passed to the on method, it does not even look at the returned value -- that is ignored.
The third version is an async function, so it now is guaranteed the function will return a promise. But as the function still does not execute a return statement, that promise is immediately resolved with value undefined. Again, the returned values in the callbacks are unrelated to the promise that the async function has already returned.
new keyword
If you want to avoid the new keyword, then realise that anything that can be done with promises can also be done without them. In the end promises are "only" a convenience.
For instance, you could do:
function readStream(stream, success, failure, encoding="utf8") {
let data = "";
stream.setEncoding(encoding);
stream.on("data", chunk => data += chunk);
stream.on("end", () => success(data));
stream.on("error", failure);
}
function processText(text) {
// ... do something with text
}
function errorHandler(error) {
// ... do something with the error
}
readStream(process.stdin, processText, errorHandler);
In typical Node style you would pass one callback, for both purposes, as last argument:
function readStream(stream, encoding="utf8", callback) {
let data = "";
stream.setEncoding(encoding);
stream.on("data", chunk => data += chunk);
stream.on("end", () => callback?.(null, data));
stream.on("error", err => callback?.(err, null));
}
function processText(err, text) {
if (err) {
// do something with err
return;
}
// ... do something with text
}
readStream(process.stdin, "utf8", processText);
And then you could use the util package to turn that into a promise-returning function:
const util = require('util');
const readStream = util.promisify(function (stream, encoding="utf8", callback) {
let data = "";
stream.setEncoding(encoding);
stream.on("data", chunk => data += chunk);
stream.on("end", () => callback?.(null, data));
stream.on("error", err => callback?.(err, null));
});
(async () => {
try {
const text = await readStream(stream, "utf8");
// do something with text
} catch(err) {
// do something with err
}
})();
Of course, under the hood the promisfy function performs new Promise and we're back to where we started.
You need to construct and return a Promise so that the consumer of the function has something to hook into the asynchronous action being performed. (Another option would be to define the function to also take a callback as an argument.)
If you try to do it the way you're doing with the second snippet, readStream will not return anything, so await readStream(process.stdin); will resolve immediately, and it'll resolve to undefined.
Doing
stream.on("end", () => Promise.resolve(data));
and
stream.on("error", error => Promise.reject(error));
constructs new Promises at that point in the code, but you need the consumer of the function to have access to the Promise that resolves (or rejects) - and so you must have return new Promise at the top level of the function.
The code below is an example of what may take place during development.
With the current code, the outer function may throw an error but in this case wont. However, the nested function WILL throw an error (for examples sake). Once it throws the error it cannot be caught as it is asynchronous function.
Bungie.Get('/Platform/Destiny2/Manifest/').then((ResponseText)=>{
//Async function that WILL throw an error
Bungie.Get('/Platform/Destiny2/Mnifest/').then((ResponseText)=>{
console.log('Success')
})
}).catch((error)=>{
//Catch all errors from either the main function or the nested function
doSomethingWithError(error)
});
What I want is for the outer most function to catch all asynchronous function error's but with this code I cannot. I have tried awaiting the nested function but there may be certain circumstances where it will be quicker to not wait for the function. I also tried to include a .catch() with each nested function but this would require a .catch() for each function that would allhandle the error in the same way e.g. doSomethingWithError().
you only needs return the inner function in the outside function.
see example below:
const foo = new Promise((resolve,reject) =>{
setTimeout(() => resolve('foo'), 1000);
});
foo.then((res)=>{
console.log(res)
return new Promise((resolve,reject)=>{
setTimeout(() => reject("bar fail"), 1000);
})
}).catch((e)=>{
// your own logic
console.error(e)
});
this is called promise chaining. see this post for more info https://javascript.info/promise-chaining
if you have multiple promises can do something like:
const foo1 = new Promise((resolve,reject) =>{
setTimeout(() => resolve('foo1'), 1000);
});
const foo2 = new Promise((resolve,reject) =>{
setTimeout(() => resolve('foo2'), 2000);
});
const foo3 = new Promise((resolve,reject) =>{
setTimeout(() => reject('foo3'), 3000);
});
const bar = new Promise((resolve,reject) =>{
setTimeout(() => resolve('bar'), 4000);
});
foo1
.then((res)=>{
console.log(res)
return foo2
})
.then((res)=>{
console.log(res)
return foo3 // throws the error
})
.then((res)=>{
console.log(res)
return bar
})
.catch((e)=>{
// every error will be cached here
console.error(e)
});
I would aim to use async / await unless you have very particular reasons, since it avoids callback hell and makes your code simpler and more bug free.
try {
const response1 = await Bungie.Get('/Platform/Destiny2/Manifest/');
const response2 = await Bungie.Get('/Platform/Destiny2/Mnifest/');
console.log('Success');
} catch (error) {
doSomethingWithError(error);
}
Imagine each Bungie call takes 250 milliseconds. While this is occurring, NodeJS will continue to execute other code via its event loop - eg requests from other clients. Awaiting is not the same as hanging the app.
Similarly, this type of code is used in many browser or mobile apps, and they remain responsive to the end user during I/O. I use the async await programming model in all languages these days (Javascript, Java, C#, Swift etc).
Try this:
let getMultiple = function(callback, ... keys){
let result = [];
let ctr = keys.length;
for(let i=0;i<ctr;i++)
result.push(0);
let ctr2 = 0;
keys.forEach(function(key){
let ctr3=ctr2++;
try{
Bungie.Get(key, function(data){
result[ctr3] = data;
ctr--;
if(ctr==0)
{
callback(result);
}
});
} catch(err) {
result[ctr3]=err.message;
ctr--;
if(ctr==0)
{
callback(result);
}
}
});
};
This should get all your data requests and replace relevant data with error message if it happens.
getMultiple(function(results){
console.log(results);
}, string1, string2, string3);
If the error causes by requesting same thing twice asynchronously, then you can add an asynchronous caching layer before this request.
I'm looking to write some code that chains together some promises. I have a condition that based on the result of one promise I either call the next function that returns a promise and continue chaining another few functions, or I do nothing (effectively end the promise chain).
I have the following three possible solutions, I kinda think they all are a bit messy though.
Here is my first approach, what I dislike here is the nested promises.
initalAsyncCall()
.then((shouldContinue) => {
if (shouldContinue) {
return nextStep()
.then(() => anotherStep())
}
})
.catch((error) => {
handleError(error);
})
Here is my second. This one seems a bit longer and possibly harder to read
const shouldContinuePromise = initialAsyncCall();
const nextStepPromise = shouldContinuePromise.then((shouldContinue) => {
if (shouldContinue) return nextStep();
});
Promise.all([shouldContinuePromise, nextStepPromise])
.spread((shouldContinue) => {
if (shouldContinue) return anotherStep();
})
.catch((error) => {
handleError(error);
});
And finally here is my last approach. What I don't like here is I am throwing an error when it's not really an error.
initalAsyncCall()
.then((shouldContinue) => {
if (!shouldContinue) throw new HaltException()
return nextStep();
})
.then(() => anotherStep())
.catch(HaltException, (ex) => {
// do nothing... maybe some logging
})
.catch((error) => {
handleError(error);
})
Your first approach seems fine, to avoid the nesting you can return the promise and add an extra then block for the nested part like this
initalAsyncCall()
.then((shouldContinue) => {
if (shouldContinue) {
return nextStep()
} else {
throw Error('skip next step')
}
})
.then(() => anotherStep())
.catch((error) => {
handleError(error);
})
If you don't like throwing an unnecessary error in the third approach, You could use async/await to have more control and get rid of the function scope/nesting problem, which is also recommended for the new nodejs versions due to better error stack traces.
try {
const shouldContinue = await initalAsyncCall()
if (shouldContinue) {
await nextStep()
await anotherStep()
// or await Promise.all([nextStep(), anotherStep()]) if they're not dependent
}
}
catch (error) {
handleError(error);
}
When I try to run this, i get "UnhandledPromiseRejectionWarning". But last "catch" received control anyway with correct "err" object.
I have got two warnings:
UnhandledPromiseRejectionWarning
PromiseRejectionHandledWarning
If I comment "return delay(5000);" all works fine.
Why does Node.JS handle "promiseErr" before I do that?
const delay = (ms: number) => new Promise(res => setTimeout(res, ms));
let promiseErr = new Promise( (resolve, reject) => {
reject(new Error("__error__"));
});
let promiseOk = new Promise( (resolve, reject) => {
resolve();
});
promiseOk
.then( () => {
return delay(5000);
})
.then( () => {
return promiseErr;
})
.then( () => {
console.log("OK");
})
.catch( (err) => {
console.log(`ERR ${err}`);
});
program output
You have a rejected promise asynchronously (after 5000ms). Node.js detects unhandled promises by looking at promises that were not handled synchronously or within a microtick (inside a then).
Since Node.js cannot know "for sure" when a promise rejection is unhandled at the moment it errors on the safe side and gives you the warning. It is generally better to add catch handlers to promise errors synchronously as much as possible.
You can suppress the warning by adding a .catch(() => {}) to promiseErr when you create it which will handle the exception:
var promiseErr = ...
promiseErr.catch(() => {}); // add a catch handler without changing the original
In general - the solution is to not write code in such an error prone way. If promiseOk or delay rejects (which Node has no way of knowing) - then the exception in promiseErr itself is indeed unhandeld - so this is not really a safe way to write code.
I see some modules that pipe readable streams in writable streams, and if any error occurr, they use the destroy method:
const readable = fs.createReadStream("file");
const writable = fs.createWriteStream("file2");
readable.pipe(writable);
readable.on("error", (error) => {
readable.destroy();
writable.destroy();
writable.removeListener("close");
callback(error);
});
writable.on("error", (error) => {
readable.destroy();
writable.destroy();
writable.removeListener("close");
callback(error);
});
What is the necessity of destroying the streams and removing the close event on the writable stream? If i don't do that, what could happen?
Thanks.
I believe this is necessary to avoid memory leaks. As per the Node.js documentation on the readable.pipe() method,
One important caveat is that if the Readable stream emits an error during processing, the Writable destination is not closed automatically. If an error occurs, it will be necessary to manually close each stream in order to prevent memory leaks.
In the script below, comment out the line w.destroy(err) and notice none of the Writeable events emit. Not sure why Node.js designers chose to not automatically destroy the Writeable's, maybe they didn't want Stream.pipe() to be too opinionated.
const r = new Readable({
objectMode: true,
read() {
try {
this.push(JSON.parse('{"prop": "I am the data"'))
this.push(null) // make sure we let Writeable's know there's no more to read
} catch (e) {
console.error(`Problem encountered while reading data`, e)
this.destroy(e)
}
}
}).on('error', (err) => {
console.log(`Reader error: ${err}`)
w.destroy(err)
done()
})
const w = new Writable({
objectMode: true,
write(chunk, encoding, callback) {
callback()
}
}).on('error', (err) => {
console.error(`Writer error: ${err}`)
})
.on('close', () => {
console.error(`Writer close`)
})
.on('finish', () => {
console.error(`Writer finish`)
})
r.pipe(w)