Parsing piped response sometimes gets chunked data - node.js

Question

I have a nodejs proxy for calling a service. On the response, the request is piped to the service url (I guess that's the right way to do it if you want to parse the response before returning it). The problem is that the parser sometimes fails on JSON.parse(data) because it Unexpected end of input. From what I saw while debugging the issue is that the data being parsed is not complete (even though the service returns it properly).
I don't have too much experience with pipe and stream so I'm not sure why this is failing sometimes.
//Request setup
r.on('response', function(resp) {
if (resp.statusCode === 200) {
r.pipe(responseParser(config.get('service:url'))).pipe(res);
} else {
r.pipe(res);
}
});
//Parser module
var _ = require('lodash'),
stream = require('stream');
module.exports = function responseParser(url) {
var data = '',
parser = new stream.Transform({
objectMode: true
});
parser._transform = function (chunk, encoding, done) {
data += chunk.toString();
done();
};
parser._flush = function (done) {
if (data) {
var obj = mapValues(JSON.parse(data));
this.push(JSON.stringify(obj));
}
done();
};
function mapValues(data){
...
}
return parser;
}
I still don't know why sometimes the flush gets called before all the chunks of data are returned but what I did in order to avoid that is just to parse the chunks as they arrived, by making sure that in a chunk I don't get partial data on the values I needed to map. If a chunk contains only partial information for the targeted value, I remove it, and add it at the beginning of the next chunk. This way the data is parsed as it comes in so I don't have to rely on the fact that flush is called only when all the data has returned.

Answer 1

I would disable objectMode as it's not necessary in this case. Also, you'll want to wrap the JSON parsing in a try-catch in case of malformed input:
module.exports = function responseParser(url) {
var data = '';
var parser = new stream.Transform();
parser._transform = function(chunk, encoding, done) {
data += chunk;
done();
};
parser._flush = function(done) {
var err;
if (data) {
try {
var obj = mapValues(JSON.parse(data));
this.push(JSON.stringify(obj));
this.push(null);
} catch (ex) {
err = ex;
}
}
done(err);
};
function mapValues(data){
// ...
}
return parser;
};
You may also want to check that resp.headers['content-type'] contains application/json first before trying to parse it as such and you may want to make a custom Transform subclass and instantiate that instead of creating new _transform() and _flush() functions every time.

Answer 2

Rather than writing this yourself, why don't you use a streaming JSON parser that knows how to parse a stream? JSONStream for example.
The other option to make your life easier would be to use stream-to-promise to just convert the read stream into a Promise that will resolve to a Buffer of the JSON, which you can then parse.
Also, why is your proxy parsing the JSON?