pipe streams to edit csv file in node.js - node.js

Question

I am basically trying to :
open a csv file as a stream
make some operation on each line
stream the result into a second csv file
in node.js.
Here is my code :
var fs = require("fs");
var csv = require("csv");
var readStream = fs.createReadStream("input.csv");
var writeStream = fs.createWriteStream("output.csv");
var csvStream = csv
.parse()
.on("data", function(data){
//do some stuff with data
return(JSON.stringify(data));
})
.on("end", function(){
console.log("done");
})
.on("error", function(error){
console.log(error)
});
(readStream.pipe(csvStream)).pipe(writeStream);
I am getting "TypeError: Invalid non-string/buffer chunk". What am I doing wrong ? I am totally new to node.js, so please detail your answer.

Answer 1

You are reading correctly the data. However using return is not the correct way to transform your data. CSV Stream cannot at the same time output untransformed data (that you are reading in your data event handler) and transformed data that you would pipe to writeStream.
To use pipe with the writeStream, you would have needed a readableStream outputing your transformed data. That would have meant creating a read/write stream around your transform function, and piping fileReader > csvReader > transformStream > writeStream.
It is way simpler to attach a function to the data event of the csv reader like you did, but you need to manually write to the file.
Correct code code may be more clear this way :
var fs = require("fs");
var csv = require("csv");
var readStream = fs.createReadStream("input.csv"); // readStream is a read-only stream wit raw text content of the CSV file
var writeStream = fs.createWriteStream("output.csv"); // writeStream is a write-only stream to write on the disk
var csvStream = csv.parse(); // csv Stream is a read and write stream : it reads raw text in CSV and output untransformed records
csvStream.on("data", function(data) {
//console.log(data)
writeStream.write(JSON.stringify(data));
})
.on("end", function(){
console.log("done");
})
.on("error", function(error){
console.log(error)
});
readStream.pipe(csvStream)

Answer 2

I have basically done these:
parsed (read csv strings and writes objects and arrays)
stringified (reads objects and arrays and writes csv strings)
piped all that to writeStream to my file
function readingAppendingAndWritingToCSVFile(readStream, writeStream) {
const results = [];
const p = new Promise((resolve, reject) => {
readStream.pipe(csv.parse({ columns: true }))
.on('data', (data) => {
console.log('data --------------', data);
data.name = 'somename'; // will add new column with same data in all rows
console.log('data after pushing ----------', data);
results.push(data);
})
.on('error', (err) => {
console.log('error ------------', err);
reject();
})
.on('finish', () => {
console.log();
console.log('all the csv strings parsed to objects -------------', results);
})
.pipe(csv.stringify({ header: true }))
.pipe(writeStream);
});
return p;