Nodejs streams writablestream drain event not firing - node.js

Question

I am trying to read in a large file, do some computation and then write to a much bigger file. To prevent excessive memory consumption, I am using streams. The problem that I am facing is that the writestream is not firing the "drain" event, which signals that the writes have been flushed to disk. In order to prevent "back-pressure", I am waiting for the drain event to be fired before I start writing to the buffer again. While debugging I found that after a .write() call returns false and the line fvfileStream.once('drain', test) is executed, the program just stops and does not do anything.
Here is the code:
var fs = require('fs');
//a test function I created to see if the callback is called after drain.
var test = function(){
console.log("Done Draining");
}
fs.readFile('/another/file/to/be/read', {
encoding: "utf8"
}, function(err, data) {
if (err) throw err;
//Make an array containing tags.
var tags = data.split('\n');
//create a write stream.
var fvfileStream = fs.createWriteStream('/path/TagFeatureVectors.csv');
//read in the question posts
var qfileStream = fs.createReadStream('/Big/file/QuestionsWithTags.csv', {
encoding: "utf8"
});
var partialRow = null;
var writable = true;
var count = 0;
var doRead = function() {
var qData = qfileStream.read();
var questions = qData.split('\n');
if (partialRow != null) {
questions[0] = partialRow + questions[0];
partialRow = null;
}
var lastRow = questions[questions.length - 1];
if (lastRow.charAt(lastRow.length - 1) != '\n') {
partialRow = lastRow;
}
questions.forEach(function(row, index, array) {
count++;
var fields = row.split(',');
console.log("Processing question number: " + count + " id: " + fields[0]);
var tagString = fields[1];
var regex = new RegExp(/<([^>]+)>/g);
tags.forEach(function(tag, index, array) {
var found = false;
var questionTags;
while ((questionTags = regex.exec(tagString)) != null) {
var currentTag = questionTags[1]
if (currentTag === tag) {
found = true;
break;
}
};
//This is where the writestream is written to
if (found) {
writable = fvfileStream.write("1,", "utf8");
}else {
writable = fvfileStream.write("0,","utf8");
}
});
});
fvfileStream.write("\n");
}
qfileStream.on('readable', function() {
if (writable) {
doRead();
} else {
//Waiting for drain event.
fvfileStream.once('drain', test);
}
});
qfileStream.on('end', function() {
fvfileStream.end();
});
});
Updated
Based on advise provided by #loganfsmyth, I implemented transform streams, but still ran into the same issue. Here is my updated code:
var fs = require('fs');
var stream = require('stream');
var util = require('util');
var Transform = stream.Transform;
function FVCreator(options) {
// allow use without new
if (!(this instanceof FVCreator)) {
return new FVCreator(options);
}
// init Transform
Transform.call(this, options);
}
util.inherits(FVCreator, Transform);
var partialRow = null;
var count = 0;
var tags;
FVCreator.prototype._transform = function(chunk, enc, cb) {
var that = this;
var questions = chunk.toString().split('\n');
if (partialRow != null) {
questions[0] = partialRow + questions[0];
partialRow = null;
}
var lastRow = questions[questions.length - 1];
if (lastRow.charAt(lastRow.length - 1) != '\n') {
partialRow = lastRow;
questions.splice(questions.length - 1, 1);
}
questions.forEach(function(row, index, array) {
count++;
var fields = row.split(',');
console.log("Processing question number: " + count + " id: " + fields[0]);
var tagString = fields[1];
var regex = new RegExp(/<([^>]+)>/g);
tags.forEach(function(tag, index, array) {
var found = false;
var questionTags;
while ((questionTags = regex.exec(tagString)) != null) {
var currentTag = questionTags[1]
if (currentTag === tag) {
found = true;
break;
}
};
if (found) {
that.push("1,", "utf8");
} else {
that.push("0,", "utf8");
}
});
});
this.push("\n", "utf8");
cb();
};
fs.readFile('/another/file/to/be/read', {
encoding: "utf8"
}, function(err, data) {
if (err) throw err;
//Make an array containing tags.
tags = data.split('\n');
//write to a file.
var fvfileStream = fs.createWriteStream('/path/TagFeatureVectors.csv');
//read in the question posts
var qfileStream = fs.createReadStream('/large/file/to/be/read', {
encoding: "utf8"
});
var fvc = new FVCreator();
qfileStream.pipe(fvc).pipe(fvfileStream);
});
I am running this on OSX Yosemite.