I'm quite new to javascripts. I'm using node js writable stream to write a .txt file; It works well, but I cannot understand how to properly close the file, as its content is blank as long as the program is running. More in detail I need to read from that .txt file after it has been written, but doing it this way returns an empty buffer.
let myWriteStream = fs.createWriteStream("./filepath.txt");
myWriteStream.write(stringBuffer + "\n");
myWriteStream.on('close', () => {
console.log('close event emitted');
});
myWriteStream.end();
// do things..
let data = fs.readFileSync("./filepath.txt").toString().split("\n");
Seems like the event emitted by the .end() method is triggered after the file reading, causing it to be read as empty. If I put a while() to wait for the event to be triggered, so that I know for sure the stream is closed before the reading, the program waits forever.
Do you have any clue of what I'm doing wrong?
your missing 2 things one test that write is succeed
then you need to wait for stream finish event
const { readFileSync, createWriteStream } = require('fs')
const stringBuffer = Buffer.from(readFileSync('index.js')
)
const filePath = "./filepath.txt"
const myWriteStream = createWriteStream(filePath)
let backPressureTest = false;
while (!backPressureTest) {
backPressureTest = myWriteStream.write(stringBuffer + "\n");
}
myWriteStream.on('close', () => {
console.log('close event emitted');
});
myWriteStream.on('finish', () => {
console.log('finish event emitted');
let data = readFileSync(filePath).toString().split("\n");
console.log(data);
});
myWriteStream.end();
Related
I have a need to pipe a readable stream into both a buffer (to be converted into a string) and a file. The stream is coming from node-fetch.
NodeJS streams have two states: paused and flowing. From what I understand, as soon as a 'data' listener is attached, the stream will change to flowing mode. I want to make sure the way I am reading a stream will not lose any bytes.
Method 1: piping and reading from 'data':
fetch(url).then(
response =>
new Promise(resolve => {
const buffers = []
const dest = fs.createWriteStream(filename)
response.body.pipe(dest)
response.body.on('data', chunk => buffers.push(chunk))
dest.on('close', () => resolve(Buffer.concat(buffers).toString())
})
)
Method 2: using passthrough streams:
const { PassThrough } = require('stream')
fetch(url).then(
response =>
new Promise(resolve => {
const buffers = []
const dest = fs.createWriteStream(filename)
const forFile = new PassThrough()
const forBuffer = new PassThrough()
response.body.pipe(forFile).pipe(dest)
response.body.pipe(forBuffer)
forBuffer.on('data', chunk => buffers.push(chunk))
dest.on('close', () => resolve(Buffer.concat(buffers).toString())
})
)
Is the second method required so there is no lost data? Is the second method wasteful since two more streams could be buffered? Or, is there another way to fill a buffer and write stream simultaneously?
You won't miss any data, since .pipe internally calls src.on('data') and writes any chunk to the target stream.
So any chunk written to your dest stream, will also be emitted to response.body.on('data') where you're buffering the chunks.
In any case, you should listen to 'error' events and reject if any error occurs.
And While your second mode will work, you don't need it.
This is a chunk of code from the .pipe function
src.on('data', ondata);
function ondata(chunk) {
debug('ondata');
var ret = dest.write(chunk);
debug('dest.write', ret);
if (ret === false) {
// If the user unpiped during `dest.write()`, it is possible
// to get stuck in a permanently paused state if that write
// also returned false.
// => Check whether `dest` is still a piping destination.
if (((state.pipesCount === 1 && state.pipes === dest) ||
(state.pipesCount > 1 && state.pipes.indexOf(dest) !== -1)) &&
!cleanedUp) {
debug('false write response, pause', state.awaitDrain);
state.awaitDrain++;
}
src.pause();
}
}
So, I've created a read stream that first connects to an SFTP and starts reading from a file. At any point, my code can unpipe that readstream and do something else. For example, I might use this to get the first few rows of a CSV and stop reading.
The problem is, I don't know how to listen for the unpipe event in my readStream constructor so that I can correctly close the SFTP connection. I use a flush method in write streams, is there something like that for read streams?
Here's a simplified portion of my readStream constructor:
const Client = require('ssh2').Client,
nom = require('noms');
function getStream (get) {
const self = this;
const conn = new Client();
let client,
fileData,
buffer,
totalBytes = 0,
bytesRead = 0;
let read = function(size,next) {
const read = this;
// Read each chunk of the file
client.read(fileData, buffer, bytesRead, size, bytesRead,
function (err, byteCount, buff, pos) {
bytesRead += byteCount;
read.push(buff);
next();
}
);
};
let before = function(start) {
// setup the connection BEFORE we start _read
conn.on('ready', function(){
conn.sftp(function(err,sftp) {
sftp.open(get, 'r', function(err, fd){
sftp.fstat(fd, function(err, stats) {
client = sftp;
fileData = fd;
totalBytes = stats.size;
buffer = new Buffer(totalBytes);
start();
});
});
});
}).connect(credentials);
};
return nom(read,before);
}
Later I might call myStream.pipe(writeStream) and then myStream.unpipe(). But because I have no way of listening for that unpipeevent, the reading stops, but the SFTP connection stays open and eventually times out.
Any ideas?
So, after doing more research, I learned that ReadStreams are not passed the unpipe event when you call readStream.unpipe(writeStream). That event is passed to just the writeStream. In order to listen for the unpipe, you need to explicitly emit an event on the readStream, like so:
readStream.emit('unpipe');
You can listen for this event anywhere, inside or outside your stream constructor, which is really convenient. So, that would make the code above look like this:
function getStream (get) {
/**
* ... stuff
* ... read()
* ... before()
* ... etc
*/
let readStream = nom(read,before);
readStream.on('unpipe', function(){
console.log('called unpipe on read stream');
});
return readStream;
}
Moral of the story, streams already have the Event Emitter class methods, so you can emit and listen for custom events out of the box.
I'm just learning node.js and wanted to write a simple test program that copied a file from a source folder to a destination folder. I piped a fs.ReadStream to a fs.WriteStream and that worked perfectly. I next tried to use non-flowing mode but the following program fails 99% of the time on larger files (anything over 1MB.) I'm assuming that given certain timing the event queue becomes empty and so exits. Should the following program work?
var sourcePath = "./source/test.txt";
var destinationPath = "./destination/test.txt";
// Display number of times 'readable' callback fired
var callbackCount = 0;
process.on('exit', function() {
console.log('Readable callback fired %d times', callbackCount);
})
var fs = require('fs');
var sourceStream = fs.createReadStream(sourcePath);
var destinationStream = fs.createWriteStream(destinationPath);
copyStream(sourceStream, destinationStream);
function copyStream(src, dst) {
var drained = true;
// read chunk of data when ready
src.on('readable', function () {
++callbackCount;
if (!drained) {
dst.once('drain', function () {
writeToDestination();
});
} else {
writeToDestination();
}
function writeToDestination() {
var chunk = src.read();
if (chunk !== null) {
drained = dst.write(chunk);
}
}
});
src.on('end', function () {
dst.end();
});
}
NOTE: If I remove the drain related code the program always works but the node.js documentation indicates that you should wait on a drain event if the write function returns false.
So should the above program work as is? If it shouldn't how should I reorganize it to work with both readable and drain events?
It looks like you're most of the way there; there are just a couple of things you need to change.
When writing to dst you need to keep reading from src until either you get a null chunk, or dst.write() returns false.
Instead of listening for all readable events on src, you should only be listening for those events when it's ok to write to dst and you currently have nothing to write.
Something like this:
function copyStream(src, dst) {
function writeToDestination() {
var chunk = src.read(),
drained = true;
// write until dst is saturated or there's no more data available in src
while (drained && (chunk !== null)) {
if (drained = dst.write(chunk)) {
chunk = src.read();
}
}
if (!drained) {
// if dst is saturated, wait for it to drain and then write again
dst.once('drain', function() {
writeToDestination();
});
} else {
// if we ran out of data in src, wait for more and then write again
src.once('readable', function() {
++callbackCount;
writeToDestination();
});
}
}
// trigger the initial write when data is available in src
src.once('readable', function() {
++callbackCount;
writeToDestination();
});
src.on('end', function () {
dst.end();
});
}
Consider the code below ... I am trying to pause the stream after reading the first 5 lines:
var fs = require('fs');
var readline = require('readline');
var stream = require('stream');
var numlines = 0;
var instream = fs.createReadStream("myfile.json");
var outstream = new stream;
var readStream = readline.createInterface(instream, outstream);
readStream.on('line', function(line){
numlines++;
console.log("Read " + numlines + " lines");
if (numlines >= 5) {
console.log("Pausing stream");
readStream.pause();
}
});
The output (copied next) suggests that it keeps reading lines after the pause. Perhaps readline has queued up a few more lines in the buffer, and is feeding them to me anyway ... this would make sense if it continues to read asynchronously in the background, but based on the documentation, I don't know what the proper behavior should be. Any recommendations on how to achieve the desired effect?
Read 1 lines
Read 2 lines
Read 3 lines
Read 4 lines
Read 5 lines
Pausing stream
Read 6 lines
Pausing stream
Read 7 lines
Somewhat unintuitively, the pause methods does not stop queued up line events:
Calling rl.pause() does not immediately pause other events (including 'line') from being emitted by the readline.Interface instance.
There is however a 3rd-party module named line-by-line where pause does pause the line events until it is resumed.
var LineByLineReader = require('line-by-line'),
lr = new LineByLineReader('big_file.txt');
lr.on('error', function (err) {
// 'err' contains error object
});
lr.on('line', function (line) {
// pause emitting of lines...
lr.pause();
// ...do your asynchronous line processing..
setTimeout(function () {
// ...and continue emitting lines.
lr.resume();
}, 100);
});
lr.on('end', function () {
// All lines are read, file is closed now.
});
(I have no affiliation with the module, just found it useful for dealing with this issue.)
So, it turns out that the readline stream tends to "drip" (i.e., leak a few extra lines) even after a pause(). The documentation does not make this clear, but it's true.
If you want the pause() toggle to appear immediate, you'll have to create your own line buffer and accumulate the leftover lines yourself.
add some points:
.on('pause', function() {
console.log(numlines)
})
You will get the 5. It mentioned in the node.js document :
The input stream is not paused and receives the SIGCONT event. (See events SIGTSTP and SIGCONT)
So, I created a tmp buffer in the line event. Use a flag to determine whether it is triggered paused.
.on('line', function(line) {
if (paused) {
putLineInBulkTmp(line);
} else {
putLineInBulk(line);
}
}
then in the on pause, and resume:
.on('pause', function() {
paused = true;
doSomething(bulk, function(resp) {
// clean up bulk for the next.
bulk = [];
// clone tmp buffer.
bulk = clone(bulktmp);
bulktmp = [];
lr.resume();
});
})
.on('resume', () => {
paused = false;
})
Use this way to handle this kind of situation.
In my node application im writing data to the file using write method in the createWriteStream method.Now i need to find whether the write for the particular stream is complete or not.How can i find that.
var stream = fs.createWriteStream('myFile.txt', {flags: 'a'});
var result = stream.write(data);
writeToStream();
function writeToStream() {
var result = stream.write(data + '\n');
if (!result) {
stream.once('drain',writeToStream());
}
}
I need to call other method for every time when write completes.How can i do this.
From the node.js WritableStream.write(...) documentation you can give the "write" method a callback that is called when the written data is flushed:
var stream = fs.createWriteStream('myFile.txt', {flags: 'a'});
var data = "Hello, World!\n";
stream.write(data, function() {
// Now the data has been written.
});
Note that you probably don't need to actually wait for each call to "write" to complete before queueing the next call. Even if the "write" method returns false you can still call subsequent writes and node will buffer the pending write requests into memory.
I am using maerics's answer along with error handling. The flag 'a' is used to Open file for appending. The file is created if it does not exist. There Other flags you can use.
// Create a writable stream & Write the data to stream with encoding to be utf8
var writerStream = fs.createWriteStream('MockData/output.txt',{flags: 'a'})
.on('finish', function() {
console.log("Write Finish.");
})
.on('error', function(err){
console.log(err.stack);
});
writerStream.write(outPutData,function() {
// Now the data has been written.
console.log("Write completed.");
});
// Mark the end of file
writerStream.end();