fs does not Write all received data file - node.js

My node.js script:
var net = require('net');
var fs = require('fs');
var client = new net.Socket()
client.connect(PORT, HOST, function () {
client.write('You can Send file\0');
client.on('data', function (data) {
// console.log(data);
var destinationFile = fs.createWriteStream("destination.txt");
destinationFile.write(data);
});
});
it is coded for receiving a file from remote HOST.
when i use console.log(data) its OK to log remote data file to console.
but for writing data to file its write one part of received data file.
How can write all data to file?
thanks

The cause of what you get:
client.on('data') is called multiple times, as the file is being sent in data-chunks, not as a single whole data. As a result on receiving each piece of data, you create a new file stream and write to it..
Console.log on the other hand works, because it does not create new console window each time you write to it.
Quick solution would be:
client.connect(PORT, HOST, function () {
var destinationFile = fs.createWriteStream("destination.txt");
client.write('You can Send file\0');
client.on('data', function (data) {
// console.log(data);
destinationFile.write(data);
});
});
Also notice in the net Documentation on net.connect method:
Normally this method is not needed, as net.createConnection opens the
socket. Use this only if you are implementing a custom Socket.

The problem you got is because you declare the filestream inside the event on, which is called each time a packet arrive.
client.connect(PORT, HOST, function () {
var destinationFile = fs.createWriteStream("destination.txt"); //Should be here
client.write('You can Send file\0');
client.on('data', function (data) {
destinationFile.write(data);
});
});
thanks. in tiny test its OK. but written file has 16 bit more than real size. first of file has 16 bit of unknown data. how to ignore that?
In your code, since client.on is called multiple time, multiple write happens in the same times, so it is undefined behavior for which write happens, in which order, or if they ll be complete.
The 16 bytes of unknown data are probably bytes from different packet writen one after the other in random order. Your tiny test work because the file can be send in one packet, so the event is called only once.
If you declare the filestream first, then call the write in client.on, the order of write and data are preserved, and the file is written successfully.

Related

How to correctly calculate the the number of bytes of a node.js stream that have been processed?

I have a stream I'm sending over the wire and takes a bit of time to fully send, so I want to display how far along it is on the fly. I know you can listen on the 'data' event for streams, but in newer versions of node, it also puts the stream into "flowing mode". I want to make sure i'm doing this correctly.
Currently I have the following stuff:
deploymentPackageStream.pause() // to prevent it from entering "flowing mode"
var bytesSent = 0
deploymentPackageStream.on('data', function(data) {
bytesSent+=data.length
process.stdout.write('\r ')
process.stdout.write('\r'+(bytesSent/1000)+'kb sent')
})
deploymentPackageStream.resume()
// copy over the deployment package
execute(conn, 'cat > deploymentPackage.sh', deploymentPackageStream).wait()
This gives me the right bytesSent output, but the resulting package seems to be missing some data off the front. If I put the 'resume' line after executing the copy line (the last line), it doesn't copy anything. If I don't resume, it also doesn't copy anything. What's going on and how do I do this properly without disrupting the stream and without entering flowing mode (I want back pressure)?
I should mention, i'm still using node v0.10.x
Alright, I made something that essentially is a passthrough, but calls a callback with data as it comes in:
// creates a stream that can view all the data in a stream and passes the data through
// parameters:
// stream - the stream to peek at
// callback - called when there's data sent from the passed stream
var StreamPeeker = exports.StreamPeeker = function(stream, callback) {
Readable.call(this)
this.stream = stream
stream.on('readable', function() {
var data = stream.read()
if(data !== null) {
if(!this.push(data)) stream.pause()
callback(data)
}
}.bind(this))
stream.on('end', function() {
this.push(null)
}.bind(this))
}
util.inherits(StreamPeeker, Readable)
StreamPeeker.prototype._read = function() {
this.stream.resume()
}
If I understand streams properly, this should appropriately handle backpressure.
Using this, I can just count up data.length in the callback like this:
var peeker = new StreamPeeker(stream, function(data) {
// use data.length
})
peeker.pipe(destination)

Does the new way to use streams (streams2) in node create blocking?

I realize that node is non-blocking, however, I also realize that because node has only one thread, putting a three second while loop in the middle of your event loop will cause blocking. I.e.:
var start = new Date();
console.log('Test 1');
function sleep(time, words) {
while(new Date().getTime() < start.getTime() + time);
console.log(words);
}
sleep(3000, 'Test 2'); //This will block
console.log('Test 3') //Logs Test 1, Test 2, Test 3
Many of the examples I have seen dealing with the new "Streams2" interface look like they would cause this same blocking. For instance this one, borrowed from here:
var crypto = require('crypto');
var fs = require('fs');
var readStream = fs.createReadStream('myfile.txt');
var hash = crypto.createHash('sha1');
readStream
.on('readable', function () {
var chunk;
while (null !== (chunk = readStream.read())) {
hash.update(chunk); //DOESN'T This Cause Blocking?
}
})
.on('end', function () {
console.log(hash.digest('hex'));
});
If I am following right, the readStream will emit the readable event when there is data in the buffer. So it seems that once the readable event is emitted, the entire event loop would be stopped until the readStream.read() emits null. This seems less desirable than the old way (because it would not block). Can somebody please tell me why I am wrong. Thanks.
You don't have to read until the internal stream buffer is empty. You could just read once if you wanted and then read another chunk some time later.
readStream.read() itself is not blocking, but hash.update(chunk) is (for a brief amount of time) because the hashing is done on the main thread (there is a github issue about adding an async interface that would execute crypto functions in the thread pool though).
Also, you can simplify the code you have to use the crypto stream interface:
var crypto = require('crypto'),
fs = require('fs');
var readStream = fs.createReadStream('myfile.txt'),
hasher = crypto.createHash('sha1');
readStream.pipe(hasher).on('readable', function() {
// the hash stream automatically pushes the digest
// to the readable side once the writable side is ended
console.log(this.read());
}).setEncoding('hex');
All JS code is single-threaded, so a loop will block, but you are misunderstanding how long that loop will run for. Calling .read() takes a readable item from the stream, just like a 'data' handler would be called with the item. It will stop executing and unblock as soon as there are no items. 'readable' is triggered whenever there is data, and then it empties the buffer and waits for another 'readable'. So where your first while loop relies on the time to be updated, which could be some unbounded amount of time, the other loop is basically doing:
while (items.length > 0) items.pop()
which is pretty much the minimum amount of work you need to do to process items from a stream.

HTTP request stream not firing readable when reading fixed sizes

I am trying to work with the new Streams API in Node.js, but having troubles when specifying a fixed read buffer size.
var http = require('http');
var req = http.get('http://143.226.75.100/waug_mp3_128k', function (res) {
res.on('readable', function () {
var receiveBuffer = res.read(1024);
console.log(receiveBuffer.length);
});
});
This code will receive a few buffers and then exit. However, if I add this line after the console.log() line:
res.read(0);
... all is well again. My program continues to stream as predicted.
Why is this happening? How can I fix it?
It's explained here.
As far as I understand it, by reading only 1024 bytes with each readable event, Node is left to assume that you're not interested in the rest of the data that's in the stream buffers, and discards it. Issuing the read(0) (in the same event loop iteration) 'resets' this behaviour. I'm not sure why the process exits after reading a couple of 1024-byte buffers though; I can recreate it, but I don't understand it yet :)
If you don't have a specific reason to use the 1024-byte reads, just read the entire buffer for each event:
var receiveBuffer = res.read();
Or instead of using non-flowing mode, use flowing mode by using the data/end events instead:
var http = require('http');
var req = http.get('http://143.226.75.100/waug_mp3_128k', function (res) {
var chunks = [];
res.on('data', function(chunk) {
chunks.push(chunk);
console.log('chunk:', chunk.length);
});
res.on('end', function() {
var result = Buffer.concat(chunks);
console.log('final result:', result.length);
});
});

Node js- writing data to the writable stream

In my node application im writing data to the file using write method in the createWriteStream method.Now i need to find whether the write for the particular stream is complete or not.How can i find that.
var stream = fs.createWriteStream('myFile.txt', {flags: 'a'});
var result = stream.write(data);
writeToStream();
function writeToStream() {
var result = stream.write(data + '\n');
if (!result) {
stream.once('drain',writeToStream());
}
}
I need to call other method for every time when write completes.How can i do this.
From the node.js WritableStream.write(...) documentation you can give the "write" method a callback that is called when the written data is flushed:
var stream = fs.createWriteStream('myFile.txt', {flags: 'a'});
var data = "Hello, World!\n";
stream.write(data, function() {
// Now the data has been written.
});
Note that you probably don't need to actually wait for each call to "write" to complete before queueing the next call. Even if the "write" method returns false you can still call subsequent writes and node will buffer the pending write requests into memory.
I am using maerics's answer along with error handling. The flag 'a' is used to Open file for appending. The file is created if it does not exist. There Other flags you can use.
// Create a writable stream & Write the data to stream with encoding to be utf8
var writerStream = fs.createWriteStream('MockData/output.txt',{flags: 'a'})
.on('finish', function() {
console.log("Write Finish.");
})
.on('error', function(err){
console.log(err.stack);
});
writerStream.write(outPutData,function() {
// Now the data has been written.
console.log("Write completed.");
});
// Mark the end of file
writerStream.end();

Abort ReadStream on client connection close

I am trying to send a (huge) file with a limited amount of data passing every second (using TooTallNate/node-throttle):
var fs = require('fs');
var Throttle = require('throttle');
var throttle = new Throttle(64);
throttle.on('data', function(data){
console.log('send', data.length);
res.write(data);
});
throttle.on('end', function() {
console.log('error',arguments);
res.end();
});
var stream = fs.createReadStream(filePath).pipe(throttle);
If I cancel the download at the clients browser, the stream will just continue until it completly transferred.
I also tested the scenario above with npm node-throttled-stream, same behavour.
How to cancel the stream if the browser closed his request?
Edit:
I am able to obtain the connections close event by using
req.connection.on('close',function(){});
But the stream has neither a destroy nor an end or stop property which I could use to stop the stream from further reading.
I does provide the property pause Doc, but I would rather stop node from reading the whole file than just stopping to recieve the contents (as described in the doc).
I ended up using the following dirty workaround:
var aborted = false;
stream.on('data', function(chunk){
if(aborted) return res.end();
// stream contents
});
req.connection.on('close',function(){
aborted = true;
res.end();
});
As mentioned above, this isn't really a nice solution, but it works.
Any other solution would be highly appreciated!

Resources