Chunking stream into data chunks using Node.js - node.js

I am trying to chunk a file into data chunks. I found that link does the job beautifully but when I use the above library in the following manner:
var in = fs.createReadStream(__dirname+'/try.html'),
chunker = new SizeChunker({
chunkSize: 2048
}),
output;
chunker.on('chunkStart', function(id, done) {
output = fs.createWriteStream('./output-' + id);
console.log("Chunkstart!");
console.log("Input: "+in.length);
done();
});
chunker.on('chunkEnd', function(id, done) {
output.end();
console.log("Chunkend!");
done();
});
chunker.on('data', function(dat) {
console.log("Writing chunk to output!")
output.write(dat.chunk);
console.log(dat.chunk);
});
input.pipe(chunker);
But I am getting this error :
_stream_writable.js:201
var len = state.objectMode ? 1 : chunk.length;
^
TypeError: Cannot read property 'length' of undefined
at writeOrBuffer (_stream_writable.js:201:41)
at WriteStream.Writable.write (_stream_writable.js:180:11)
at SizeChunker.<anonymous> (/Users/admin/Documents/chunk.js:16:15)
at SizeChunker.EventEmitter.emit (events.js:95:17)
at SizeChunker.<anonymous> (_stream_readable.js:746:14)
at SizeChunker.EventEmitter.emit (events.js:92:17)
at emitReadable_ (_stream_readable.js:408:10)
at emitReadable (_stream_readable.js:404:5)
at readableAddChunk (_stream_readable.js:165:9)
at SizeChunker.Readable.push (_stream_readable.js:127:10)
Also, in.length is undefined when displayed using console.log(). Can anyone please help me resolve this issue? Thanks in advance.

When you listen for data on the chunker stream, the dat argument has no property chunk. You can read on the chunking-stream readme the following:
Each data chunk is an object with the following fields:
id: number of chunk (starts from 1) data: Buffer with data
You can do something like this instead:
chunker.on('data', function(dat) {
console.log("Writing chunk to output!")
output.write(dat.data);
console.log(dat);
});
Also, in is a stream and has no length property defined.

Related

Error: stream.push() after EOF

Playing with node steams
This code reads from index.js and writes to indexCopy.js - kind of file copy.
Target file got created, but during execution exception is thrown:
node index.js
events.js:183
throw er; // Unhandled 'error' event
^
Error: stream.push() after EOF
at readableAddChunk (_stream_readable.js:240:30)
at MyStream.Readable.push (_stream_readable.js:208:10)
at ReadStream.f.on (C:\Node\index.js:16:28)
at emitOne (events.js:116:13)
at ReadStream.emit (events.js:211:7)
at addChunk (_stream_readable.js:263:12)
at readableAddChunk (_stream_readable.js:250:11)
at ReadStream.Readable.push (_stream_readable.js:208:10)
at fs.read (fs.js:2042:12)
at FSReqWrap.wrapper [as oncomplete] (fs.js:658:17)
C:\Node>
This is code:
var util = require('util');
var stream = require('stream');
var fs = require('fs');
var MyStream = function(){
stream.Readable.call(this)
}
util.inherits(MyStream,stream.Readable);
MyStream.prototype._read = function(d){
f = fs.createReadStream("index.js");
f.on('data',(d)=>{this.push(d)});
f.on('end',()=>{this.push(null)}); //when file finished need to close stream
}
var f = fs.createWriteStream("indexCopy.js")
var myStream = new MyStream()
myStream.pipe(f);
I tried to call this.push(null) in 'data' event, in that case even target file is not created and code fails with the exception.
I realize that copy file should be done easier with pipe() function - I am just experimenting/learning.
What is wrong with my approach?
You don't want the f = fs.createReadStream("index.js") line inside the _read method -- _read gets called repeatedly so you're creating multiple read streams. Put that in your constructor instead.
function MyStream () {
stream.Readable.call(this);
this.source = fs.createReadStream("index.js");
this.haveBound = false;
}
MyStream.prototype._read = function () {
if (this.haveBound) return; // Don't bind to events repeatedly
this.haveBound = true;
this.source.on("data", d => this.push(d));
this.source.on("end", () => this.push(null));
};
This is awkward though. Streams are meant to be pipe'ed.

UDP multicast failing - NodeJS / Windows 10

I am beating my brains out trying to get this to work. I read all the other answers related to NodeJS UDP on SO already, but to no avail. I am on Windows 10.
Here is the error I am getting:
Uncaught Exception: Error: write ENOTSUP
at exports._errnoException (util.js:1022:11)
at ChildProcess.target._send (internal/child_process.js:654:20)
at ChildProcess.target.send (internal/child_process.js:538:19)
at sendHelper (cluster.js:751:15)
at send (cluster.js:534:12)
at cluster.js:509:7
at SharedHandle.add (cluster.js:99:3)
at queryServer (cluster.js:501:12)
at Worker.onmessage (cluster.js:450:7)
at ChildProcess.<anonymous> (cluster.js:765:8)
at emitTwo (events.js:111:20)
at ChildProcess.emit (events.js:191:7)
at process.nextTick (internal/child_process.js:744:12)
at _combinedTickCallback (internal/process/next_tick.js:67:7)
at process._tickDomainCallback [as _tickCallback] (internal/process/next_tick.js:122:9)
Here is my code:
let dgram = require('dgram'),
server = dgram.createSocket('udp4'),
multicastAddress = '239.255.255.250',
multicastPort = 1900,
myIp = '192.168.51.133';
server.bind(multicastPort, myIp, function () {
server.setBroadcast(true);
server.setMulticastTTL(128);
server.setInterface.getbyname(myIp);
server.addMembership(multicastAddress, myIp);
});
//wait for incoming messages and print ip address
server.on('message', function (data, rinfo) {
console.log(new Date() + ' RECEIVER received from ', rinfo.address, ':');
console.log(data.toString());
});
//Set up discovery message. Make sure to leave out any extra space in the message.
var discover_message = new Buffer('M-SEARCH * HTTP/1.1\r\nHost: 239.255.255.250:1900\r\nMan: ssdp:discover\r\nST: colortouch:ecp\r\n');
server.send(discover_message, 0, discover_message.length, 1900, multicastAddress);
Finally found an answer for this. The issue is due to being on Windows and using clusters in Node. The problem is on the server.bind call. Here is the correct, working code:
server.bind({port: 1900, exclusive: true}, function () {
console.log('PORT BIND SUCCESS');
server.setBroadcast(true);
server.setMulticastTTL(128);
server.addMembership(multicastAddress, myIp);
});
The fix was to pass in the object {port: 1900, exclusive: true}. Source: https://github.com/misterdjules/node/commit/1a87a95d3d7ccc67fd74145c6f6714186e56f571

Viewing piped data inside pipe error

In the following code block how can I log the piped data causing JSONStream.parse to fail?
data.pipe(JSONStream.parse('*'))
.on('error', () => {
// here I'd like to see the data causing JOSONStream.parse to blow up
reject("Error parsing the json!");
})
.pipe(objectStream);
I ask because currently when it blows up I get this sort of error message but no context to what the Invalid UTF-8 character is:
Error: Invalid JSON (Invalid UTF-8 character at position 2397 in state STRING1)
at Parser.proto.write (/var/www/data-site/pop-service/node_modules/JSONStream/node_modules/jsonparse/jsonparse.js:120:31)
at Stream.<anonymous> (/var/www/data-site/pop-service/node_modules/JSONStream/index.js:23:12)
at Stream.stream.write (/var/www/data-site/pop-service/node_modules/JSONStream/node_modules/through/index.js:26:11)
at IncomingMessage.ondata (_stream_readable.js:536:20)
at emitOne (events.js:82:20)
at IncomingMessage.emit (events.js:169:7)
at readableAddChunk (_stream_readable.js:153:18)
at IncomingMessage.Readable.push (_stream_readable.js:111:10)
at HTTPParser.parserOnBody (_http_common.js:124:22)
at TLSSocket.socketOnData (_http_client.js:320:20)
something like .on('error', (data) => {... doesn't seem to work
Solution using answer from #drinchev
//save the last chunk so that we can log it in case of parsing error
let lastRetrievedChunk = '';
data.on('data', (dd: any) => {
lastRetrievedChunk = dd.toString();
});
let jsonParser = JSONStream.parse('*');
jsonParser.on('error', (err: any) => {
reject(err.stack + ' lastchunk = ' + lastRetrievedChunk);
});
data.pipe(jsonParser).pipe(objectStream);
It does not work indeed, since it is event-driven process.
You can try to store the data in a variable and pass it to the error
var someString = '*';
data.pipe(JSONStream.parse(someString))
.on('error', () => {
console.log( someString );
reject("Error parsing the json!");
})
.pipe(objectStream);
At any point you can also use some library like through2 which can help you to make a stream that logs the piped data.

nodejs throw er; // Unhandled 'error' event

I made following, to play a bit around with node.js.
The files in the folder zipfiles are zipped accordingly and everything seems to work.
But I got an error on the cmd and I don't know where it comes from or how to solve it.
events.js:72
throw er; // Unhandled 'error' event
^
Error: write after end
at writeAfterEnd (_stream_writable.js:130:12)
at Gzip.Writable.write (_stream_writable.js:178:5)
at write (_stream_readable.js:583:24)
at flow (_stream_readable.js:592:7)
at ReadStream.pipeOnReadable (_stream_readable.js:624:5)
at ReadStream.EventEmitter.emit (events.js:92:17)
at emitReadable_ (_stream_readable.js:408:10)
at emitReadable (_stream_readable.js:404:5)
at readableAddChunk (_stream_readable.js:165:9)
at ReadStream.Readable.push (_stream_readable.js:127:10)
Here's my script:
var zlib = require('zlib');
var gzip = zlib.createGzip();
var fs = require('fs');
var zip = {
zipAll: function(dir){
//files to zip
fs.readdir(dir, function(err, data){
if(err) throw(err);
var arrayValue = data.toString().split(',');
//files with .gz at the end, needs to be excluded
for(var i=0; i<arrayValue.length; i+=1){
console.log("Zipping following files: " + arrayValue[i]);
var input = fs.createReadStream('zipfiles/' + arrayValue[i]);
var output = fs.createWriteStream('zipfiles/input'+[i]+'.txt'+'.gz');
input.pipe(gzip).pipe(output);
}
});
}
};
zip.zipAll('zipfiles');
Thanks
The Gzip object is a bit wonky (afaik undocumented) to reuse for multiple files. The easiest way to fix your problem is to simply use a separate gzip object per file to compress, something like;
for(var i=0; i<arrayValue.length; i+=1){
console.log("Zipping following files: " + arrayValue[i]);
var input = fs.createReadStream('zipfiles/' + arrayValue[i]);
var output = fs.createWriteStream('zipfiles/input'+[i]+'.txt'+'.gz');
input.pipe(zlib.createGzip()).pipe(output);
}

ForEachLine() in node.js

Referring to slide no 35 in ppt on slideshare
When I run this code
var server = my_http.createServer();
server.on("request", function(request,response){
var chunks = [];
output = fs.createWriteStream("./output");
request.on("data",function(chunk){
chunks = forEachLine(chunks.concat(chunk),function(line){
output.write(parseInt(line,10)*2);
output.write("\n");
})
});
request.on("end",function(){
response.writeHeader(200,{"Content-Type":"plain/text"})
response.end("OK\n");
output.end()
server.close()
})
});
server.listen("8080");
I get error as
chunks = forEachLine(chunks.concat(chunk),function(line){
^
ReferenceError: forEachLine is not defined
Of course I unserstand that I need to include some library but when I googled this I found nothing . Since I am complete newbie to this I have absolutely no idea how to resolve it.
Any suggestions will be appreciable.
EDIT
Using the suggested answer I am getting error as
events.js:72
throw er; // Unhandled 'error' event
^
TypeError: Invalid non-string/buffer chunk
at validChunk (_stream_writable.js:150:14)
at WriteStream.Writable.write (_stream_writable.js:179:12)
at /var/www/html/experimentation/nodejs/first.js:18:20
at Array.forEach (native)
at forEachLine (/var/www/html/experimentation/nodejs/first.js:8:60)
at IncomingMessage.<anonymous> (/var/www/html/experimentation/nodejs/first.js:17:18)
at IncomingMessage.EventEmitter.emit (events.js:95:17)
at IncomingMessage.<anonymous> (_stream_readable.js:736:14)
at IncomingMessage.EventEmitter.emit (events.js:92:17)
at emitReadable_ (_stream_readable.js:408:10)
Thanks
See proxy_stream.js
function forEachLine(chunks, callback) {
var buffer = chunks.join("")
buffer.substr(0, buffer.lastIndexOf("\n")).split("\n").forEach(callback)
return buffer.substr(buffer.lastIndexOf("\n") + 1).split("\n")
}
The link to the repo was on the first slide.
EDIT BY LET's CODE FOR ERROR MESSAGE
Came to know the actual issue now .
I was using nod v0.10 and it is buggy in getting the streams so I was getting the error. Downgraded to v0.8 and same code is working perfect .

Resources