I have a large xml which is a combination of xml documents. I'm trying to use a nodejs xml splitter and respond back with the number of documents i have found.
My code looks something like this. I'm looking to get the number of documents outside the function(in the last line). Is there something I can do to achieve this?
var XmlSplit = require('./xmlsplitter.js')
const fs = require('fs')
var xmlsplit = new XmlSplit()
var no_of_docs = 0
var inputStream = fs.createReadStream('./files/input/test.xml')
inputStream.pipe(xmlsplit).on('data', function(data,callback) {
var xmlDocument = data.toString();
no_of_docs = no_of_docs + 1;
})
inputStream.pipe(xmlsplit).on('end', function(){
console.log('Stream ended');
console.log(no_of_docs); <-- This prints the correct value but the value is lost as soon as we exit this.
});
console.log("This is outside the function " + no_of_docs); <-- I need the value here.
When I run this code I want to be printed as "Mohammed said: hi guys"
but it comes to an error
which is #person is not a function.
Why it is?
var events = require('events');
var util = require('util');
var person = function(name){
this.name = name;
};
util.inherits(person, events.EventEmitter);
var mohammed = new person('mohammed');
var tahir = new person('tahir');
var taha = new person('taha');
var people = ['mohammed', 'tahir', 'taha'];
people.forEach(function(person){
person.on('speak', function(msg){
console.log(person.name + 'said:' + msg);
});
});
mohammed.emit('speak', "Hi guys");
tahir.emit('speak', "i want a chicked");
can anyone fix this? and let me know where I am gone wrong and why?
You're trying to call an .on method on a string in the people array.
You'll need to put the person objects you've created, not strings, into the people array:
var people = [mohammed, tahir, taha];
instead of
var people = ['mohammed', 'tahir', 'taha'];
I need to run two commands in series that need to read data from the same stream.
After piping a stream into another the buffer is emptied so i can't read data from that stream again so this doesn't work:
var spawn = require('child_process').spawn;
var fs = require('fs');
var request = require('request');
var inputStream = request('http://placehold.it/640x360');
var identify = spawn('identify',['-']);
inputStream.pipe(identify.stdin);
var chunks = [];
identify.stdout.on('data',function(chunk) {
chunks.push(chunk);
});
identify.stdout.on('end',function() {
var size = getSize(Buffer.concat(chunks)); //width
var convert = spawn('convert',['-','-scale',size * 0.5,'png:-']);
inputStream.pipe(convert.stdin);
convert.stdout.pipe(fs.createWriteStream('half.png'));
});
function getSize(buffer){
return parseInt(buffer.toString().split(' ')[2].split('x')[0]);
}
Request complains about this
Error: You cannot pipe after data has been emitted from the response.
and changing the inputStream to fs.createWriteStream yields the same issue of course.
I don't want to write into a file but reuse in some way the stream that request produces (or any other for that matter).
Is there a way to reuse a readable stream once it finishes piping?
What would be the best way to accomplish something like the above example?
You have to create duplicate of the stream by piping it to two streams. You can create a simple stream with a PassThrough stream, it simply passes the input to the output.
const spawn = require('child_process').spawn;
const PassThrough = require('stream').PassThrough;
const a = spawn('echo', ['hi user']);
const b = new PassThrough();
const c = new PassThrough();
a.stdout.pipe(b);
a.stdout.pipe(c);
let count = 0;
b.on('data', function (chunk) {
count += chunk.length;
});
b.on('end', function () {
console.log(count);
c.pipe(process.stdout);
});
Output:
8
hi user
The first answer only works if streams take roughly the same amount of time to process data. If one takes significantly longer, the faster one will request new data, consequently overwriting the data still being used by the slower one (I had this problem after trying to solve it using a duplicate stream).
The following pattern worked very well for me. It uses a library based on Stream2 streams, Streamz, and Promises to synchronize async streams via a callback. Using the familiar example from the first answer:
spawn = require('child_process').spawn;
pass = require('stream').PassThrough;
streamz = require('streamz').PassThrough;
var Promise = require('bluebird');
a = spawn('echo', ['hi user']);
b = new pass;
c = new pass;
a.stdout.pipe(streamz(combineStreamOperations));
function combineStreamOperations(data, next){
Promise.join(b, c, function(b, c){ //perform n operations on the same data
next(); //request more
}
count = 0;
b.on('data', function(chunk) { count += chunk.length; });
b.on('end', function() { console.log(count); c.pipe(process.stdout); });
You can use this small npm package I created:
readable-stream-clone
With this you can reuse readable streams as many times as you need
For general problem, the following code works fine
var PassThrough = require('stream').PassThrough
a=PassThrough()
b1=PassThrough()
b2=PassThrough()
a.pipe(b1)
a.pipe(b2)
b1.on('data', function(data) {
console.log('b1:', data.toString())
})
b2.on('data', function(data) {
console.log('b2:', data.toString())
})
a.write('text')
I have a different solution to write to two streams simultaneously, naturally, the time to write will be the addition of the two times, but I use it to respond to a download request, where I want to keep a copy of the downloaded file on my server (actually I use a S3 backup, so I cache the most used files locally to avoid multiple file transfers)
/**
* A utility class made to write to a file while answering a file download request
*/
class TwoOutputStreams {
constructor(streamOne, streamTwo) {
this.streamOne = streamOne
this.streamTwo = streamTwo
}
setHeader(header, value) {
if (this.streamOne.setHeader)
this.streamOne.setHeader(header, value)
if (this.streamTwo.setHeader)
this.streamTwo.setHeader(header, value)
}
write(chunk) {
this.streamOne.write(chunk)
this.streamTwo.write(chunk)
}
end() {
this.streamOne.end()
this.streamTwo.end()
}
}
You can then use this as a regular OutputStream
const twoStreamsOut = new TwoOutputStreams(fileOut, responseStream)
and pass it to to your method as if it was a response or a fileOutputStream
If you have async operations on the PassThrough streams, the answers posted here won't work.
A solution that works for async operations includes buffering the stream content and then creating streams from the buffered result.
To buffer the result you can use concat-stream
const Promise = require('bluebird');
const concat = require('concat-stream');
const getBuffer = function(stream){
return new Promise(function(resolve, reject){
var gotBuffer = function(buffer){
resolve(buffer);
}
var concatStream = concat(gotBuffer);
stream.on('error', reject);
stream.pipe(concatStream);
});
}
To create streams from the buffer you can use:
const { Readable } = require('stream');
const getBufferStream = function(buffer){
const stream = new Readable();
stream.push(buffer);
stream.push(null);
return Promise.resolve(stream);
}
What about piping into two or more streams not at the same time ?
For example :
var PassThrough = require('stream').PassThrough;
var mybiraryStream = stream.start(); //never ending audio stream
var file1 = fs.createWriteStream('file1.wav',{encoding:'binary'})
var file2 = fs.createWriteStream('file2.wav',{encoding:'binary'})
var mypass = PassThrough
mybinaryStream.pipe(mypass)
mypass.pipe(file1)
setTimeout(function(){
mypass.pipe(file2);
},2000)
The above code does not produce any errors but the file2 is empty
I have node js files restservice.js and mysql.js
In mysql.js i have two functions as elementlevelpricing and pricingdetail
In restservice.js i have api which has the code as :
var workload = req.body;
var workloadinfo = {
workloadId: workload.workloadId,
ownerId: workload.ownerId,
uniqueName: workload.uniqueName,
name: workload.name
}
if(workload.elements && workload.elements.length > 0)
{
var elementlevelpricingSummary = {};
var elementArray = [];
var elementinfo = {};
var metadataModified = {};
var pricingDetail = {};
async.forEachSeries(workload.elements, createResponse, function (err) {
res.send(workloadinfo);
});
function createResponse(elements,callback) {
var resourceIdentifierArray = [];
elementinfo = elements;
resourceIdentifierArray.push(elements.uri);
var resourceIdentifiers = resourceIdentifierArray.join(',');
// Get element level pricing summary
mysql.elementlevelpricing(resourceIdentifiers, function(result){
// do some stuff here
return callback();
});
};
};
I need to call the function pricingdetail in mysql.js and append the result to global variable workloadinfo (which already should have result set of elementlevelpricing and Can thats what is sent within foreachSeries ). Can anyone suggest me the profession way to accomplish this?
Use asynchronous functions. The whole point of Node.js is to avoid blocking. Blocking in Node.js is worse than blocking in threaded environments, because there aren't any other threads (though there may be other clustered processes). You're blocking the only event loop available. That means that your whole server has to wait, doing absolutely no work until your I/O is done.
I'm using Node.JS with Socket.IO and Express. My goal is to create a module that creates a custom Socket.IO+Express server, loaded with all the functions that my application needs. For example, keeping track of the number of clients connected at any given time.
The catch is, I'd like to have the server work as a class, if possible. I've seen other Node.JS modules use classes just fine (including Socket.IO itself) so I assume it shouldn't conflict with Node's module-oriented architecture. The reason why I need it to work with classes is that I want to be able to create multiple instances of the server with ease.
Here's what I have (simplified for brevity):
index.js
var myserver = require("./myserver");
var myserver1 = myserver.createServer();
var myserver2 = myserver.createServer();
myserver1.port = 8080;
myserver2.port = 8081;
myserver1.start();
myserver2.start();
myserver.js
var io = require("socket.io");
var express = require("express");
var path = require("path");
function MyServer()
{
this.port = 8080;
this.expressServer = null;
this.ioServer = null;
this.clientCount = 0;
}
Server.prototype.getClientCount = function()
{
return this.clientCount;
}
Server.prototype.start = function()
{
this.expressServer = express.createServer();
this.ioServer = io.listen(this.expressServer);
this.expressServer.listen(this.port);
this.ioServer.sockets.on(
"connection",
function()
{
this.clientCount++;
console.log(this.clientCount + " clients connected.");
}
);
}
exports.createServer = function() { return new MyServer(); };
The code inside the "connection" callback is incorrect, because the this keyword in Socket.IO's callback refers to the client object that triggered the "connection" event, not to the MyServer object. So is there a way to access the clientCount property from inside the callback?
Create a proper closure by copying this into another a variable (most people like to call it self but some prefer that) and using it in your connection handler:
var self=this;
this.ioServer.sockets.on(
"connection",
function()
{
self.clientCount++;
console.log(self.clientCount + " clients connected.");
}
);