I am new to node js. I have written a script that spawns 5 child process and captures log from them,
As of now the log looks like mess.
But i want to display the output in a cleaner format, like each line gets reused and data gets loged next to the process id.
something like this:
below is my code:
var state = "node stealth.js ";
var exec = require('child_process').exec;
workers = 5
p_list = [];
for (var i = 0; i < workers; i++) {
(function(i){
var child = exec(state+'user'+i);
console.log('started Worker '+i)
// Add the child process to the list for tracking
p_list.push({process:child, content:""});
// Listen for any response:
child.stdout.on('data', function (data) {
console.log(child.pid, data);
p_list[i].content += data;
process.stdout.cursorTo(i);
process.stdout.clearLine();
process.stdout.write(child.pid+data);
process.stdout.write("\n"); // end the line
});
// Listen for any errors:
child.stderr.on('data', function (data) {
console.log(child.pid, data);
p_list[i].content += data;
});
// Listen if the process closed
child.on('close', function(exit_code) {
console.log('Closed before stop: Closing code: ', exit_code);
});
})(i)
}
You are looking for VT100 Terminal Control Escape Sequences. These sequences allow to move the cursor around or color the text.
Example
console.log('Line 1');
console.log('Line 2');
console.log('\x1B[1AAgain line 2');
The \x1B[1A sequence at the start moves the cursor one line up and therefore the text after it overwrites the previous text. The result looks like this:
Line 1
Again line 2
For more sequences, check out the part of the page about cursor movement.
Finally i figured out a solution.
Since i need to gather data from different processes and display in a neat way, i started using pm2 process manager, it does a lot more than just console log from processes.
for anyone facing same problem please check out this node package
Related
I am trying to write a GUI with Node and Electron around an existing ruby command line application. I found an example how to get output from a child process by doing something like:
var spawn = require('child_process').spawn;
var child = spawn('node', ['child.js']);
child.stdout.on('data', function (data) {
console.log('stdout: ' + data);
});
child.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
And the child.js looks like this
while(true) {
console.log('blah');
}
This is working fine for me, but if I try and do this with a ruby application switching to this
var child = spawn('ruby', ['test.rb']);
and this being the ruby code
while true
sleep 2
puts 'test'
end
I get no output. It seems to hang on the output. I would expect to see 'test' printed every 2 seconds.
Ruby doesn't know you want the output right away, and buffers it for efficiency. If you make it flush the output buffer (IO#flush), it works.
while true
sleep 2
puts 'test'
STDOUT.flush
end
Alternately, you can tell Ruby you want all output right away (IO#sync=):
STDOUT.sync = true
while true
sleep 2
puts 'test'
end
Incidentally, your original code works on JRuby, so the buffering behaviour differs... I did not know that...
I'm trying to write a small node application that will search through and parse a large number of files on the file system.
In order to speed up the search, we are attempting to use some sort of map reduce. The plan would be the following simplified scenario:
Web request comes in with a search query
3 processes are started that each get assigned 1000 (different) files
once a process completes, it would 'return' it's results back to the main thread
once all processes complete, the main thread would continue by returning the combined result as a JSON result
The questions I have with this are:
Is this doable in Node?
What is the recommended way of doing it?
I've been fiddling, but come no further then following example using Process:
initiator:
function Worker() {
return child_process.fork("myProcess.js");
}
for(var i = 0; i < require('os').cpus().length; i++){
var process = new Worker();
process.send(workItems.slice(i * itemsPerProcess, (i+1) * itemsPerProcess));
}
myProcess.js
process.on('message', function(msg) {
var valuesToReturn = [];
// Do file reading here
//How would I return valuesToReturn?
process.exit(0);
}
Few sidenotes:
I'm aware the number of processes should be dependent of the number of CPU's on the server
I'm also aware of speed restrictions in a file system. Consider it a proof of concept before we move this to a database or Lucene instance :-)
Should be doable. As a simple example:
// parent.js
var child_process = require('child_process');
var numchild = require('os').cpus().length;
var done = 0;
for (var i = 0; i < numchild; i++){
var child = child_process.fork('./child');
child.send((i + 1) * 1000);
child.on('message', function(message) {
console.log('[parent] received message from child:', message);
done++;
if (done === numchild) {
console.log('[parent] received all results');
...
}
});
}
// child.js
process.on('message', function(message) {
console.log('[child] received message from server:', message);
setTimeout(function() {
process.send({
child : process.pid,
result : message + 1
});
process.disconnect();
}, (0.5 + Math.random()) * 5000);
});
So the parent process spawns an X number of child processes and passes them a message. It also installs an event handler to listen for any messages sent back from the child (with the result, for instance).
The child process waits for messages from the parent, and starts processing (in this case, it just starts a timer with a random timeout to simulate some work being done). Once it's done, it sends the result back to the parent process and uses process.disconnect() to disconnect itself from the parent (basically stopping the child process).
The parent process keeps track of the number of child processes started, and the number of them that have sent back a result. When those numbers are equal, the parent received all results from the child processes so it can combine all results and return the JSON result.
For a distributed problem like this, I've used zmq and it has worked really well. I'll give you a similar problem that I ran into, and attempted to solve via processes (but failed.) and then turned towards zmq.
Using bcrypt, or an expensive hashing algorith, is wise, but it blocks the node process for around 0.5 seconds. We had to offload this to a different server, and as a quick fix, I used essentially exactly what you did. Run a child process and send messages to it and get it to
respond. The only issue we found is for whatever reason our child process would pin an entire core when it was doing absolutely no work.(I still haven't figured out why this happened, we ran a trace and it appeared that epoll was failing on stdout/stdin streams. It would also only happen on our Linux boxes and would work fine on OSX.)
edit:
The pinning of the core was fixed in https://github.com/joyent/libuv/commit/12210fe and was related to https://github.com/joyent/node/issues/5504, so if you run into the issue and you're using centos + kernel v2.6.32: update node, or update your kernel!
Regardless of the issues I had with child_process.fork(), here's a nifty pattern I always use
client:
var child_process = require('child_process');
function FileParser() {
this.__callbackById = [];
this.__callbackIdIncrement = 0;
this.__process = child_process.fork('./child');
this.__process.on('message', this.handleMessage.bind(this));
}
FileParser.prototype.handleMessage = function handleMessage(message) {
var error = message.error;
var result = message.result;
var callbackId = message.callbackId;
var callback = this.__callbackById[callbackId];
if (! callback) {
return;
}
callback(error, result);
delete this.__callbackById[callbackId];
};
FileParser.prototype.parse = function parse(data, callback) {
this.__callbackIdIncrement = (this.__callbackIdIncrement + 1) % 10000000;
this.__callbackById[this.__callbackIdIncrement] = callback;
this.__process.send({
data: data, // optionally you could pass in the path of the file, and open it in the child process.
callbackId: this.__callbackIdIncrement
});
};
module.exports = FileParser;
child process:
process.on('message', function(message) {
var callbackId = message.callbackId;
var data = message.data;
function respond(error, response) {
process.send({
callbackId: callbackId,
error: error,
result: response
});
}
// parse data..
respond(undefined, "computed data");
});
We also need a pattern to synchronize the different processes, when each process finishes its task, it will respond to us, and we'll increment a count for each process that finishes, and then call the callback of the Semaphore when we've hit the count we want.
function Semaphore(wait, callback) {
this.callback = callback;
this.wait = wait;
this.counted = 0;
}
Semaphore.prototype.signal = function signal() {
this.counted++;
if (this.counted >= this.wait) {
this.callback();
}
}
module.exports = Semaphore;
here's a use case that ties all the above patterns together:
var FileParser = require('./FileParser');
var Semaphore = require('./Semaphore');
var arrFileParsers = [];
for(var i = 0; i < require('os').cpus().length; i++){
var fileParser = new FileParser();
arrFileParsers.push(fileParser);
}
function getFiles() {
return ["file", "file"];
}
var arrResults = [];
function onAllFilesParsed() {
console.log('all results completed', JSON.stringify(arrResults));
}
var lock = new Semaphore(arrFileParsers.length, onAllFilesParsed);
arrFileParsers.forEach(function(fileParser) {
var arrFiles = getFiles(); // you need to decide how to split the files into 1k chunks
fileParser.parse(arrFiles, function (error, result) {
arrResults.push(result);
lock.signal();
});
});
Eventually I used http://zguide.zeromq.org/page:all#The-Load-Balancing-Pattern, where the client was using the nodejs zmq client, and the workers/broker were written in C. This allowed us to scale this across multiple machines, instead of just a local machine with sub processes.
I spawn a child process like this:
var child = require('child_process');
var proc = child.spawn('python', ['my_script.py', '-p', 'example']);
I also set some data handling:
proc.stdin.setEncoding('utf8');
proc.stdout.setEncoding('utf8');
proc.stderr.setEncoding('utf8');
proc.stdout.on('data', function (data) {
console.log('out: ' + data);
});
proc.stderr.on('data', function (data) {
console.log('err: ' + data);
});
proc.on('close', function (code) {
console.log('subprocess exited with status ' + code);
proc.stdin.end();
});
My Python script reads lines from stdin and for each line does some operations and prints to stdout. It works fine in the shell (I write a line and I get the output immediately) but when I do this in Node:
for (var i = 0; i < 10; i++) {
proc.stdin.write('THIS IS A TEST\n');
}
I get nothing.
I got to fix it calling proc.stdin.end() but that also terminates the child process (which I want to stay in background, streaming data).
I also triggered a flush filling the buffer with lots of writes, but that's not really an option.
Is there any way to manually flush the stream?
You are not flushing the output from Python after print statement. I had similar problem and #Alfe answered my question. Take a look at this:
Stream child process output in flowing mode
Consider the following C program (test.c):
#include <stdio.h>
int main() {
printf("string out 1\n");
fprintf(stderr, "string err 1\n");
getchar();
printf("string out 2\n");
fprintf(stderr, "string err 2\n");
fclose(stdout);
}
Which should print a line to stdout, a line to stderr, then wait for user input, then another line to stdout and another line to stderr. Very basic!
When compiled and run on the command line the output of the program when complete (user input is received for getchar()):
$ ./test
string out 1
string err 1
string out 2
string err 2
When trying to spawn this program as a child process using nodejs with the following code:
var TEST_EXEC = './test';
var spawn = require('child_process').spawn;
var test = spawn(TEST_EXEC);
test.stdout.on('data', function (data) {
console.log('stdout: ' + data);
});
test.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
// Simulate entering data for getchar() after 1 second
setTimeout(function() {
test.stdin.write('\n');
}, 1000);
The output appears like this:
$ nodejs test.js
stderr: string err 1
stdout: string out 1
string out 2
stderr: string err 2
Very different from the output as seen when running ./test in the terminal. This is because the ./test program isn't running in an interactive shell when spawned by nodejs. The test.c stdout stream is buffered and when run in a terminal as soon as a \n is reached the buffer is flushed but when spawned in this way with node the buffer isn't flushed. This could be resolved by either flushing stdout after every print, or changing the stdout stream to be unbuffered so it flushes everything immediately.
Assuming that test.c source isn't available or modifiable, neither of the two flushing options mentioned can be implemented.
I then started looking at emulating an interactive shell, there's pty.js (pseudo terminal) which does a good job, for example:
var spawn = require('pty.js').spawn;
var test = spawn(TEST_EXEC);
test.on('data', function (data) {
console.log('data: ' + data);
});
// Simulate entering data for getchar() after 1 second
setTimeout(function() {
test.write('\n');
}, 1000);
Which outputs:
$ nodejs test.js
data: string out 1
string err 1
data:
data: string out 2
string err 2
However both stdout and stderr are merged together (as you would see when running the program in a terminal) and I can't think of a way to separate the data from the streams.
So the question..
Is there any way using nodejs to achieve the output as seen when running ./test without modifying the test.c code? Either by terminal emulation or process spawning or any other method?
Cheers!
I tried the answer by user568109 but this does not work, which makes sense since the pipe only copies the data between streams. Hence, it only gets to process.stdout when the buffer is flushed...
The following appears to work:
var TEST_EXEC = './test';
var spawn = require('child_process').spawn;
var test = spawn(TEST_EXEC, [], { stdio: 'inherit' });
//the following is unfortunately not working
//test.stdout.on('data', function (data) {
// console.log('stdout: ' + data);
//});
Note that this effectively shares stdio's with the node process. Not sure if you can live with that.
I was just revisiting this since there is now a 'shell' option available for the spawn command in node since version 5.7.0. Unfortunately there doesn't seem to be an option to spawn an interactive shell (I also tried with shell: '/bin/sh -i' but no joy).
However I just found this which suggests using 'stdbuf' allowing you to change the buffering options of the program that you want to run. Setting them to 0 on everything produces unbuffered output for all streams and they're still kept separate.
Here's the updated javascript:
var TEST_EXEC = './test';
var spawn = require('child_process').spawn;
var test = spawn('stdbuf', ['-i0', '-o0', '-e0', TEST_EXEC]);
test.stdout.on('data', function (data) {
console.log('stdout: ' + data);
});
test.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
// Simulate entering data for getchar() after 1 second
setTimeout(function() {
test.stdin.write('\n');
}, 1000);
Looks like this isn't pre-installed on OSX and of course not available for Windows, may be similar alternatives though.
You can do this :
var TEST_EXEC = 'test';
var spawn = require('child_process').spawn;
var test = spawn(TEST_EXEC);
test.stdin.pipe(process.stdin);
test.stdout.pipe(process.stdout);
test.stderr.pipe(process.stderr);
When you use events on stdout and stderr to print the output on console.log, you will get jumbled output because of asynchronous execution of the functions. The output will be ordered for a stream independently, but output can still get interleaved among stdin,stdout and stderr.
Is there any stdout flush for nodejs just like python or other languages?
sys.stdout.write('some data')
sys.stdout.flush()
Right now I only saw process.stdout.write() for nodejs.
process.stdout is a WritableStream object, and the method WritableStream.write() automatically flushes the stream (unless it was explicitly corked). However, it will return true if the flush was successful, and false if the kernel buffer was full and it can't write yet. If you need to write several times in succession, you should handle the drain event.
See the documentation for write.
In newer NodeJS versions, you can pass a callback to .write(), which will be called once the data is flushed:
sys.stdout.write('some data', () => {
console.log('The data has been flushed');
});
This is exactly the same as checking .write() result and registering to the drain event:
let write = sys.stdout.write('some data');
if (!write) {
sys.stdout.once('drain', () => {
console.log('The data has been flushed');
});
}
write returns true if the data has been flushed. If it returns false, you can wait for the 'drain' event.
I think there is no flush, because that would be a blocking operation.
There is another function stdout which to clear last output to the terminal which is kind of work like flush
function flush() {
process.stdout.clearLine();
process.stdout.cursorTo(0);
}
var total = 5000;
var current = 0;
var percent = 0;
var waitingTime = 500;
setInterval(function() {
current += waitingTime;
percent = Math.floor((current / total) * 100);
flush();
process.stdout.write(`downloading ... ${percent}%`);
if (current >= total) {
console.log("\nDone.");
clearInterval(this);
}
}, waitingTime);
cursorTo will move the cursor to position 0 which is the starting point
use the flush function before stdout.write because it will clear the screen, if you put after you will not see any output