Creating NodeJS child processes asynchronously - node.js

I am creating child processes in NodeJS in a function called "pythonGraphTools" in a for loop after generating some variables that need to be passed in. This for loop may run 50 times.
Then I am writing to the stdin of the spawned process. However, sometimes I am getting a "Error:EPIPE writing to closed socket" error for this line py.stdin.write(JSON.stringify(dotfilepath));
I suspect it is because the child process has not yet completed spawning and am attempting to write to it when it is not ready. I have seen asynchronous spawning such as that on https://nodejs.org/api/child_process.html#child_process_child_process_spawn_command_args_options but these only seem to have asynch events for the flow of data/messages from child to parent.
Any insight in how I might make sure that the child is fully spawned before I call py.stdin.write()
function pythonGraphTools(dotfilepath,allGraphsPerTrans,graphtools_color,graphtools_label,transHashArray){
var spawn = require('child_process').spawn,
py = spawn('python', ['python_module.py']);
//write file to disk temporarily.
console.log("dotfilepath is "+dotfilepath)
fs.writeFile(dotfilepath,allGraphsPerTrans, function(err){ //must create a file first //2nd param was res_str_dot_no_lbl
if(err){
console.log("there was an error writing to file" + err);
}
//now send this dot file path to the python module which will make the graph
console.log("now writing to python module!"+py.pid)
console.log("nodejs colorarray length for debuging "+ graphtools_color.length)
py.stdin.write(JSON.stringify(dotfilepath)); //sending data to the python process!
py.stdin.write("\n")
py.stdin.write(JSON.stringify(graphtools_color)); // sending colours
py.stdin.write("\n")
py.stdin.write(JSON.stringify(graphtools_label));//sending opcodes
py.stdin.write("\n");
py.stdin.write(JSON.stringify(transHashArray));//sending opcodes
py.stdin.write("\n");
py.stdin.end();
});
var dataString=""; //variable to store return from python module
py.stdout.on('data', function(data){ // listen for data coming back from python!
dataString += data.toString();
});
py.stdout.on('end', function(){ //pythons stdout has finished - now do stuff
console.log(dataString); // print out everything collected from python stdout
//now delete temp dot file (with all dot files in it)
fs.stat(dotfilepath, function (err, stats) { //check first if there is a dot file
console.log(stats);//here we got all information of file in stats variable
if (err) {
return console.error(err);
}
fs.unlink(dotfilepath,function(err){ //actually deleting comment this functiont to not delete
if(err) return console.log(err);
console.log('file deleted successfully');
});//end unlink
});//end file stat
py.stdout.end();
}); // on python 'finish'
py.on('exit', function (code, signal) { //which process? add pid ?
console.log('child process '+py.pid +' exited with ' +
`code ${code} and signal ${signal}`);
});
}

Related

Can I “listen” for a specific output with child_process?

So far I have gotten my script to execute a windows .bat file with child_process, my issue is that it opens it in the background with no way to “connect” to it to see what happens and debug, is there a way to “listen” for a certain output to happen? For example, if the .bat outputs a “Done!” in the shell at one point, is there a way to make my node.js script detect that certain keyword and run further commands if it does?
Thanks!
Some clarification: The .bat outputs "Done!" and stays running, it doesn't stop, all I want to do is detect that "Done!" so that I can send a message to the user that the server has successfully started
My current code:
exec('D:\\servers\\game_server_1\\start.bat', {shell: true, cwd: 'D:\\servers\\game_server_1'});
Well, if you're trying to do a one and done type of NodeJS script, you can just spawn a process that launches with the given command and exits when all commands completed. This creates a one and done streaming interface that you can monitor. The stdout returns a data buffer that returns the command you ran, unless it's something like START to launch a program-- it returns null. You could just issue a KILL command after the START -- your_program.exe:
const spawn = require('child_process').spawn;
const child = spawn('cmd.exe', ['/c', 'commands.bat']);
let DONE = 0;
const done = () => {
console.log("log it");
DONE++;
};
child.stdout.on('data', function (data) {
console.log('stdout: ' + data);
//it's important to add some type of counter to
//prevent any logic from running twice, since
//this will run twice for any given command
if ( data.toString().includes("DONE") && DONE === 0 ) {
done();
}
});
child.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
child.on('exit', function (code) {
console.log('child process exited with code ' + code);
});
Keep in mind, when you run a command to launch a program and the program launches, the data buffer will be null in stdout event listener. The error event will only fire if there was an issue with launching the program.
YOUR .BAT:
ECHO starting batch script
//example launching of program
START "" https://localhost:3000
//issue a command after your program launch
ECHO DONE
EXIT
You could also issue an ECHO DONE command right after the command where you launched the program and listen for that, and try and parse out that command from stdout.
You could use a Regular expression.
const { spawn } = require('child_process');
const child = spawn(...);
child.stdout.on('data', function (data) {
console.log('stdout: ' + data);
// Now use a regular expression to detect a done event
// For example
data.toString().match(/Done!/);
});
// Error handling etc. here

NodeJS bug in Linux when executing child_process.fork?

I cannot reliably get a forked child process to send back a message to the parent that exceeds 219262 bytes.
The issue is only on Linux. In Windows, it works as expected. And this issue seems have been introduced between Node versions 1.0.1 and 1.0.2 - works fine on Node versions prior to 1.0.1 but not after.
(the maxBuffer option is not relevent for child_process.fork, it only applies to child_process.exec and child_process.execFile)
Below is the failing sample. Executing "node parent" on the command line will fail to output the child's "messageToParent" if it exceeds 219262 bytes on Linux.
parent.js is:
var cp = require('child_process');
var child = cp.fork('./child', [], {});
console.log('>>>PARENT ---> SENDING MESSAGE TO CHILD');
child.send({});
child.on('message', function(msg) {
console.log('>>>PARENT ---> MESSAGE RECEIVED FROM CHILD = ' + JSON.stringify(msg));
});
child.on('error', function(err) {
console.log('>>>PARENT ---> ERROR FROM CHILD. err = '+ err);
});
child.on('exit', function(code, signal) {
console.log('>>>PARENT ---> EXIT FROM CHILD. code='+code+' signal = '+ signal);
});
child.on('close', function(code, signal) {
console.log('>>>PARENT ---> CLOSE FROM CHILD. code='+code+' signal = '+signal);
});
child.on('disconnect', function() {
console.log('>>>PARENT ---> DISCONNECT FROM CHILD');
});
child.js is
process.on('message', function(messageFromParent) {
console.log('>>>>>>CHILD ---> RECEIVED MESSAGE FROM PARENT');
var messageToParent = "It would be too long to post on stackoverflow, but if I make this string longer than 219262 bytes, it fails to return to the parent in Linux. There is no such issue in Windows";
var ret = process.send(messageToParent);
console.log('>>>>>>CHILD ---> SENDING MESSAGE TO PARENT process.send returned ' + ret);
process.exit(0);
});
process.on('uncaughtException', function(err) {
process.send({ output: {ERROR:err} });
process.exit(-1);
});
Posting an answer in case anyone else stumbles into this issue (https://github.com/nodejs/node/issues/36268)
The above child.js works perfectly in Node versions prior to 1.0.1 since child_process.fork() used to be synchronous. So "process.send(messageToParent)", followed by "process.exit(0)" will always return messageToParent to parent.js.
In later versions of Node, however, process.send() is async. Therefore, the child must exit via process.exit() within a process.send callback, else a race condition is created between V8 javascript thread and IPC pipe.
Also - in Windows, the default IPC pipe buffer is large enough that the message is always returned to parent prior to child exiting. This is not the case in Linux. This explains why the above code works in Windows even with later versions of Node where process.send() is async.

child_process spawn Race condition possibility in nodejs

I'm starting to learn and use node and I like it but I'm not really sure how certain features work. Maybe you can help me resolve one such issue:
I want to spawn local scripts and programs from my node server upon rest commands. looking at the fs library I saw the example below of how to spawn a child process and add some pipes/event handlers on it.
var spawn = require('child_process').spawn,
ps = spawn('ps', ['ax']),
grep = spawn('grep', ['ssh']);
ps.stdout.on('data', function (data) {
grep.stdin.write(data);
});
ps.stderr.on('data', function (data) {
console.log('ps stderr: ' + data);
});
ps.on('close', function (code) {
if (code !== 0) {
console.log('ps process exited with code ' + code);
}
grep.stdin.end();
});
grep.stdout.on('data', function (data) {
console.log('' + data);
});
grep.stderr.on('data', function (data) {
console.log('grep stderr: ' + data);
});
grep.on('close', function (code) {
if (code !== 0) {
console.log('grep process exited with code ' + code);
}
});
What's weird to me is that I don't understand how I can be guaranteed that the event handler code will be registered before the program starts to run. It's not like there's a 'resume' function that you run to start up the child. Isn't this a race condition? Granted the condition would be minisculy small and would almost never hit because its such a short snipping of code afterward but still, if it is I'd rather not code it this way out of good habits.
So:
1) if it's not a race condition why?
2) if it is a race condition how could I write it the right way?
Thanks for your time!
Given the slight conflict and ambiguity in the accepted answer's comments, the sample and output below tells me two things:
The child process (referring to the node object returned by spawn) emits no events even though the real underlying process is live / executing.
The pipes for the IPC are setup before the child process is executed.
Both are obvious. The conflict is w.r.t. interpretation of the OP's question:-
Actually 'yes', this is the epitome of a data race condition if one needs to consider the real child process's side effects. But 'no', there's no data race as far as IPC pipe plumbing is concerned. The data is written to a buffer and retrieved as a (bigger) blob as and when (as already well described) the context completes allowing the event loop to continue.
The first data event seen below pushes not 1 but 5 chunks written to stdout by the child process whilst we were blocking.. thus nothing is lost.
sample:
let t = () => (new Date()).toTimeString().split(' ')[0]
let p = new Promise(function (resolve, reject) {
console.log(`[${t()}|info] spawning`);
let cp = spawn('bash', ['-c', 'for x in `seq 1 1 10`; do printf "$x\n"; sleep 1; done']);
let resolved = false;
if (cp === undefined)
reject();
cp.on('error', (err) => {
console.log(`error: ${err}`);
reject(err);
});
cp.stdout.on('data', (data) => {
if (!resolved) {
console.log(`[${t()}|info] spawn succeeded`);
resolved = true;
resolve();
}
process.stdout.write(`[${t()}|data] ${data}`);
});
let ts = parseInt(Date.now() / 1000);
while (parseInt(Date.now() / 1000) - ts < 5) {
// waste some cycles in the current context
ts--; ts++;
}
console.log(`[${t()}|info] synchronous time wasted`);
});
Promise.resolve(p);
output:
[18:54:18|info] spawning
[18:54:23|info] synchronous time wasted
[18:54:23|info] spawn succeeded
[18:54:23|data] 1
2
3
4
5
[18:54:23|data] 6
[18:54:24|data] 7
[18:54:25|data] 8
[18:54:26|data] 9
[18:54:27|data] 10
It is not a race condition. Node.js is single threaded and handles events on a first come first serve basis. New events are put at the end of the event loop. Node will execute your code in a synchronous manner, part of which will involve setting up event emitters. When these event emitters emit events, they will be put to the end of the queue, and will not be handled until Node finishes executing whatever piece of code its currently working on, which happens to be the same code that registers the listener. Therefore, the listener will always be registered before the event is handled.

Prevent sending data to stdin if spawn fails

In my Node.js (v0.10.9) code I'm trying to detect 2 cases:
an external tool (dot) is installed - in that case I want to send some data to stdin of created process
the external tool is not installed - in that case I want to display warning and I don't want to send anything to process' stdin
My problem is that I don't know how to send data to child's stdin if and only if the process was spawned successfully (i.e. stdin is ready for writing).
Following code works fine if dot is installed, but otherwise it tries to send data to the child although the child wasn't spawned.
var childProcess = require('child_process');
var child = childProcess.spawn('dot');
child.on('error', function (err) {
console.error('Failed to start child process: ' + err.message);
});
child.stdin.on('error', function(err) {
console.error('Working with child.stdin failed: ' + err.message);
});
// I want to execute following lines only if child process was spawned correctly
child.stdin.write('data');
child.stdin.end();
I'd need something like this
child.on('successful_spawn', function () {
child.stdin.write('data');
child.stdin.end();
});
From the node.js docs: http://nodejs.org/api/child_process.html#child_process_child_process_spawn_command_args_options
Example of checking for failed exec:
var spawn = require('child_process').spawn,
child = spawn('bad_command');
child.stderr.setEncoding('utf8');
child.stderr.on('data', function (data) {
if (/^execvp\(\)/.test(data)) {
console.log('Failed to start child process.');
}
});
Have a look at core-worker:
https://www.npmjs.com/package/core-worker
This package makes it a lot easier to handle processes.
I think what you want to do is something like that (from the docs):
import { process } from "core-worker";
const simpleChat = process("node chat.js", "Chat ready");
setTimeout(() => simpleChat.kill(), 360000); // wait an hour and close the chat
simpleChat.ready(500)
.then(console.log.bind(console, "You are now able to send messages."))
.then(::simpleChat.death)
.then(console.log.bind(console, "Chat closed"))
.catch(() => /* handle err */);
So if the process is not started correctly, none of the .then statements are executed which is exactly what you want to do, right?

Force Node.js to flush writes to child processes

I spawn a child process like this:
var child = require('child_process');
var proc = child.spawn('python', ['my_script.py', '-p', 'example']);
I also set some data handling:
proc.stdin.setEncoding('utf8');
proc.stdout.setEncoding('utf8');
proc.stderr.setEncoding('utf8');
proc.stdout.on('data', function (data) {
console.log('out: ' + data);
});
proc.stderr.on('data', function (data) {
console.log('err: ' + data);
});
proc.on('close', function (code) {
console.log('subprocess exited with status ' + code);
proc.stdin.end();
});
My Python script reads lines from stdin and for each line does some operations and prints to stdout. It works fine in the shell (I write a line and I get the output immediately) but when I do this in Node:
for (var i = 0; i < 10; i++) {
proc.stdin.write('THIS IS A TEST\n');
}
I get nothing.
I got to fix it calling proc.stdin.end() but that also terminates the child process (which I want to stay in background, streaming data).
I also triggered a flush filling the buffer with lots of writes, but that's not really an option.
Is there any way to manually flush the stream?
You are not flushing the output from Python after print statement. I had similar problem and #Alfe answered my question. Take a look at this:
Stream child process output in flowing mode

Resources