I try to create a child process from node.js script (parent) in the way where will be no interference between the parent and child data flow. The parent should only have the child process id when a child process start and be able to terminate it. child_process.exec does part of what I need but unfortunately it buffers child's output and therefore as the buffer gets full the child crashes. Is there a way to eliminate the buffering? The child process is infinite in its nature (data stream). Or may be there is other way to implement the controls?
Code sample (just for demo):
const { exec } = require('child_process');
const keypress = require('keypress');
keypress(process.stdin);
var c1;
process.stdin.on('keypress', (letter, key) => {
if (key && key.name == 'a') {
c1.kill();
} else if (key && key.name == 'b') {
c1 = exec('ffplay -i udp://localhost:4000', (err, stdout, stderr) => {
if (err) {
console.error(`error: ${err}`);
}
});
}
});
I've tried spawn but there child - parent interference is even more severe
If you use child_process.spawn instead, you should be able to call it with options for stdio:
spawn(cmd, [], { stdio: 'ignore' });
child_process.spawn docs
EDIT:
If you are a fan of Promise, here's a util fn I wrote to help with things
const quietSpawn = (cmd, args = []) => {
const splitCmd = cmd.split(' ');
if (splitCmd.length > 1) {
[cmd, ...args] = splitCmd;
}
return new Promise((resolve, reject) => {
const proc = spawn(cmd, args, { stdio: 'ignore' });
proc.on('exit', resolve);
proc.on('error', reject);
});
};
Related
I have a Node JS CLI script that automates some migrations to a third-party service. I've largely avoided asynchronous methods (e.g. prefixing with async) as 1) I don't fully understand it in this context and 2) it hasn't been necessary for the script thus far.
Where I'm having trouble, is that I'm looping through a set of files and attempting to call a method on each entry, but the method doesn't execute before the script exits.
Here's the primary method:
const migrateAll = (app, env, source) => {
const self = this;
fs.promises
.readFile(config, "utf8")
.then((contents) => {
self.config = JSON.parse(contents);
})
.then(() => {
const spaceId = self.config.applications[app].space_id;
fs.readdir(source, "utf8", (err, files) => {
if (err) throw err;
files.forEach((file) => {
console.log(chalk.yellow(`Migrating "${file}" to the "${env}" environment for "${app}"`))
migrate(file, env, app);
});
process.exit();
});
});
};
The call to migrate(file, env, app) doesn't appear to run at all. The contents of that function are:
const migrate = (space, env, migration) => {
exec(
`migrate ${migration} "${space}" ${env}`,
(error, stdout, stderr) => {
if (error) {
// A `switch` to handle errors.
}
process.exit();
}
if (stderr) {
console.log(`stderr: ${stderr}`);
process.exit();
}
success(stdout);
);
};
The rest of the script, in context, looks like this:
const parseFlags = () => {
process.argv.splice(2).forEach((arg) => {
let parts = arg.split("=");
switch (parts[0]) {
// parse flags into variables
}
});
if (all) {
migrateAll(app, env, source);
}
return { app, env, source };
};
const run = () => {
try {
intro();
checkSettings();
const { app, env, source } = parseFlags();
// continue on here if migrateAll doesn't get called
} catch (err) {
complain(err);
}
};
run();
So, with the right flags, we call migrateAll() which in turn calls the migrate function for each file we find that needs to migrate. Some things I've noticed/tried
The console.log inside of the forEach in migrateAll runs as expected
I've tried various combinations of await and async, .then(), promisify, etc, but it feels like I'm throwing things at the wall just to see what sticks to no avail.
A few things:
You're calling async functions (fs.promises.readFile, readdir and exec) from within synchronous contexts. So in your script execution you have basically this:
migrate() ---------------+
| |
execution complete |
| readFile()
exit |
parse()
|
readdir()
|
exec()
You're synchronous execution completes before you finish running the async stuff.
You seem to spawning off a bunch of child processes to run these modules, you should instead require and just run them in-process
exec is not a safe way to spawn a child process as you're passing the string directly into a shell. If I as a user can control any of those three arguments I can pop a remote shell in netcat.
Using shorthand migrate is not a safe way to call a child process as it resolves from the PATH environment variable. If I have access to the runtime environment I can make migrate point to whatever I want.
Don't call process.exit(). The exit code you pass lets the caller or operating system know whether something went wrong. On success call process.exit(0), on error use any integer that's greater than 0 and less than 256. You can assign a unique exit code to each error situation if you wish.
Try this
// migrate.js
const {spawn} = require('child_process');
module.exports = async (space,env,migration) => new Promise((resolve,reject)=>{
let stdout = '';
let stderr = '';
let args = [migration,space,env];
let cp = spawn('/absolute/path/to/migrate', args);
cp.on('error',reject);
cp.stdout.setEncoding('utf8').on('data',(d)=>stdout+=d);
cp.stderr.setEncoding('utf8').on('data',(d)=>stderr+=d);
cp.on('exit',(code,signal) => {
if(signal)
code = signal;
if(code != 0) {
console.error(stderr);
return reject(new Error(`migrate returned exit code ${code}`));
}
resolve(stdout);
});
}).then(success); // not sure what success does, but this is equivalent to what you had
// migrate-all.js
const fsp = require('fs').promises;
const migrate = require('./migrate');
module.exports = async (app,env,source) => {
let contents = await fsp.readFile(config,'utf8');
self.config = JSON.parse(contents);
const spaceId = self.config.applications[app].space_id;
let files = await fsp.readdir(source);
for(let i in files) { // avoid async calls in .forEach loops
let file = files[i];
console.log(chalk.yellow(`Migrating "${file}" to the "${env}" environment for "${app}"`))
await migrate(file, env, app);
}
}
// index.js
const migrateAll = require('./migrate-all');
const parseFlags = async () => {
process.argv.splice(2).forEach((arg) => {
let parts = arg.split("=");
switch (parts[0]) {
// parse flags into variables
}
});
if (all) {
await migrateAll(app, env, source);
}
return { app, env, source };
};
const run = async () => {
try {
intro();
checkSettings();
const { app, env, source } = await parseFlags();
// continue on here if migrateAll doesn't get called
} catch (err) {
complain(err);
}
};
run();
I am using node to execute a jar file that usually takes a CSV file as an input path.
I would like to try and circumvent writing the CSV file and pipe in the CSV as a string into the process if possible.
I have this working with execSync but I would prever to use exec wrapped with promisify
The problem is that exec does not have the input option like execSync so I can't pipe data into it. How do you get around this? Or is the best practice to wrap execSync in a Promise?
import {execSync} from 'child_process';
export const runJar = async (input: string, cwd: string) => {
const out = execSync(`java -jar model.jar`, {
cwd,
input,
})
return out.toString('utf-8');
};
Minimalistic example usage of a childs process stdio.
https://nodejs.org/dist/latest-v14.x/docs/api/child_process.html#child_process_child_process_exec_command_options_callback
const child_process = require("child_process");
const fs = require("fs");
// exec returns a child process instance
// https://nodejs.org/dist/latest-v14.x/docs/api/child_process.html#child_process_class_childprocess
const child = child_process.exec("cat");
// write to child process stdin
child.stdin.write("Hello World");
// to read/parse your csv file
//fs.createReadStream("./file.csv").pipe(child.stdin);
// listen on child process stdout
child.stdout.on("data", (chunk) => {
console.log(chunk);
child.kill();
});
To promisfy this, you can listen on the exit (status) on the child process and resolve or reject the promise based on the exit code:
child.on("close", (code) => {
if (code != 0) {
reject();
} else {
resolve();
}
});
Example given:
const readParseCSV = function (file = "./file.csv") {
return new Promise((resolve, reject) => {
const child = child_process.exec("java -jar model.jar");
fs.createReadStream(file).pipe(child.stdin);
let response = "";
// listen on child process stdout
child.stdout.on("data", (chunk) => {
response += chunk;
});
child.on("close", (code) => {
if (code != 0) {
reject();
} else {
resolve(response);
}
});
});
};
Im not sure if this works on windows the same way as on linux.
Say I have a loop that creates a child process per iteration, how does one create a timeout call back function per child, or maybe one function that can service all child processes?
var theSpawn = require("child_process").spawn;
for (var i=0; i<4;i++) {
theChild = spawn('ls', ['/']);
}
It's not clear exactly what you're trying to accomplish with the separate callbacks. The child process object that spawn() returns is an eventEmitter and it supports a number of events:
const spawn = require("child_process").spawn;
for (let i=0; i<4;i++) {
let child = spawn('ls', ['/']);
child.on('close', () => {
console.log(`child ${i} closed`);
});
}
If you're trying to create some sort of timeout for each child, you can do that in the loop:
const spawn = require("child_process").spawn;
for (let i=0; i<4;i++) {
const child = spawn('ls', ['/']);
const timer = setTimeout(() => {
console.log(`Hit timeout on child ${i} before it closed`);
}, 10 * 1000);
child.on('close', () => {
clearTimeout(timer);
console.log(`child ${i} closed`);
});
}
I've written a node script to manage deployment of a git repository to a AWS autoscaling group.
The script uses child_process.spawn() to automate git, to clone repositories, checkout tags etc.
It works fine if git can find appropriate credentials. However if credentials aren't automatically found, then the spawned process will attempt to prompt for credentials, and at that point will hang. Even Ctrl-C cannot exit. The whole shell session must be ended.
The spawn() call is wrapped in a function to return a Promise. My function looks like so...
const cp = require('child_process');
let spawn_promise = (command, args, options, stream_output) => {
return new Promise((resolve, reject) => {
console.log(chalk.cyan(`${command} [${args}]`));
let childProcess = cp.spawn(command, args, options);
let std_out = '';
let std_err = '';
childProcess.stdout.on('data', function (data) {
std_out += data.toString();
if (stream_output)
console.log(chalk.green(data.toString()));
});
childProcess.stderr.on('data', function (data) {
std_err += data.toString();
if (stream_output)
console.log(chalk.red(data.toString()));
});
childProcess.on('close', (code) => {
if (code === 0) {
console.log(chalk.blue(`exit_code = ${code}`));
return resolve(std_out);
}
else {
console.log(chalk.yellow(`exit_code = ${code}`));
return reject(std_err);
}
});
childProcess.on('error', (error) => {
std_err += error.toString();
if (stream_output)
console.log(chalk.red(error.toString()));
});
});
}
I call it like so...
return spawn_promise('git', ['fetch', '--all'], {env: process.env})
.then(() => {
...
It mostly works very well, and allows easily manipulation of output and errors etc.
I'm having trouble figuring out a nice way to to handle input though, if a spawned process needs it.
A temporary work-around for the problem is to add an environment variable to prevent git from prompting for credentials, and instead to throw an error if it can't find credentials in the users environment. However this isn't ideal. Ideally I would like to be able to gracefully handle standard input, and still be able to capture and process the output and errors as I'm currently doing.
I can fix the problem with input by doing this...
let childProcess = cp.spawn(command, args, { stdio: [process.stdin, process.stdout, process.stderr] });
This allows git to prompt for credentials correctly. However I then lose the ability to capture the command output.
What is the correct way to be able to handle this?
I should also mention, that the function also automates some relatively long running processes, to build AMI's etc. This is what the "stream_output" parameter is for. I want to be able to view the output from the command in real-time, rather than waiting until the process completes.
The child_process has stdin to handle the input and same can be used to enter the input when the child_process is running.
See below an example:
test.sh:
#!/bin/sh
echo "Please enter something:"
read ch
echo "Thanks"
When I run on this terminal:
shell-input $ ./test.sh
Please enter something:
something
Thanks
shell-input $
When I use your code to run this:
test.js:
const cp = require('child_process');
const chalk = require('chalk');
let spawn_promise = (command, args, options, stream_output) => {
return new Promise((resolve, reject) => {
console.log(chalk.cyan(`${command} [${args}]`));
let childProcess = cp.spawn(command, args, options);
let std_out = '';
let std_err = '';
childProcess.stdout.on('data', function (data) {
std_out += data.toString();
if (stream_output)
console.log(chalk.green(data.toString()));
});
childProcess.stderr.on('data', function (data) {
std_err += data.toString();
if (stream_output)
console.log(chalk.red(data.toString()));
});
childProcess.on('close', (code) => {
if (code === 0) {
console.log(chalk.blue(`exit_code = ${code}`));
return resolve(std_out);
}
else {
console.log(chalk.yellow(`exit_code = ${code}`));
return reject(std_err);
}
});
childProcess.on('error', (error) => {
std_err += error.toString();
if (stream_output)
console.log(chalk.red(error.toString()));
});
});
}
spawn_promise('./test.sh', { env: process.env})
.then(() => {
});
Output:
$ node test.js
./test.sh [[object Object]]
<stuck here>
I modify your code to include the following:
...
childProcess.stdout.on('data', function (data) {
if (data == "Please enter something:\n")
{
childProcess.stdin.write("something\n");
//childProcess.stdin.end(); // Call this to end the session
}
std_out += data.toString();
if (stream_output)
console.log(chalk.green(data.toString()));
});
...
Then I run again:
$ node test.js
./test.sh [[object Object]]
exit_code = 0
It works. Basically you need to find out when stdin is waiting for input. You can use data event on stdout for that and then write on stdin. If you don't have credentials to write, you can end the session by calling childProcess.stdin.end();
Is there an easy way to mock the Node.js child_process spawn function?
I have code like the following, and would like to test it in a unit test, without having to rely on the actual tool calls:
var output;
var spawn = require('child_process').spawn;
var command = spawn('foo', ['get']);
command.stdout.on('data', function (data) {
output = data;
});
command.stdout.on('end', function () {
if (output) {
callback(null, true);
}
else {
callback(null, false);
}
});
Is there a (proven and maintained) library that allows me to mock the spawn call and lets me specify the output of the mocked call?
I don't want to rely on the tool or OS to keep the tests simple and isolated. I want to be able to run the tests without having to set up complex test fixtures, which could mean a lot of work (including changing system configuration).
Is there an easy way to do this?
you can use sinon.stubs sinon stubs guide
// i like the sandbox, or you can use sinon itself
let sandbox = sinon.sandbox.create();
let spawnEvent = new events.EventEmitter();
spawnEvent.stdout = new events.EventEmitter();
sandbox.stub(child_process, 'spawn').returns(spawnEvent);
// and emit your event
spawnEvent.stdout.emit('data', 'hello world');
console.log(output) // hello world
Came across this and nwinkler's answer put me on the path. Below is a Mocha, Sinon and Typescript example that wraps the spawn in a promise, resolving if the exit code is a zero, and rejecting otherwise, It gathers up STDOUT/STDERR output, and lets you pipe text in through STDIN. Testing for a failure would be just a matter of testing for the exception.
function spawnAsPromise(cmd: string, args: ReadonlyArray<string> | undefined, options: child_process.SpawnOptions | undefined, input: string | undefined) {
return new Promise((resolve, reject) => {
// You could separate STDOUT and STDERR if your heart so desires...
let output: string = '';
const child = child_process.spawn(cmd, args, options);
child.stdout.on('data', (data) => {
output += data;
});
child.stderr.on('data', (data) => {
output += data;
});
child.on('close', (code) => {
(code === 0) ? resolve(output) : reject(output);
});
child.on('error', (err) => {
reject(err.toString());
});
if(input) {
child.stdin.write(input);
child.stdin.end();
}
});
}
// ...
describe("SpawnService", () => {
it("should run successfully", async() => {
const sandbox = sinon.createSandbox();
try {
const CMD = 'foo';
const ARGS = ['--bar'];
const OPTS = { cwd: '/var/fubar' };
const STDIN_TEXT = 'I typed this!';
const STDERR_TEXT = 'Some diag stuff...';
const STDOUT_TEXT = 'Some output stuff...';
const proc = <child_process.ChildProcess> new events.EventEmitter();
proc.stdin = new stream.Writable();
proc.stdout = <stream.Readable> new events.EventEmitter();
proc.stderr = <stream.Readable> new events.EventEmitter();
// Stub out child process, returning our fake child process
sandbox.stub(child_process, 'spawn')
.returns(proc)
.calledOnceWith(CMD, ARGS, OPTS);
// Stub our expectations with any text we are inputing,
// you can remove these two lines if not piping in data
sandbox.stub(proc.stdin, "write").calledOnceWith(STDIN_TEXT);
sandbox.stub(proc.stdin, "end").calledOnce = true;
// Launch your process here
const p = spawnAsPromise(CMD, ARGS, OPTS, STDIN_TEXT);
// Simulate your program's output
proc.stderr.emit('data', STDERR_TEXT);
proc.stdout.emit('data', STDOUT_TEXT);
// Exit your program, 0 = success, !0 = failure
proc.emit('close', 0);
// The close should get rid of the process
const results = await p;
assert.equal(results, STDERR_TEXT + STDOUT_TEXT);
} finally {
sandbox.restore();
}
});
});
I've found the mock-spawn library, which pretty much does what I want. It allows to mock the spawn call and provide expected results back to the calling test.
An example:
var mockSpawn = require('mock-spawn');
var mySpawn = mockSpawn();
require('child_process').spawn = mySpawn;
mySpawn.setDefault(mySpawn.simple(1 /* exit code */, 'hello world' /* stdout */));
More advanced examples can be found on the project page.
For anyone who still has problems with this particular problem and for some reason, the recommendations in other answers don't help, I was able to get it to work with proxyrequire (https://github.com/thlorenz/proxyquire) by replacing the real child_process spawn with an event emitter that I then used in my tests to mock the emission.
var stdout = new events.EventEmitter();
var stderr = new events.EventEmitter();
var spawn = new events.EventEmitter();
spawn.stderr = stderr;
spawn.stdout = stdout;
var child_process = {
spawn: () => spawn,
stdout,
stderr
};
// proxyrequire replaces the child_process require in the file pathToModule
var moduleToTest = proxyquire("./pathToModule/", {
'child_process': child_process
});
describe('Actual test', function () {
var response;
before(function (done) {
// your regular method call
moduleToTest.methodToTest()
.then(data => {
response = data;
done();
}).catch(err => {
response = err;
done();
});
// emit your expected response
child_process.stdout.emit("data", "the success message sent");
// you could easily use the below to test an error
// child_process.stderr.emit("data", "the error sent");
});
it('test your expectation', function () {
expect(response).to.equal("the success message or whatever your moduleToTest
resolves with");
});
});
Hope this helps...