I have a NodeJS child process that on invocation calculates a unique pid (used by redis)
I and console logging it (e.g: console.log('PID: ' + pid))
In the parent I call spawn(command, arg, {stdio: ['pipe', 'pipe', 'pipe'], detached: true})
Some things I tried:
Close the child processes stdout and pass it to a function (that returns a promise) and then
return new Promise((resolve, reject) => {
inputStream.on('data', (i) => {
const isPidString = RegExp('/^PID:.+/g').test(i.toString('utf8'))
console.log('isPidString', isPidString)
if (isPidString) {
console.log('found string')
console.log('i', i.toString())
const pidValue = parseInt(i.toString('utf8').split(': ')[1])
console.log('pidVal', pidValue)
inputStream.destroy()
resolve(pidValue)
}
})
setTimeout(() => {
reject()
}, 5000);
});
Somehow it doesn't work. My question is: how do i listen to a stream (thats going to run nonstop) for just one specific console.log() that I'm concerned with.
Edit: Was able to figure this out
The code largely works, heres what was required:
Clone the child processes stdout (where we want to read data from)
read the string (convert to string using utf-8 encoding)
When the output matches a regex string you use string manipulators to extrac the value, destroy the stream and resolve.
Related
I am launching an external program (adb shell) via node's spawn method.
Then I am sending commands via childProc.stdin.write(input) and the shell returns stuff accordingly.
My data callback gets called each time there is a new data chunk available. All good!
But since the connection stays open, I don't know when the external program is finished sending data.
Is there a way to solve this? Isn't there some standard way of showing "that's the data you asked for, I am done returning"?
static async execCommand2(command, args, input) {
if(!this.adbShellProc || this.adbShellProc.killed){
this.adbShellProc = spawn(command, args)
this.adbShellProc.stdout.setEncoding('utf8')
}
const self = this
let promise = new Promise((resolve, reject) => {
log.debug('Shell.execCommand2:', command, 'args:', args)
log.debug('send input: ', input)
self.adbShellProc.stdin.write(input)
self.adbShellProc.stdin.write('\n')
self.adbShellProc.stdout.on('data', chunk => {
// resolve(chunk)
// PROBLEM: I can't resolve here, cause there might be more data coming in
})
self.adbShellProc.on('close', code => {
self.adbShellProc.stdin.end()
self.adbShellProc = null
})
})
return promise
}
I am sure your answer is in your question itself. You have added close event listener, so when process is ended, your control will be in close event(Just one modification is, close event will be of adbShellProc.stdout), so you can resolve/return data from there.
static async execCommand2(command, args, input) {
let finalOutcome = ''
if(!this.adbShellProc || this.adbShellProc.killed){
this.adbShellProc = spawn(command, args)
this.adbShellProc.stdout.setEncoding('utf8')
}
const self = this
let promise = new Promise((resolve, reject) => {
log.debug('Shell.execCommand2:', command, 'args:', args)
log.debug('send input: ', input)
self.adbShellProc.stdin.write(input)
self.adbShellProc.stdin.write('\n')
self.adbShellProc.stdout.on('data', chunk => {
// resolve(chunk)
finalOutcome += data
// PROBLEM: I can't resolve here, cause there might be more data coming in
})
self.adbShellProc.stdout.on('close', code => {
self.adbShellProc.stdin.end()
self.adbShellProc = null
resolve(finalOutcome)
})
})
return promise
}
Also, it has another event end of stdout,
ls.stdout.on('end', (d) =>{
// You can resolve from here also
})
You can get more details here Child Process
No. There's no way of knowing if the output is final unless an exit code is returned, this is the linux way.
Fortunately, adb shell gives you a way of running one off commands using the adb shell without having to start it in interactive mode.
Instead of starting the shell and then sending the commands as input, make the whole thing a single command. Example:
adb shell am start -a android.intent.action.VIEW
Then the close event will give you a clear indication that the command has finished so you can resolve and you will also get the return code, so you can check if the command has succeeded or not.
There's no real benefit in starting adb shell in interactive mode, because the real processing is not happening in adb but in the actual server, which is adbd.
https://developer.android.com/studio/command-line/adb#shellcommands
static async execCommand2(command, args, input) {
let finalOutcome = ''
//append the input to the arguments and don't use this, all your commands are one offs and do not require a state check
let adbShellProc = spawn(command, args+input)
adbShellProc.stdout.setEncoding('utf8')
let promise = new Promise((resolve, reject) => {
log.debug('Shell.execCommand2:', command, 'args:', args+input)
self.adbShellProc.stdout.on('data', chunk => {
finalOutcome += data
})
self.adbShellProc.stdout.on('close', code => {
//unix return code, 0 is good, not zero means error
if(code != 0){
reject(finalOutcome)
}
resolve(finalOutcome)
})
})
return promise
}
You can end your shell commands with ; echo some_marker and wait until some_marker appears in the stdout. (And you can also get the exit code with ; echo some_marker $?)
I'm using Next.js to pipe system calls. The code looks something like this (not exactly this, but this is simple enough to illustrate what I'm doing):
export async function find(file, searchTerm) {
const cat = execFile('cat', [file], {maxBuffer: 1024 * 1024});
const grep = execFile('grep', [searchTerm], {maxBuffer: 1024 * 1024});
cat.stdout.pipe(grep.stdin);
return new Promise((resolve, reject) => {
grep.stdout.on('data', async (d) => {
setTimeout(() => resolve(d.toString().trim()), 100)
});
});
}
Notice there are two processes:
cat
grep
cat.stdout is piped to grep.stdin, and when grep.stdout receives data, the whole function returns a Promise.
All fine and good. The problem is that if grep doesn't find searchTerm inside the results returned from cat, the callback for grep.stdout.on('data', ... isn't invoked and the whole chain just hangs.
In production I have an abstraction that lets me chain together an arbitrary number of processes (started with execFile as above).
Is there a way to detect if any process in the chain returns "nothing", and to just send "nothing" (e.g. the empty string) along the pipe?
Thanks.
You can use the on 'exit' event here.
The 'exit' event is emitted after the child process ends. If the
process exited, code is the final exit code of the process, otherwise
null. If the process terminated due to receipt of a signal, signal is
the string name of the signal, otherwise null. One of the two will
always be non-null.
So you can reject the promise when this event is occurred.
more info : https://nodejs.org/api/child_process.html#child_process_event_exit
You can call the process.exit() function in the callback, Like this:
const { execFile } = require('child_process');
async function find(file, searchTerm) {
const cat = execFile('cat', [file], {maxBuffer: 1024 * 1024});
const grep = execFile('grep', [searchTerm], {maxBuffer: 1024 * 1024}, (error, stdout, stderr) => {
//Because grep gives an empty string when no match
if(stdout ===''){
console.log('emptty')
process.exit()
}
});
cat.stdout.pipe(grep.stdin)
return new Promise((resolve, reject) => {
grep.stdout.on('data', async (d) => {
setTimeout(() => resolve(d.toString().trim()), 100)
});
})
.then(d => console.log('succes'))
}
What I ended up doing is this, for all processes in the chain, I added the following event listener:
p.on('close', (code) => {
if (code > 0) {
reject(`child process PID ${p.pid} exited with code ${code}`);
}
});
If the exit code of any of the processes isn't 0 (0 means no errors), reject the Promise.
I'm building a discord bot that wraps a terraria server in node.js so server users can restart the server and similar actions. I've managed to finish half the job, but I can't seem to create a command to execute commands on the terraria server. I've set it to write the command to the stdin of the child process and some basic debugging verifies that it does, but nothing apparently happens.
In the Node.js docs for child process stdin, it says "Note that if a child process waits to read all of its input, the child will not continue until this stream has been closed via end()." This seems likely to be the problem, as calling the end() function on it does actually send the command as expected. That said, it seems hard to believe that I'm unable to continuously send commands to stdin without having to close it.
Is this actually the problem, and if so what are my options for solving it? My code may be found below.
const discordjs = require("discord.js");
const child_process = require("child_process");
const tokens = require("./tokens");
const client = new discordjs.Client();
const terrariaServerPath = "C:\\Program Files (x86)\\Steam\\steamapps\\common\\Terraria\\TerrariaServer.exe"
const terrariaArgs = ['-port', '7777', "-maxplayers", "8", "-world", "test.wld"]
var child = child_process.spawn(terrariaServerPath, terrariaArgs);
client.on('ready', () => {
console.log(`Logged in as ${client.user.tag}!`);
});
client.on('disconnect', () => {
client.destroy();
});
client.on('message', msg => {
if (msg.channel.name === 'terraria') {
var msgSplit = msg.content.split(" ");
if (msgSplit[0] === "!restart") {
child.kill();
child = child_process.spawn(terrariaServerPath, terrariaArgs);
registerStdio();
msg.reply("restarting server")
}
if (msgSplit[0] === "!exec") {
msg.reply(msgSplit[1]);
child.stdin.write(msgSplit[1] + "\n");
child.stdin.end();
}
}
});
client.login(tokens.discord_token);
var registerStdio = function () {
child.stdout.on('data', (data) => {
console.log(`${data}`);
});
child.stderr.on('data', (data) => {
console.error(`${data}`);
});
}
registerStdio();
I was able to solve the problem by using the library node-pty. As near as I can tell, the problem was that the child process was not reading the stdin itself and I was unable to flush it. Node-pty creates a virtual terminal object which can be written to instead of stdin. This object does not buffer writes and so any input is immediately sent to the program.
I'd like to build a promise that spawns a child process using require('child_process').spawn. The process streams its output to stdout and its errors to stderr.
I would like the promise to:
reject(child.stderr stream (or its data)) if child.stderr emits any data.
resolve(child.stdout stream) only if no error is emitted.
I'm doing this because I want to chain the promise to:
a then that processes the child.stdout stream (upload the stream to an S3 bucket).
a catch that can process the child.stderr stream, allowing me to properly handle errors.
Is it feasible to combine promises and process streams like this ?
I was thinking of working around stderr but unsure about whats happening in between to stdout if a lot of data is coming into it and I don't process it fast enough.
As I see it, the issue is that you don't know whether you ever got data on stderr until the entire process is done as it could put data there at any time.
So, you have to wait for the entire process to be done before calling resolve() or reject(). And, if you then want the entire data to be sent to either one of those, you'd have to buffer them. You could call reject() as soon as you got data on stderr, but you aren't guaranteed to have all the data yet because it's a stream.
So, if you don't want to buffer, you're better off just letting the caller see the streams directly.
If you are OK with buffering the data, you can buffer it yourself like this:
Based on the spawn example in the node.js doc, you could add promise support to it like this:
const spawn = require('child_process').spawn;
function runIt(cmd, args) {
return new Promise(function(resolve, reject) {
const ls = spawn(cmd, args);
// Edit thomas.g: My child process generates binary data so I use buffers instead, see my comments inside the code
// Edit thomas.g: let stdoutData = new Buffer(0)
let stdoutData = "";
let stderrData= "";
ls.stdout.on('data', (data) => {
// Edit thomas.g: stdoutData = Buffer.concat([stdoutData, chunk]);
stdoutData += data;
});
ls.stderr.on('data', (data) => {
stderrData += data;
});
ls.on('close', (code) => {
if (stderrData){
reject(stderrData);
} else {
resolve(stdoutData);
}
});
ls.on('error', (err) => {
reject(err);
});
})
}
//usage
runIt('ls', ['-lh', '/usr']).then(function(stdoutData) {
// process stdout data here
}, function(err) {
// process stdError data here or error object (if some other type of error)
});
I am doing something that involves running a sequence of child_process.spawn() in order (to do some setup, then run the actual meaty command that the caller is interested in, then do some cleanup).
Something like:
doAllTheThings()
.then(function(exitStatus){
// all the things were done
// and we've returned the exitStatus of
// a command in the middle of a chain
});
Where doAllTheThings() is something like:
function doAllTheThings() {
runSetupCommand()
.then(function(){
return runInterestingCommand();
})
.then(function(exitStatus){
return runTearDownCommand(exitStatus); // pass exitStatus along to return to caller
});
}
Internally I'm using child_process.spawn(), which returns an EventEmitter and I'm effectively returning the result of the close event from runInterestingCommand() back to the caller.
Now I need to also send data events from stdout and stderr to the caller, which are also from EventEmitters. Is there a way to make this work with (Bluebird) Promises, or are they just getting in the way of EventEmitters that emit more than one event?
Ideally I'd like to be able to write:
doAllTheThings()
.on('stdout', function(data){
// process a chunk of received stdout data
})
.on('stderr', function(data){
// process a chunk of received stderr data
})
.then(function(exitStatus){
// all the things were done
// and we've returned the exitStatus of
// a command in the middle of a chain
});
The only way I can think to make my program work is to rewrite it to remove the promise chain and just use a raw EventEmitter inside something that wraps the setup/teardown, something like:
withTemporaryState(function(done){
var cmd = runInterestingCommand();
cmd.on('stdout', function(data){
// process a chunk of received stdout data
});
cmd.on('stderr', function(data){
// process a chunk of received stderr data
});
cmd.on('close', function(exitStatus){
// process the exitStatus
done();
});
});
But then since EventEmitters are so common throughout Node.js, I can't help but think I should be able to make them work in Promise chains. Any clues?
Actually, one of the reasons I want to keep using Bluebird, is because I want to use the Cancellation features to allow the running command to be cancelled from the outside.
There are two approaches, one provides the syntax you originally asked for, the other takes delegates.
function doAllTheThings(){
var com = runInterestingCommand();
var p = new Promise(function(resolve, reject){
com.on("close", resolve);
com.on("error", reject);
});
p.on = function(){ com.on.apply(com, arguments); return p; };
return p;
}
Which would let you use your desired syntax:
doAllTheThings()
.on('stdout', function(data){
// process a chunk of received stdout data
})
.on('stderr', function(data){
// process a chunk of received stderr data
})
.then(function(exitStatus){
// all the things were done
// and we've returned the exitStatus of
// a command in the middle of a chain
});
However, IMO this is somewhat misleading and it might be desirable to pass the delegates in:
function doAllTheThings(onData, onErr){
var com = runInterestingCommand();
var p = new Promise(function(resolve, reject){
com.on("close", resolve);
com.on("error", reject);
});
com.on("stdout", onData).on("strerr", onErr);
return p;
}
Which would let you do:
doAllTheThings(function(data){
// process a chunk of received stdout data
}, function(data){
// process a chunk of received stderr data
})
.then(function(exitStatus){
// all the things were done
// and we've returned the exitStatus of
// a command in the middle of a chain
});