I'm creating a node application that I want to run from the command line. As input, it should take in a .txt or .json file, perform some operations on the data, and return something to stdout. However, I'm not able to figure out how to read a file from stdin. This is what I have right now. I copied this from the nodeJS documentation.
process.stdin.on('readable', () => {
let chunk;
// Use a loop to make sure we read all available data.
while ((chunk = process.stdin.read()) !== null) {
process.stdout.write(`data: ${chunk}`);
}
});
process.stdin.on('end', () => {
process.stdout.write('end');
});
If I run this program from the command line, I can type some stuff into stdin and see it returned in stdout. However, if I run
node example.js < example.json from the command line, I get the error
stdin is not a tty. I understand that piping a file means it is not reading from a tty, but what part of my code is requiring it to read from a tty?
What can I do to read in a file from stdin?
Thanks in advance!
Take a look at https://gist.github.com/kristopherjohnson/5065599
If you prefer a promise based approach here is a refactored version of that code, hope it helps
function readJsonFromStdin() {
let stdin = process.stdin
let inputChunks = []
stdin.resume()
stdin.setEncoding('utf8')
stdin.on('data', function (chunk) {
inputChunks.push(chunk);
});
return new Promise((resolve, reject) => {
stdin.on('end', function () {
let inputJSON = inputChunks.join('')
resolve(JSON.parse(inputJSON))
})
stdin.on('error', function () {
reject(Error('error during read'))
})
stdin.on('timeout', function () {
reject(Error('timout during read'))
})
})
}
async function main() {
let json = await readJsonFromStdin();
console.log(json)
}
main()
Related
I have a Node JS CLI script that automates some migrations to a third-party service. I've largely avoided asynchronous methods (e.g. prefixing with async) as 1) I don't fully understand it in this context and 2) it hasn't been necessary for the script thus far.
Where I'm having trouble, is that I'm looping through a set of files and attempting to call a method on each entry, but the method doesn't execute before the script exits.
Here's the primary method:
const migrateAll = (app, env, source) => {
const self = this;
fs.promises
.readFile(config, "utf8")
.then((contents) => {
self.config = JSON.parse(contents);
})
.then(() => {
const spaceId = self.config.applications[app].space_id;
fs.readdir(source, "utf8", (err, files) => {
if (err) throw err;
files.forEach((file) => {
console.log(chalk.yellow(`Migrating "${file}" to the "${env}" environment for "${app}"`))
migrate(file, env, app);
});
process.exit();
});
});
};
The call to migrate(file, env, app) doesn't appear to run at all. The contents of that function are:
const migrate = (space, env, migration) => {
exec(
`migrate ${migration} "${space}" ${env}`,
(error, stdout, stderr) => {
if (error) {
// A `switch` to handle errors.
}
process.exit();
}
if (stderr) {
console.log(`stderr: ${stderr}`);
process.exit();
}
success(stdout);
);
};
The rest of the script, in context, looks like this:
const parseFlags = () => {
process.argv.splice(2).forEach((arg) => {
let parts = arg.split("=");
switch (parts[0]) {
// parse flags into variables
}
});
if (all) {
migrateAll(app, env, source);
}
return { app, env, source };
};
const run = () => {
try {
intro();
checkSettings();
const { app, env, source } = parseFlags();
// continue on here if migrateAll doesn't get called
} catch (err) {
complain(err);
}
};
run();
So, with the right flags, we call migrateAll() which in turn calls the migrate function for each file we find that needs to migrate. Some things I've noticed/tried
The console.log inside of the forEach in migrateAll runs as expected
I've tried various combinations of await and async, .then(), promisify, etc, but it feels like I'm throwing things at the wall just to see what sticks to no avail.
A few things:
You're calling async functions (fs.promises.readFile, readdir and exec) from within synchronous contexts. So in your script execution you have basically this:
migrate() ---------------+
| |
execution complete |
| readFile()
exit |
parse()
|
readdir()
|
exec()
You're synchronous execution completes before you finish running the async stuff.
You seem to spawning off a bunch of child processes to run these modules, you should instead require and just run them in-process
exec is not a safe way to spawn a child process as you're passing the string directly into a shell. If I as a user can control any of those three arguments I can pop a remote shell in netcat.
Using shorthand migrate is not a safe way to call a child process as it resolves from the PATH environment variable. If I have access to the runtime environment I can make migrate point to whatever I want.
Don't call process.exit(). The exit code you pass lets the caller or operating system know whether something went wrong. On success call process.exit(0), on error use any integer that's greater than 0 and less than 256. You can assign a unique exit code to each error situation if you wish.
Try this
// migrate.js
const {spawn} = require('child_process');
module.exports = async (space,env,migration) => new Promise((resolve,reject)=>{
let stdout = '';
let stderr = '';
let args = [migration,space,env];
let cp = spawn('/absolute/path/to/migrate', args);
cp.on('error',reject);
cp.stdout.setEncoding('utf8').on('data',(d)=>stdout+=d);
cp.stderr.setEncoding('utf8').on('data',(d)=>stderr+=d);
cp.on('exit',(code,signal) => {
if(signal)
code = signal;
if(code != 0) {
console.error(stderr);
return reject(new Error(`migrate returned exit code ${code}`));
}
resolve(stdout);
});
}).then(success); // not sure what success does, but this is equivalent to what you had
// migrate-all.js
const fsp = require('fs').promises;
const migrate = require('./migrate');
module.exports = async (app,env,source) => {
let contents = await fsp.readFile(config,'utf8');
self.config = JSON.parse(contents);
const spaceId = self.config.applications[app].space_id;
let files = await fsp.readdir(source);
for(let i in files) { // avoid async calls in .forEach loops
let file = files[i];
console.log(chalk.yellow(`Migrating "${file}" to the "${env}" environment for "${app}"`))
await migrate(file, env, app);
}
}
// index.js
const migrateAll = require('./migrate-all');
const parseFlags = async () => {
process.argv.splice(2).forEach((arg) => {
let parts = arg.split("=");
switch (parts[0]) {
// parse flags into variables
}
});
if (all) {
await migrateAll(app, env, source);
}
return { app, env, source };
};
const run = async () => {
try {
intro();
checkSettings();
const { app, env, source } = await parseFlags();
// continue on here if migrateAll doesn't get called
} catch (err) {
complain(err);
}
};
run();
I am using node to execute a jar file that usually takes a CSV file as an input path.
I would like to try and circumvent writing the CSV file and pipe in the CSV as a string into the process if possible.
I have this working with execSync but I would prever to use exec wrapped with promisify
The problem is that exec does not have the input option like execSync so I can't pipe data into it. How do you get around this? Or is the best practice to wrap execSync in a Promise?
import {execSync} from 'child_process';
export const runJar = async (input: string, cwd: string) => {
const out = execSync(`java -jar model.jar`, {
cwd,
input,
})
return out.toString('utf-8');
};
Minimalistic example usage of a childs process stdio.
https://nodejs.org/dist/latest-v14.x/docs/api/child_process.html#child_process_child_process_exec_command_options_callback
const child_process = require("child_process");
const fs = require("fs");
// exec returns a child process instance
// https://nodejs.org/dist/latest-v14.x/docs/api/child_process.html#child_process_class_childprocess
const child = child_process.exec("cat");
// write to child process stdin
child.stdin.write("Hello World");
// to read/parse your csv file
//fs.createReadStream("./file.csv").pipe(child.stdin);
// listen on child process stdout
child.stdout.on("data", (chunk) => {
console.log(chunk);
child.kill();
});
To promisfy this, you can listen on the exit (status) on the child process and resolve or reject the promise based on the exit code:
child.on("close", (code) => {
if (code != 0) {
reject();
} else {
resolve();
}
});
Example given:
const readParseCSV = function (file = "./file.csv") {
return new Promise((resolve, reject) => {
const child = child_process.exec("java -jar model.jar");
fs.createReadStream(file).pipe(child.stdin);
let response = "";
// listen on child process stdout
child.stdout.on("data", (chunk) => {
response += chunk;
});
child.on("close", (code) => {
if (code != 0) {
reject();
} else {
resolve(response);
}
});
});
};
Im not sure if this works on windows the same way as on linux.
I am trying to access what a function returns in node.js
I have the following function:
function getImg(callback) {
https.get('https://api.nasa.gov/planetary/apod?api_key=api-key', response => {
let data = "";
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
let img = JSON.parse(data).hdurl;
callback(null, img);
})
}).end();
}
let image = getImg(function(err, image) {
console.log(image);
})
res.render('index', {
indexCSS: true,
image
})
It can log it to the console correctly, but if I want to access the value of the variable like I do in the last line of my code or if I console.log(image) I get undefined.
What have I done wrong. How can I access what the function produces?
It is a callback style function which wouldn't return any thing. Better to convert it to promise sttyle and use async/await to get the value in a variable
function getImg() {
return new Promise((resolve, reject) => {
https.get('https://api.nasa.gov/planetary/apod?api_key=api-key', response => {
let data = "";
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
let img = JSON.parse(data).hdurl;
resolve(img);
})
}).end();
});
}
(async() => {
let image = await getImg();
res.render('index', {
indexCSS: true,
image
});
})();
You can't really store the return value of your function like that. Unfortunately JS is non-blocking, so the code will continue to execute past it, before it has a chance to return that value from the https request. I am not sure exactly when you call this function, but you could call res.render in the callbacks response after calling getImg() without assigning its value to something. You can use promises, otherwise it's better to handle the response you need when it is returned from the callback. That would just be a simple call like:
getImg(function(err, image) {
res.render('index', {
indexCSS: true,
image
});
})
Within whatever route is calling this function. You just cannot assign any kind of return value from a callback to a variable (really not recommended at least) in the normal way.
I've written a node script to manage deployment of a git repository to a AWS autoscaling group.
The script uses child_process.spawn() to automate git, to clone repositories, checkout tags etc.
It works fine if git can find appropriate credentials. However if credentials aren't automatically found, then the spawned process will attempt to prompt for credentials, and at that point will hang. Even Ctrl-C cannot exit. The whole shell session must be ended.
The spawn() call is wrapped in a function to return a Promise. My function looks like so...
const cp = require('child_process');
let spawn_promise = (command, args, options, stream_output) => {
return new Promise((resolve, reject) => {
console.log(chalk.cyan(`${command} [${args}]`));
let childProcess = cp.spawn(command, args, options);
let std_out = '';
let std_err = '';
childProcess.stdout.on('data', function (data) {
std_out += data.toString();
if (stream_output)
console.log(chalk.green(data.toString()));
});
childProcess.stderr.on('data', function (data) {
std_err += data.toString();
if (stream_output)
console.log(chalk.red(data.toString()));
});
childProcess.on('close', (code) => {
if (code === 0) {
console.log(chalk.blue(`exit_code = ${code}`));
return resolve(std_out);
}
else {
console.log(chalk.yellow(`exit_code = ${code}`));
return reject(std_err);
}
});
childProcess.on('error', (error) => {
std_err += error.toString();
if (stream_output)
console.log(chalk.red(error.toString()));
});
});
}
I call it like so...
return spawn_promise('git', ['fetch', '--all'], {env: process.env})
.then(() => {
...
It mostly works very well, and allows easily manipulation of output and errors etc.
I'm having trouble figuring out a nice way to to handle input though, if a spawned process needs it.
A temporary work-around for the problem is to add an environment variable to prevent git from prompting for credentials, and instead to throw an error if it can't find credentials in the users environment. However this isn't ideal. Ideally I would like to be able to gracefully handle standard input, and still be able to capture and process the output and errors as I'm currently doing.
I can fix the problem with input by doing this...
let childProcess = cp.spawn(command, args, { stdio: [process.stdin, process.stdout, process.stderr] });
This allows git to prompt for credentials correctly. However I then lose the ability to capture the command output.
What is the correct way to be able to handle this?
I should also mention, that the function also automates some relatively long running processes, to build AMI's etc. This is what the "stream_output" parameter is for. I want to be able to view the output from the command in real-time, rather than waiting until the process completes.
The child_process has stdin to handle the input and same can be used to enter the input when the child_process is running.
See below an example:
test.sh:
#!/bin/sh
echo "Please enter something:"
read ch
echo "Thanks"
When I run on this terminal:
shell-input $ ./test.sh
Please enter something:
something
Thanks
shell-input $
When I use your code to run this:
test.js:
const cp = require('child_process');
const chalk = require('chalk');
let spawn_promise = (command, args, options, stream_output) => {
return new Promise((resolve, reject) => {
console.log(chalk.cyan(`${command} [${args}]`));
let childProcess = cp.spawn(command, args, options);
let std_out = '';
let std_err = '';
childProcess.stdout.on('data', function (data) {
std_out += data.toString();
if (stream_output)
console.log(chalk.green(data.toString()));
});
childProcess.stderr.on('data', function (data) {
std_err += data.toString();
if (stream_output)
console.log(chalk.red(data.toString()));
});
childProcess.on('close', (code) => {
if (code === 0) {
console.log(chalk.blue(`exit_code = ${code}`));
return resolve(std_out);
}
else {
console.log(chalk.yellow(`exit_code = ${code}`));
return reject(std_err);
}
});
childProcess.on('error', (error) => {
std_err += error.toString();
if (stream_output)
console.log(chalk.red(error.toString()));
});
});
}
spawn_promise('./test.sh', { env: process.env})
.then(() => {
});
Output:
$ node test.js
./test.sh [[object Object]]
<stuck here>
I modify your code to include the following:
...
childProcess.stdout.on('data', function (data) {
if (data == "Please enter something:\n")
{
childProcess.stdin.write("something\n");
//childProcess.stdin.end(); // Call this to end the session
}
std_out += data.toString();
if (stream_output)
console.log(chalk.green(data.toString()));
});
...
Then I run again:
$ node test.js
./test.sh [[object Object]]
exit_code = 0
It works. Basically you need to find out when stdin is waiting for input. You can use data event on stdout for that and then write on stdin. If you don't have credentials to write, you can end the session by calling childProcess.stdin.end();
Ok, now I understand that this is "asynchronous" but can I assume that the input of a file passed by cat from a pipe will be in the same order it is read in ( or is it more like an unlocked mess that i can cross my figures will be in the correct order )?
Or is this going to be in order because the pipe is sync and the reads are buffered in order
Am I making any sense here ? Is there a chance a file 1 -> inifinity will be out of order ?
var input = ''
process.stdin.setEncoding('utf8')
process.stdin.on('data',function(chunk){
input+=chunk
})
process.stdin.on('end',function(){
run()
})
function run(){
console.log(input)
}
Here's a 'synchronous' way of reading the stdin data by using async/await:
async function readStdinSync() {
return new Promise(resolve => {
let data = ''
process.stdin.setEncoding('utf8')
process.stdin.resume()
const t = setTimeout(() => {
process.stdin.pause()
resolve(data)
}, 1e3)
process.stdin.on('readable', () => {
let chunk
while ((chunk = process.stdin.read())) {
data += chunk
}
}).on('end', () => {
clearTimeout(t)
resolve(data)
})
})
}
async function main() {
const input = await readStdinSync()
process.stdout.write(input) // "hello world"
process.exit(0)
}
main()
Note that you have to set a timeout in case there is no stdin data, otherwise it'll hang.
Example usage:
$ echo 'hello world' | node stdin.js
hello world