I am using NodeJs to run JXA (JavaScript for Automation). Below is my code:
const spawn = require("child_process").spawn;
const StringToStream = require('string-to-stream');
const code = `
ObjC.import('stdlib');
console.log('inited');
`;
const child = spawn(
"osascript",
["-il", "JavaScript"],
{
maxBuffer: 1024 * 1024 * 1024
}
);
child.stdout.on('data', (data) => {
// Very strange, it didn't run to here
});
child.stderr.on('data', (data) => {
console.log(data.toString());
// run to hear. but the data be cut off
});
StringToStream(`
console.log('... Very-Long-String ...');
`).pipe(child.stdin, { end: false })
Very strange, stderr.on('data') be fired, and data be cut off. It seems that the output stream is length limited.
Does anyone know how to solve this problem ?
Related
My node program is crashed without any log.
I run the program below with node index.js.
const childProcess = require('child_process')
const util = require('util');
const exec = util.promisify(childProcess.exec);
console.time('total');
console.log("start");
const urlList = Array(500).fill("https://google.com");
const pList = urlList.map(function(url) {
return exec('curl --max-time 20 --connect-timeout 10 -iSs "' + url + '"', function (error, stdout, stderr) { });
});
Promise.all(pList).then(() => {
console.timeEnd('total');
}).catch((e) => {
console.log('error: ' + e);
});
I think it might be memory problem because of too many curl connections.
But how to figure out it?
Thank you in advance!
because util.promisify(childProcess.exec)() only receive string not string and function
So you must change it into
const pList = urlList.map(function(url) {
return exec('curl --max-time 20 --connect-timeout 10 -i "' + url + '"');
});
The real use-case is sending a huge string through SSH, but for simplicity, I'll demonstrate it with two processes. See the code below.
When I convert this huge stream to a Readable stream and pipe it to the child process, it sends very small chunks of data, just a few bytes (~80) each and as a result, the transfer rate is extremely slow.
If, as an alternative, I write directly to the child process (stdin.write), it's super fast and each chunk is the correct size - 8KB. However, this method doesn't support back-pressure, so with a huge string and a slow consumer, it gets overwhelmed.
My question is, how do I pipe the string to the child process but with the normal chunk size.
parent.js
const stream = require('stream');
const spawn = require('child_process').spawn;
const child = spawn('node', ['child.js'], {});
const strSize = 1000000;
const hugeStr = [...Array(strSize)].map(i=>(~~(Math.random()*36)).toString(36)).join('');
console.log('PARENT: hugeStr size: ', hugeStr.length);
const readable = stream.Readable.from(hugeStr);
readable.pipe(child.stdin);
// an alternative to the pipe, but with no support for back-pressure
// child.stdin.write(hugeStr);
// child.stdin.end();
child.js
const fs = require('fs');
const debug = (str) => fs.writeFileSync('debug.log', `\n${str}`, { flag: 'a' });
function getDataFromStdin() {
let data = '';
return new Promise((resolve, reject) => {
process.stdin.on('data', (chunk) => {
data += chunk.toString();
const size = chunk.byteLength;
debug(`DATA ${size}B. ${Math.floor(size / 1024)}KB`);
});
process.stdin.on('end', () => {
const size = data.length;
debug(`END TOTAL DATA ${size}B. ${Math.floor(size / 1024)}KB ${Math.floor(size / 1024 / 1024)}MB`);
resolve();
});
process.stderr.on('data', (dataErr) => debug(`child error ${dataErr.toString()}`));
});
}
getDataFromStdin()
.then(() => debug('CHILD: COMPLETED'))
.catch(err => debug(`CHILD ERR: ${err}`))
What i am trying to do is taking a video and diving it to frames and passing this frames to a Model to detect objects in each frame but the problem is the extraction process cost so much time and i don't need the frames on my disk.
fmpeg-stream offers stream capabilities. So there is no need to write to a file.
It is also possible to use directly ffmpeg and spawn a new child process. Its .stdout property is a readable stream. On the event data, the chunk can be read.
const fs = require("fs");
const tf = require("#tensorflow/tfjs-node")
const logStream = fs.createWriteStream('./logFile.log');
const spawnProcess = require('child_process').spawn,
ffmpeg = spawnProcess('ffmpeg', [
'-i', 'videfile.mp4',
'-vcodec', 'png',
'-f', 'rawvideo',
'-s', 'h*w', // size of one frame
'pipe:1'
]);
ffmpeg.stderr.pipe(logStream); // for debugging
let i = 0
ffmpeg.stdout.on('data', (data) => {
try {
console.log(tf.node.decodeImage(data).shape)
console.log(`${++i} frames read`)
// dispose all tensors
} catch(e) {
console.log(e)
}
})
ffmpeg.on('close', function (code) {
console.log('child process exited with code ' + code);
});
Decoding the image is in a try catch block to prevent error raised when the chunk does not match a frame.
A more robust code to prevent decoding chunks that do not correspond to images will be the following:
const { Transform } = require("stream")
class ExtractFrames extends Transform {
constructor(delimiter) {
super({ readableObjectMode: true })
this.delimiter = Buffer.from(delimiter, "hex")
this.buffer = Buffer.alloc(0)
}
_transform(data, enc, cb) {
// Add new data to buffer
this.buffer = Buffer.concat([this.buffer, data])
const start = this.buffer.indexOf(this.delimiter)
if (start < 0) return // there's no frame data at all
const end = this.buffer.indexOf(
this.delimiter,
start + this.delimiter.length,
)
if (end < 0) return // we haven't got the whole frame yet
this.push(this.buffer.slice(start, end)) // emit a frame
this.buffer = this.buffer.slice(end) // remove frame data from buffer
if (start > 0) console.error(`Discarded ${start} bytes of invalid data`)
cb()
}
_flush(callback) {
// push remaining buffer to readable stream
callback(null, this.buffer);
}
}
const fs = require("fs");
const tf = require("#tensorflow/tfjs-node")
const logStream = fs.createWriteStream('./logFile.log');
const spawnProcess = require('child_process').spawn,
ffmpeg = spawnProcess('ffmpeg', [
'-i', 'generique.mp4',
'-vcodec', 'mjpeg',
'-f', 'rawvideo',
'-s', '420x360', // size of one frame
'pipe:1'
]);
ffmpeg.stderr.pipe(logStream); // for debugging
let i = 0
ffmpeg.stdout
.pipe(new ExtractFrames("FFD8FF")).on('data', (data) => {
try {
console.log(tf.node.decodeImage(data).shape)
console.log(`${++i} frames read`)
// dispose all tensors
} catch(e) {
console.log(e)
}
})
ffmpeg.on('close', function (code) {
console.log('child process exited with code ' + code);
});
Though, the above code works, it will still fill up quickly the memory. Separating the frame extraction from the data processing itself will help.
async function* frames() {
let resolve;
let promise = new Promise(r => resolve = r);
let bool = true;
ls.stdout.pipe(new ExtractFrames("FFD8FF")).on('data', data => {
resolve(data);
promise = new Promise(r => resolve = r);
});
ls.on('close', function (code) {
bool = false
console.log('code')
});
while (bool) {
const data = await promise;
yield data;
}
}
(async() => {
// data processing
// possibly create tf.dataset for training
for await (const data of stream()) {
console.log(tf.node.decodeImage(data).shape)
console.log(data);
}
})()
I've written this cloud function which transcodes a MOV file to MP4 but after function runs I only see saved video 1.3kb or something but everything runs smoothly not sure what's happening.
I don't see any errors generated in console but I'm sure I'm doing something wrong in code, below is the function:
const storage = require('#google-cloud/storage')();
const ffmpegPath = require('#ffmpeg-installer/ffmpeg').path;
const ffmpeg = require('fluent-ffmpeg');
ffmpeg.setFfmpegPath(ffmpegPath);
exports.vsTranscodeVideo = (req, res) => {
const fileName = req.body.filename;
const userBucket = storage.bucket(req.body.vsuserbucket);
const uploadBucket = storage.bucket(req.body.vstempbucket);
const remoteWriteStream = userBucket.file(fileName.replace('.mov', '.mp4'))
.createWriteStream({
metadata: {
contentLanguage: 'en',
contentType: 'video/mp4'
}
});
const remoteReadStream = uploadBucket.file(fileName).createReadStream();
// Transcode
ffmpeg()
.input(remoteReadStream)
.outputOptions('-c:v copy') // Change these options to whatever suits your needs
.outputOptions('-c:a aac')
.outputOptions('-b:a 160k')
.outputOptions('-f mp4')
.outputOptions('-preset fast')
.outputOptions('-movflags frag_keyframe+empty_moov')
.outputFormat('mp4')
//https://github.com/fluent-ffmpeg/node-fluent-ffmpeg/issues/346#issuecomment-67299526 // end: true, emit end event when readable stream ends
.on('start', (cmdLine) => {
console.log('Started ffmpeg with command:', cmdLine);
})
.on('end', () => {
//uploadBucket.file(fileName).delete();
console.log('Successfully re-encoded video.');
res.status(200).send("success");
})
.on('error', (err, stdout, stderr) => {
console.error('stdout:', stdout);
console.error('stderr:', stderr);
res.status(200).send(err.message);
})
.pipe(remoteWriteStream, { end: true });
};
Is there something I'm doing wrong? Any help much appreciated. Thanks
I have vue (axios) making a get call to an express route which triggers a child_process of ffmpeg in an infinite loop. ffmpeg streams one file over udp , on close it re calls itself and streams another file.
I'd like to be able to kill this process from a button on a web page, but can't seem to work it out.
This is my express route code
router.get('/test', function(req, res) {
const childProcess = require('child_process');
const fs = require('fs')
const path = require('path')
//Grabs a random index between 0 and length
function randomIndex(length) {
return Math.floor(Math.random() * (length));
}
function Stream () {
const FILE_SRC = '/path/to/file'
//Read the directory and get the files
const dirs = fs.readdirSync(FILE_SRC)
.map(file => {
return path.join(FILE_SRC, file);
});
const srcs_dup = [];
const hashCheck = {}; //used to check if the file was already added to srcs_dup
var numberOfFiles = dirs.length - 1; //OR whatever # you want
console.log(numberOfFiles)
//While we haven't got the number of files we want. Loop.
while (srcs_dup.length < numberOfFiles) {
var fileIndex = randomIndex(dirs.length-1);
//Check if the file was already added to the array
if (hashCheck[fileIndex] == true) {
continue; //Already have that file. Skip it
}
//Add the file to the array and object
srcs_dup.push(dirs[fileIndex]);
hashCheck[fileIndex] = true;
}
var chosen = "'" + srcs_dup[0] + "'"
var call = "ffmpeg -re -i " + chosen + " -content_type audio/mpeg -f mp3 udp://224.1.2.3:1234"
const stop = childProcess.exec(call, { shell: true });
stop.stdout.on('data', function (data) {
console.log('stdout: ' + data.toString());
});
stop.stderr.on('data', (data) => {
console.log(`stderr: ${data}`);
});
stop.on('close', (code) => {
console.log ('child exited with code ' + code)
Stream();
});
stop.on('error', function(err) {
console.log('sh error' + err)
});
}