I'm running nodejs on a raspberry pi and I want to run a child process to spawn a webcam stream.
Outside of node my command is:
raspivid -n -mm matrix -w 320 -h 240 -fps 18 -g 100 -t 0 -b 5000000 -o - | ffmpeg -y -f h264 -i - -c:v copy -map 0:0 -f flv -rtmp_buffer 100 -rtmp_live live "rtmp://example.com/big/test"
With child_process I have to break each argument up
var args = ["-n", "-mm", "matrix", "-w", "320", "-h", "240", "-fps", "18", "-g", "100", "-t", "0", "-b", "5000000", "-o", "-", "|", "ffmpeg", "-y", "-f", "h264", "-i", "-", "-c:v", "copy", "-map", "0:0", "-f", "flv", "-rtmp_buffer", "100", "-rtmp_live", "live", "rtmp://example.com/big/test"];
camera.proc = child.spawn('raspivid', args);
However it chokes on the | character:
error, exit code 64
Invalid command line option (|)
How do I use this pipe character as an argument?
This has been answered in another question: Using two commands (using pipe |) with spawn
In summary, with child.spawn everything in args should be an argument of your 'raspivid' command. In your case, the pipe and everything after it are actually arguments for sh.
A workaround is to call child.spawn('sh', args) where args is:
var args = ['-c', <the entire command you want to run as a string>];
Related
Im running a bash command with subprocess.popen in python:
cmd = "bwa-mem2/bwa-mem2 mem -R \'#RG\\tID:2064-01\\tSM:2064-01\\tLB:2064-01\\tPL:ILLUMINA\\tPU:2064-01\' reference_genome/human_g1k_v37.fasta BHYHT7CCXY.RJ-1967-987-02.2_1.fastq BHYHT7CCXY.RJ-1967-987-02.2_2.fastq -t 14 | samtools view -bS -o dna_seq/aligned/2064-01/2064-01.6.bam -"
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
The problem is that I get returncode 0 even if the first command fails.
I have googled and found out about pipefail and it seems that this is what I should use.
However, I don't understand where to write it. I have tried:
"set -o pipefail && bwa-mem2/bwa-mem2 mem -R \'#RG\\tID:2064-01\\tSM:2064-01\\tLB:2064-01\\tPL:ILLUMINA\\tPU:2064-01\' reference_genome/human_g1k_v37.fasta BHYHT7CCXY.RJ-1967-987-02.2_1.fastq BHYHT7CCXY.RJ-1967-987-02.2_2.fastq -t 14 | samtools view -bS -o dna_seq/aligned/2064-01/2064-01.6.bam -"
which gives: /bin/sh: 1: set: Illegal option -o pipefail
any ideas how I should incorporate this?
Edit:
I'm not sure if it is correct to edit my answer when responding to an answer? there was not enough characters to respond in a comment:/
Anyway,
I tried your second approach without shell=True #Charles Duffy.
(cmd_1 and cmd_2 are equal to what you wrote in your solution)
This is the code I use:
try:
p1 = Popen(shlex.split(cmd_1), stdout=PIPE)
p2 = Popen(shlex.split(cmd_2), stdin=p1.stdout, stdout=PIPE, stderr=STDOUT, text=True)
p1.stdout.close()
output, error = p2.communicate()
p1.wait()
rc_1 = p1.poll()
rc_2 = p2.poll()
print("rc_1:", rc_1)
print("rc_2:", rc_2)
if rc_1 == 0 and rc_2 == 0:
self.log_to_file("DEBUG", "# Process ended with returncode = 0")
if text: self.log_to_file("INFO", f"{text} succesfully
else:
print("Raise exception")
raise Exception(f"stdout: {output} stderr: {error}")
except Exception as e:
print(f"Error: {e} in misc.run_command()")
self.log_to_file("ERROR", f"# Process ended with returncode != 0, {e}")
this is the result i get when deliberately causing an error by renaming one file:
[E::main_mem] failed to open file `/home/jonas/BASE/dna_seq/reads/2064-01/test_BHYHT7CCXY.RJ-1967-987-02.2_2.fastq.gz'.
free(): double free detected in tcache 2
rc_1: -6
rc_2: 0
Raise exception
Error: stdout: stderr: None in misc.run_command()
ERROR: # Process ended with returncode != 0, stdout: stderr: None
It seems to capture the faulty returncode.
But why is stdout empty and stderr= None?
How can I capture the output to have it logged to a logger both when the process is successful and when it fails?
First, With A Shell
Instead of letting shell=True specify sh by default, specify bash explicitly to ensure that pipefail is an available feature:
shell_script = r'''
set -o pipefail || exit
bwa-mem2/bwa-mem2 mem \
-R '#RG\tID:2064-01\tSM:2064-01\tLB:2064-01\tPL:ILLUMINA\tPU:2064-01' \
reference_genome/human_g1k_v37.fasta \
BHYHT7CCXY.RJ-1967-987-02.2_1.fastq \
BHYHT7CCXY.RJ-1967-987-02.2_2.fastq \
-t 14 \
| samtools view -bS \
-o dna_seq/aligned/2064-01/2064-01.6.bam -
'''
process = subprocess.Popen(["bash", "-c", shell_script],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
This works, but it's not the best available option.
Second, With No Shell At All
p1 = subprocess.Popen(
['bwa-mem2/bwa-mem2', 'mem',
'-R', r'#RG\tID:2064-01\tSM:2064-01\tLB:2064-01\tPL:ILLUMINA\tPU:2064-01',
'reference_genome/human_g1k_v37.fasta',
'BHYHT7CCXY.RJ-1967-987-02.2_1.fastq',
'BHYHT7CCXY.RJ-1967-987-02.2_2.fastq', '-t', '14'],
stdout=subprocess.PIPE)
p2 = subprocess.Popen(
['samtools', 'view', '-bS',
'-o', 'dna_seq/aligned/2064-01/2064-01.6.bam', '-'],
stdin=p1.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
p1.stdout.close()
output, _ = p2.communicate() # let p2 finish running
p1.wait() # ensure p1 has properly exited
print(f'bwa-mem2 exited with status {p1.returncode}')
print(f'samtools exited with status {p2.returncode}')
...which lets you check p1.returncode and p2.returncode separately.
This question already has answers here:
Python Subprocess: Unable to Escape Quotes
(2 answers)
Closed last year.
I need to quote part of the rsync line that subprocess.run uses that contains the ssh parameters, unfortunately nothing I have tried has worked so far.
Can someone please advise me on the correct way to quote the ssh parameters, so that it will run under rsync.
At first I had a list of lists that got passed to subprocess.run, that fails with:
Traceback (most recent call last):
File "./tmp.py", line 20, in <module>
process = subprocess.run(rsync_cmd, stderr=subprocess.PIPE)
File "/usr/lib/python3.6/subprocess.py", line 423, in run
with Popen(*popenargs, **kwargs) as process:
File "/usr/lib/python3.6/subprocess.py", line 729, in __init__
restore_signals, start_new_session)
File "/usr/lib/python3.6/subprocess.py", line 1295, in _execute_child
restore_signals, start_new_session, preexec_fn)
TypeError: expected str, bytes or os.PathLike object, not list
Flatten it to an ordinary list:
Unexpected remote arg: example.com:/var/log/maillog
rsync error: syntax or usage error (code 1) at main.c(1361) [sender=3.1.2]
Which makes sense, as part of the command line for rsync needs to be quoted.
So I try to quote it:
rsync: Failed to exec /usr/bin/ssh -F /home/rspencer/.ssh/config -o PreferredAuthentications=publickey -o StrictHostKeyChecking=accept-new -o TCPKeepAlive=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=24 -o ConnectTimeout=30 -o ExitOnForwardFailure=yes -o ControlMaster=autoask -o ControlPath=/run/user/1000/foo-ssh-master-%C -l root -p 234 -o Compression=yes: No such file or directory (2)
rsync error: error in IPC code (code 14) at pipe.c(85) [Receiver=3.1.2]
rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
rsync error: error in IPC code (code 14) at io.c(235) [Receiver=3.1.2]
Which is due, I expect, to it being a string instead of a list. Although I'm guessing and that does not make complete sense to me.
Summarized code of my last attempt:
#!/usr/bin/python3
import subprocess
ssh_args = [
"-F",
"/home/rspencer/.ssh/config",
"-o",
"PreferredAuthentications=publickey",
"-o",
"StrictHostKeyChecking=accept-new",
"-o",
"TCPKeepAlive=yes",
"-o",
"ServerAliveInterval=5",
"-o",
"ServerAliveCountMax=24",
"-o",
"ConnectTimeout=30",
"-o",
"ExitOnForwardFailure=yes",
"-o",
"ControlMaster=autoask",
"-o",
"ControlPath=/run/user/1000/foo-ssh-master-%C",
"-l",
"root",
"-p",
"234",
]
rsync_params = []
src = "example.com:/var/log/maillog"
dest = "."
# Build SSH command
ssh_cmd = ["/usr/bin/ssh"] + ssh_args
# Use basic compression
ssh_cmd.extend(["-o", "Compression=yes"])
ssh_cmd = " ".join(ssh_cmd)
ssh_cmd = f'"{ssh_cmd}"'
# Build rsync command
rsync_cmd = ["/usr/bin/rsync", "-vP", "-e", ssh_cmd] + rsync_params + [src, dest]
# Run rsync
process = subprocess.run(rsync_cmd, stderr=subprocess.PIPE)
if process.returncode != 0:
print(process.stderr.decode("UTF-8").strip())
What the correct command would look like on the command line:
/usr/bin/rsync -vP -e "/usr/bin/ssh -F /home/rspencer/.ssh/config -o \
PreferredAuthentications=publickey -o StrictHostKeyChecking=accept-new -o \
TCPKeepAlive=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=24 -o \
ConnectTimeout=30 -o ExitOnForwardFailure=yes -o ControlMaster=autoask \
-o ControlPath=/run/user/1000/foo-ssh-master-%C -l root -p 234 -o \
Compression=yes" example.com:/var/log/maillog .
Turns out the trick is to not try to quote it.
I removed the following line and it worked without further modification:
ssh_cmd = f'"{ssh_cmd}"'
I've read so much documentation and missed it until asking the question. Murphy.
Rereading the post "How not to quote argument in subprocess?" and finally understanding what Greg Hewgill was saying helped me. I blame lack of sleep.
"If you use quotes on the shell command line, then put the whole contents in one element of args (without the quotes). ..." - Greg Hewgill
This is a small python3 script reading off picam using OpenCV :
#picamStream.py
import sys, os
from picamera.array import PiRGBArray
from picamera import PiCamera
import time
import cv2
# initialize the camera and grab a reference to the raw camera capture
camera = PiCamera()
camera.resolution = (960, 540)
camera.framerate = 30
rawCapture = PiRGBArray(camera, size=(960, 540))
# allow the camera to warmup
time.sleep(0.1)
# capture frames from the camera
for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
image = frame.array
# ---------------------------------
# .
# Opencv image processing goes here
# .
# ---------------------------------
os.write(1, image.tostring())
# clear the stream in preparation for the next frame
rawCapture.truncate(0)
# end
And I am trying to pipe it to ffmpeg to Youtube stream
My understanding is that I need to reference below two commands to somehow come up with a new ffmpeg command.
Piping picam live video to ffmpeg for Youtube streaming.
raspivid -o - -t 0 -vf -hf -w 960 -h 540 -fps 25 -b 1000000 | ffmpeg -re -ar 44100 -ac 2 -acodec pcm_s16le -f s16le -ac 2 -i /dev/zero -f h264 -i - -vcodec copy -acodec aac -ab 128k -g 50 -strict experimental -f flv rtmp://a.rtmp.youtube.com/live2/[STREAMKEY]
Piping OPENCV raw video to ffmpeg for mp4 file.
python3 picamStream.py | ffmpeg -f rawvideo -pixel_format bgr24 -video_size 960x540 -framerate 30 -i - foo.mp4
So far I've had no luck. Can anyone help me with this?
This is the program I use in raspberry pi.
#main.py
import subprocess
import cv2
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
command = ['ffmpeg',
'-f', 'rawvideo',
'-pix_fmt', 'bgr24',
'-s','640x480',
'-i','-',
'-ar', '44100',
'-ac', '2',
'-acodec', 'pcm_s16le',
'-f', 's16le',
'-ac', '2',
'-i','/dev/zero',
'-acodec','aac',
'-ab','128k',
'-strict','experimental',
'-vcodec','h264',
'-pix_fmt','yuv420p',
'-g', '50',
'-vb','1000k',
'-profile:v', 'baseline',
'-preset', 'ultrafast',
'-r', '30',
'-f', 'flv',
'rtmp://a.rtmp.youtube.com/live2/[STREAMKEY]']
pipe = subprocess.Popen(command, stdin=subprocess.PIPE)
while True:
_, frame = cap.read()
pipe.stdin.write(frame.tostring())
pipe.kill()
cap.release()
Youtube needs an audio source, so use -i /dev/zero.
I hope it helps you.
for encoding a video in ffmpeg and stream to a server, i need to use the pipe "|" command to copy the video before re-encoding it and sending it to the server.
This command works perfectly in the shell :
./ffmpeg -f x11grab -s 640x480 -framerate 25 -i :0.0 -vcodec libx264 -framerate 25 -rtbufsize 2500k -s 640x480 -preset veryfast -pix_fmt yuv420p -crf 26 -force_key_frames 'expr:gte(t,n_forced*2) -minrate 850k -maxrate 850k -b:v 900k -bufsize 280k -f flv -
| ./ffmpeg -f flv -i - -c copy -f flv "rtmp://SERVER_ADRESS.twitch.tv/app/STREAM_KEY"
In the shell, i see the normal output of ffmpeg, wich contains multiple lines like this:
frame= 218 fps=0.0 q=-1.0 Lsize= 1860kB time=00:00:09.08 bitrate=1677.5kbits/s dup=1 drop=7 speed=11.2x
frame= 219 fps=0.0 q=-1.0 Lsize= 1860kB time=00:00:10.08 bitrate=1677.5kbits/s dup=1 drop=7 speed=11.2x
...
Now how to translate it to NODEJS with spawn ? If i do something like this :
var arguments = [
'-f', 'xgrab',
'-s', '640x480',
'-framerate', '25',
'-i', ':0.0',
'-vcodec', 'libx264',
'-framerate', '25'
'-rtbufsize', '2500k',
'-framerate', framerate,
'-s', '640x4',50
'-preset', 'veryfast',
'-pix_fmt', 'yuv420p',
'-crf', '26',
'-force_key_frames', 'expr:gte(t,n_forced*2)',
'-minrate', 850 +'k',
'-maxrate',850+'k',
'-b:v', 900+'k',
'-bufsize', 280+'k',
'-f', 'flv',
'-', '|',
'./ffmpeg', '-f','flv', '-i', '-',
'-c', 'copy',
'-f', 'flv', 'rtmp://SERVER_ADRESS.twitch.tv/app/STREAM_KEY'
]);
var childProcess = spawn(cmd, arguments);
childProcess.stdout.on('data', function(data){
console.log('stream: '+data.toString());
});
childProcess.stderr.on('data', function(data){
console.log('stream: '+data.toString());
});
I only get the output from the first part of the command, before the "|"
and the 2nd part never runs.
Also, i think something disastrous is happening in background because i get multiple instances of ffmpeg on my computer when i check the running processes.
The pipe is a shell construct, so you'll have to do something like:
spawn('/bin/sh', '-c', cmd_plus_arguments_and_pipes);
I'm trying to get a little ffmpeg converter-service up and running, made pretty good progress so far. But when it comes to spawning the actual ffmpeg process for conversion, i'm hitting a brick wall.
// options.ffmpegopts is an array containing format-specific parameters
var args = [ '-y', '"' + options.targetfile + '"' ];
args = options.ffmpegopts.concat(args);
var ffmpegProc = spawn('ffmpeg ', args);
ffmpegProc.stderr.on('data', function(data) {
console.log('stderr: ' + data);
});
When executing this code, i get the following console output:
stderr: execvp(): No such file or directory
I already checked different node versions (0.4.0, 0.4.2 and 0.5.0-pre) without any effect.
Another really strange behavior is the fact that i have to call spawn including a space ('ffmpeg ' instead of just 'ffmpeg'). If i omit this space, i get a different error (stderr: "/path/to/my/movie.mpeg": no such file or directory). When calling ffmpeg directly from the shell, the command sent to child_process.spawn() executes without any problems.
Any hints on that one? I already checked other projects who achieve the same (like node-imagemagick or ffmpeg-node, but the enlightment didn't hit me...
Update: strace() output
When running my application using strace -fF -o strace.log node server.js, i can grep the following process spawning calls:
execve("/usr/local/sbin/ffmpeg", ["ffmpeg", "-i", "\"/data/media_dev/test/ORG_mymovi"..., "-sameq", "-ab", "128k", "-ar", "44100", "-b", "512k", "-r", "25", "-s", "320x240", "-f", "flv", ...], [/* 20 vars */]) = -1 ENOENT (No such file or directory)
execve("/usr/local/bin/ffmpeg", ["ffmpeg", "-i", "\"/data/media_dev/test/ORG_mymovi"..., "-sameq", "-ab", "128k", "-ar", "44100", "-b", "512k", "-r", "25", "-s", "320x240", "-f", "flv", ...], [/* 20 vars */]) = -1 ENOENT (No such file or directory)
execve("/usr/sbin/ffmpeg", ["ffmpeg", "-i", "\"/data/media_dev/test/ORG_mymovi"..., "-sameq", "-ab", "128k", "-ar", "44100", "-b", "512k", "-r", "25", "-s", "320x240", "-f", "flv", ...], [/* 20 vars */]) = -1 ENOENT (No such file or directory)
execve("/usr/bin/ffmpeg", ["ffmpeg", "-i", "\"/data/media_dev/test/ORG_mymovi"..., "-sameq", "-ab", "128k", "-ar", "44100", "-b", "512k", "-r", "25", "-s", "320x240", "-f", "flv", ...], [/* 20 vars */]) = 0
After seeing that strangely escaped double quotes on the path, i tried to call ffmpeg without the quotes...worked like a charm. But the problem remains, i need to be able to work with spaces in my paths.
Any suggestions?
Update: Solution
Got it working with spaces, a simple inputfile.replace(' ', '\ ') was enough.
I'd wager money that the space at the end of "ffmpeg " is the cause of the current problem. A quick little C program will show that:
#include <unistd.h>
int main(int argc, char *argv[]) {
execvp(argv[1], &argv[1]);
perror(argv[0]);
}
gives the following output:
$ ./execvp "ffmpeg"
FFmpeg version 0.6-4:0.6-2ubuntu6, Copyright (c) 2000-2010 the FFmpeg developers
...
$ ./execvp "ffmpeg "
./execvp: No such file or directory
$
I suggest removing the space again, and re-run under strace(1) -fF. Look for the command that is actually executed, and look to see if the error message about /path/to/my/movie.mpeg is coming from ffmpeg or from node.js.