How to create a video from image buffers using fluent-ffmpeg? - node.js

Question

I've been trying to create a slideshow from a series of images using nodejs + fluent-ffmpeg, however it is not working well or consistently. ffmpeg occasionally emits "Error: ffmpeg exited with code 1: pipe:0: Invalid data found when processing input", and in case an eventual video (mp4) is created it seems to be missing images/frames.
The process is as follows: images are loaded into memory, transformed resized to same dimensions using lwip, written sequentially into a passthrough stream, which is fed to ffmpeg as input.
Relevant code snippets:
var lwip = require('lwip');
var ffmpeg = require('fluent-ffmpeg');
var stream = require('stream');
var imagesStream = new stream.PassThrough();
...
image.batch()
.contain(options.video.width, options.video.height, 'lanczos')
.toBuffer(options.frames.format, {quality: 100}, (err, buffer) => {
if (err) {
throw ('error convering image to buffer. ' + err);
}
imagesStream.write(buffer, 'utf8');
resolve();
});
...
ffmpeg(imagesStream)
.inputOptions('-framerate 1/' + options.frames.secsPerImage)
.input(path.join(AUDIO_ROOT, options.audio.track))
.save(path.join(path.join(OUTPUT_FOLDER, `${options.video.output.prefix}${timestamp}.${options.video.output.format}`)))
.size(`${options.video.width}x${options.video.height}`)
.on('start', () => {
console.log('creating the clip now...')
})
.on('progress', (progress) => {
var progPercent = Math.round(100 * progress.frames / (numImages * options.frames.secsPerImage * 25));
progPercent = Math.min(progPercent, 100);
console.log(`processing: ${progPercent}% done`)
})
.on('stderr', (line) => {
console.error('ffmpeg error: ' + line);
})
.on('error', (error) => {
reject('ffmpeg transcoding error: ' + error);
})
.on('end', () => {
console.log('done!');
resolve(true);
})
.run();
And here is the output:
"C:\Program Files (x86)\JetBrains\WebStorm 2016.1.1\bin\runnerw.exe" "C:\Program Files\nodejs\node.exe" vm2.js
image count: 18
image count: 6
creating the clip now...
ffmpeg error: ffmpeg version N-80335-gcb46b78 Copyright (c) 2000-2016 the FFmpeg developers
ffmpeg error: built with gcc 5.4.0 (GCC)
ffmpeg error: configuration: --enable-gpl --enable-version3 --disable-w32threads --enable-nvenc --enable-avisynth --enable-bzlib --enable-fontconfig --enable-frei0r --enable-gnutls --enable-iconv --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libfreetype --enable-libgme --enable-libgsm --enable-libilbc --enable-libmodplug --enable-libmfx --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-libopus --enable-librtmp --enable-libschroedinger --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvo-amrwbenc --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxavs --enable-libxvid --enable-libzimg --enable-lzma --enable-decklink --enable-zlib
ffmpeg error: libavutil 55. 24.100 / 55. 24.100
ffmpeg error: libavcodec 57. 46.100 / 57. 46.100
ffmpeg error: libavformat 57. 38.100 / 57. 38.100
ffmpeg error: libavdevice 57. 0.101 / 57. 0.101
ffmpeg error: libavfilter 6. 46.101 / 6. 46.101
ffmpeg error: libswscale 4. 1.100 / 4. 1.100
ffmpeg error: libswresample 2. 1.100 / 2. 1.100
ffmpeg error: libpostproc 54. 0.100 / 54. 0.100
creating the clip now...
ffmpeg error: ffmpeg version N-80335-gcb46b78 Copyright (c) 2000-2016 the FFmpeg developers
ffmpeg error: built with gcc 5.4.0 (GCC)
ffmpeg error: configuration: --enable-gpl --enable-version3 --disable-w32threads --enable-nvenc --enable-avisynth --enable-bzlib --enable-fontconfig --enable-frei0r --enable-gnutls --enable-iconv --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libfreetype --enable-libgme --enable-libgsm --enable-libilbc --enable-libmodplug --enable-libmfx --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-libopus --enable-librtmp --enable-libschroedinger --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvo-amrwbenc --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxavs --enable-libxvid --enable-libzimg --enable-lzma --enable-decklink --enable-zlib
ffmpeg error: libavutil 55. 24.100 / 55. 24.100
ffmpeg error: libavcodec 57. 46.100 / 57. 46.100
ffmpeg error: libavformat 57. 38.100 / 57. 38.100
ffmpeg error: libavdevice 57. 0.101 / 57. 0.101
ffmpeg error: libavfilter 6. 46.101 / 6. 46.101
ffmpeg error: libswscale 4. 1.100 / 4. 1.100
ffmpeg error: libswresample 2. 1.100 / 2. 1.100
ffmpeg error: libpostproc 54. 0.100 / 54. 0.100
ffmpeg error: pipe:0: Invalid data found when processing input
ffmpeg error:
an error has occurred: ffmpeg transcoding error: Error: ffmpeg exited with code 1: pipe:0: Invalid data found when processing input
ffmpeg error: [jpeg_pipe # 0000000000308fe0] Format jpeg_pipe detected only with low score of 6, misdetection possible!
ffmpeg error: Input #0, jpeg_pipe, from 'pipe:0':
ffmpeg error: Duration: N/A, bitrate: N/A
ffmpeg error: Stream #0:0: Video: mjpeg, yuvj420p(pc, bt470bg/unknown/unknown), 1920x1080 [SAR 1:1 DAR 16:9], 0.33 tbr, 0.33 tbn, 0.33 tbc
ffmpeg error: [mp3 # 0000000002fa0720] Estimating duration from bitrate, this may be inaccurate
ffmpeg error: Input #1, mp3, from 'audio\avicii.mp3':
ffmpeg error: Metadata:
ffmpeg error: album : True
ffmpeg error: genre : House
ffmpeg error: copyright : ℗ 2013 Avicii Music AB, / PRMD under exclusive license to Universal Music AB
ffmpeg error: encoded_by : Oz
ffmpeg error: title : Wake Me Up
ffmpeg error: artist : Avicii
ffmpeg error: album_artist : Avicii
ffmpeg error: disc : 1/1
ffmpeg error: track : 1/12
ffmpeg error: TYER : 2013-09-13T07:00:00Z
ffmpeg error: Duration: 00:04:09.73, start: 0.000000, bitrate: 321 kb/s
ffmpeg error: Stream #1:0: Audio: mp3, 44100 Hz, stereo, s16p, 320 kb/s
ffmpeg error: Stream #1:1: Video: mjpeg, yuvj444p(pc, bt470bg/unknown/unknown), 600x600 [SAR 305:305 DAR 1:1], 90k tbr, 90k tbn, 90k tbc
ffmpeg error: Metadata:
ffmpeg error: comment : Cover (front)
ffmpeg error: No pixel format specified, yuvj420p for H.264 encoding chosen.
ffmpeg error: Use -pix_fmt yuv420p for compatibility with outdated media players.
ffmpeg error: [libx264 # 000000000030e860] using SAR=1/1
ffmpeg error: [libx264 # 000000000030e860] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX
ffmpeg error: [libx264 # 000000000030e860] profile High, level 4.0
ffmpeg error: [libx264 # 000000000030e860] 264 - core 148 r2694 3b70645 - H.264/MPEG-4 AVC codec - Copyleft 2003-2016 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=12 lookahead_threads=2 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
ffmpeg error: [mp4 # 0000000002ec6980] Using AVStream.codec to pass codec parameters to muxers is deprecated, use AVStream.codecpar instead.
ffmpeg error: Last message repeated 1 times
ffmpeg error: Output #0, mp4, to 'output\clip_2016-06-22_06-17-25.mp4':
ffmpeg error: Metadata:
ffmpeg error: encoder : Lavf57.38.100
ffmpeg error: Stream #0:0: Video: h264 (libx264) ([33][0][0][0] / 0x0021), yuvj420p(pc), 1920x1080 [SAR 1:1 DAR 16:9], q=-1--1, 25 fps, 12800 tbn, 25 tbc
ffmpeg error: Metadata:
ffmpeg error: encoder : Lavc57.46.100 libx264
ffmpeg error: Side data:
ffmpeg error: cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: -1
ffmpeg error: Stream #0:1: Audio: aac (LC) ([64][0][0][0] / 0x0040), 44100 Hz, stereo, fltp, 128 kb/s
ffmpeg error: Metadata:
ffmpeg error: encoder : Lavc57.46.100 aac
ffmpeg error: Stream mapping:
ffmpeg error: Stream #0:0 -> #0:0 (mjpeg (native) -> h264 (libx264))
ffmpeg error: Stream #1:0 -> #0:1 (mp3 (native) -> aac (native))
ffmpeg error: frame= 75 fps=0.0 q=28.0 size= 0kB time=00:00:00.64 bitrate= 0.6kbits/s dup=74 drop=0 speed=1.09x
processing: 17% done
ffmpeg error: frame= 150 fps= 97 q=28.0 size= 371kB time=00:00:03.64 bitrate= 835.6kbits/s dup=148 drop=0 speed=2.35x
processing: 33% done
processing: 33% done
ffmpeg error: frame= 150 fps= 73 q=28.0 size= 879kB time=00:00:07.36 bitrate= 977.8kbits/s dup=148 drop=0 speed= 3.6x
processing: 33% done
ffmpeg error: frame= 150 fps= 59 q=28.0 size= 952kB time=00:00:18.36 bitrate= 424.7kbits/s dup=148 drop=0 speed=7.21x
processing: 33% done
ffmpeg error: frame= 150 fps= 49 q=28.0 size= 1190kB time=00:00:32.99 bitrate= 295.3kbits/s dup=148 drop=0 speed=10.8x
ffmpeg error: frame= 150 fps= 42 q=28.0 size= 1409kB time=00:00:46.64 bitrate= 247.4kbits/s dup=148 drop=0 speed=13.1x
processing: 33% done
processing: 33% done
ffmpeg error: frame= 150 fps= 37 q=28.0 size= 1628kB time=00:01:00.30 bitrate= 221.1kbits/s dup=148 drop=0 speed=14.9x
processing: 33% done
ffmpeg error: frame= 150 fps= 33 q=28.0 size= 1878kB time=00:01:15.83 bitrate= 202.8kbits/s dup=148 drop=0 speed=16.7x
processing: 33% done
ffmpeg error: frame= 150 fps= 30 q=28.0 size= 2130kB time=00:01:31.64 bitrate= 190.4kbits/s dup=148 drop=0 speed=18.2x
processing: 33% done
ffmpeg error: frame= 150 fps= 27 q=28.0 size= 2375kB time=00:01:47.18 bitrate= 181.5kbits/s dup=148 drop=0 speed=19.3x
processing: 33% done
ffmpeg error: frame= 150 fps= 25 q=28.0 size= 2626kB time=00:02:03.15 bitrate= 174.7kbits/s dup=148 drop=0 speed=20.4x
processing: 33% done
ffmpeg error: frame= 150 fps= 23 q=28.0 size= 2832kB time=00:02:16.20 bitrate= 170.3kbits/s dup=148 drop=0 speed=20.8x
ffmpeg error: frame= 150 fps= 21 q=28.0 size= 3063kB time=00:02:30.34 bitrate= 166.9kbits/s dup=148 drop=0 speed=21.3x
processing: 33% done
processing: 33% done
ffmpeg error: frame= 150 fps= 20 q=28.0 size= 3298kB time=00:02:44.93 bitrate= 163.8kbits/s dup=148 drop=0 speed=21.8x
processing: 33% done
ffmpeg error: frame= 150 fps= 19 q=28.0 size= 3522kB time=00:02:58.93 bitrate= 161.3kbits/s dup=148 drop=0 speed=22.2x
processing: 33% done
ffmpeg error: frame= 150 fps= 18 q=28.0 size= 3792kB time=00:03:15.83 bitrate= 158.6kbits/s dup=148 drop=0 speed=22.9x
processing: 33% done
ffmpeg error: frame= 150 fps= 17 q=28.0 size= 4035kB time=00:03:31.11 bitrate= 156.6kbits/s dup=148 drop=0 speed=23.3x
processing: 33% done
ffmpeg error: frame= 150 fps= 16 q=28.0 size= 4294kB time=00:03:47.62 bitrate= 154.5kbits/s dup=148 drop=0 speed=23.8x
processing: 33% done
ffmpeg error: frame= 150 fps= 15 q=28.0 size= 4566kB time=00:04:04.87 bitrate= 152.7kbits/s dup=148 drop=0 speed=24.4x
processing: 33% done
ffmpeg error: frame= 150 fps= 14 q=-1.0 Lsize= 4851kB time=00:04:09.73 bitrate= 159.1kbits/s dup=148 drop=0 speed=23.6x
ffmpeg error: video:826kB audio:3978kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.967442%
ffmpeg error: [libx264 # 000000000030e860] frame I:2 Avg QP:14.53 size:414604
ffmpeg error: [libx264 # 000000000030e860] frame P:38 Avg QP:16.59 size: 222
ffmpeg error: [libx264 # 000000000030e860] frame B:110 Avg QP:12.67 size: 69
ffmpeg error: [libx264 # 000000000030e860] consecutive B-frames: 1.3% 2.7% 0.0% 96.0%
ffmpeg error: [libx264 # 000000000030e860] mb I I16..4: 25.5% 49.4% 25.1%
ffmpeg error: [libx264 # 000000000030e860] mb P I16..4: 0.0% 0.0% 0.0% P16..4: 0.7% 0.0% 0.0% 0.0% 0.0% skip:99.2%
ffmpeg error: [libx264 # 000000000030e860] mb B I16..4: 0.0% 0.0% 0.0% B16..8: 0.0% 0.0% 0.0% direct: 0.0% skip:100.0% L0: 1.2% L1:98.8% BI: 0.0%
ffmpeg error: [libx264 # 000000000030e860] 8x8 transform intra:49.4% inter:92.1%
ffmpeg error: [libx264 # 000000000030e860] coded y,uvDC,uvAC intra: 74.2% 73.7% 69.0% inter: 0.0% 0.2% 0.0%
ffmpeg error: [libx264 # 000000000030e860] i16 v,h,dc,p: 97% 0% 2% 1%
ffmpeg error: [libx264 # 000000000030e860] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 13% 19% 15% 7% 9% 7% 10% 7% 13%
ffmpeg error: [libx264 # 000000000030e860] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 12% 21% 8% 8% 11% 9% 12% 7% 13%
ffmpeg error: [libx264 # 000000000030e860] i8c dc,h,v,p: 55% 19% 16% 10%
ffmpeg error: [libx264 # 000000000030e860] Weighted P-Frames: Y:0.0% UV:0.0%
ffmpeg error: [libx264 # 000000000030e860] ref P L0: 95.4% 0.6% 3.1% 0.9%
ffmpeg error: [libx264 # 000000000030e860] ref B L1: 98.8% 1.2%
ffmpeg error: [libx264 # 000000000030e860] kb/s:1127.01
ffmpeg error: [aac # 0000000002ecc880] Qavg: 541.237
ffmpeg error:
done!

Answer 1

Apparently this is a bug in fluent-ffmpeg, so I reverted to using ffmpeg directly as a child process. Here is the final code for creating an mp4 from a stream of raw in-memory jpeg images:
var timestamp = moment().format(options.video.output.timestampFormat);
var framerate = '1/' + options.frames.secsPerImage;
var videosize = `${options.video.width}x${options.video.height}`;
var audiotrack = path.join(AUDIO_ROOT, options.audio.track);
var outputFilename = path.join(path.join(OUTPUT_FOLDER,
`${options.video.output.prefix}${timestamp}.${options.video.output.format}`));
var childProcess = spawn('bin/ffmpeg.exe', ['-y', '-f', 'image2pipe',
'-s', videosize,
'-framerate', framerate,
'-pix_fmt', 'yuv420p',
'-i', '-',
'-i', audiotrack,
'-vcodec', 'mpeg4',
'-shortest',
outputFilename
]);
childProcess.stdout.on('data', data => console.log(data.toString()));
childProcess.stderr.on('data', data => console.log(data.toString()));
childProcess.on('close', code => {
console.log(`done! (${code})`);
resolve();
});
imagesStream.pipe(childProcess.stdin);
And as for the stream:
var stream = require('stream');
var imagesStream = new stream.PassThrough();
...
// repeat for every image...
imagesStream.write(buffer, 'utf8');
...
// then finally end the stream
imagesStream.end();