I'm trying to create a media server using nodejs, and i have no problem streaming any formats but
Since web browsers cant play H265 codec videos i need to convert them to H264 while createReadStream creates a chunk so that i dont have to convert them completely before hand, just that chunk which is sent by the server to the browser.
const path = 'assets/yourfavmov.mkv';
const stat = fs.statSync(path);
const fileSize = stat.size;
const range = req.headers.range;
if (range) {
const rangeArray = range.replace(/bytes=/, "").split("-");
console.log(rangeArray)
const start = parseInt(rangeArray[0], 10);
const end = rangeArray[1] ? parseInt(rangeArray[1], 10) : fileSize-1;
const chunksize = (end-start) + 1;
const fileChunk = fs.createReadStream(path, {start, end});
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/x-matroska'
};
res.writeHead(206, head);
fileChunk.pipe(res);
} else {
res.end("wont let you stream");
}
I tried to convert the stream using ffmpeg-stream, like so
const converter = new Converter();
const input = converter.createInputStream({
f: "matroska,webm",
vcodec : "hevc"
})
const fileChunk = fs.createReadStream(path, {start, end});
fileChunk.pipe(input);
converter
.createOutputStream({ f: "matroska,webm", vcodec: "h264" })
.pipe(res);
But i have no idea what i did is correct or wrong, so no luck
so is there an way to do it right?
Thanks in advance.
"is there an way to do it right":
Not out of the box, no. What you are trying to do is called "Just in time" transcoding, and it is extremely complex. You must take into account container timestamps, audio priming samples, video sequence headers, and a few other things. ffmpeg does not do this for you. For all intents and purposes, what you want to do is not possible with open source tools today.
What you should do is do the encoding BEFORE the video is requested. (just in time encoding means you re-encode EVERY time someone watches the movie, which is really inefficient)
Netflix is not encoding your videos when you request them - Netflix has a library of videos encoded and ready to go when the request is made.
Related
I am using Amazon polly for text to speech.
Here is the code
static async _ttsUsingPolly(text, gender, destPath, speed) {
let params = {
'Text': 'Hi, my name is #anaptfox.',
'OutputFormat': 'pcm',
'VoiceId': 'Kimberly',
SampleRate: "8000"
}
const data = await this.Polly.synthesizeSpeech(params).promise();
if (data.AudioStream instanceof Buffer) {
console.log("buffer", data);
// Initiate the source
const bufferStream = new Stream.PassThrough();
// convert AudioStream into a readable stream
bufferStream.end(data.AudioStream);
// Pipe into Player
bufferStream.pipe(fs.createWriteStream(destPath));
}
}
this saves the file to .wav format. destPath is public\audio\abc\1212_1660649369899.wav.
But when i play this file it says
This file isn’t playable. That might be because the file type is unsupported, the file extension is incorrect, or the file is corrupt.
0xc00d36c4
what is the issue (if someone can explain)? and how can i fix this?
Update1
actually this generates the pcm format file, so i tried wav converter
var pcmData = fs.readFileSync(path.join(__dirname, './audios/audio_wav', fileName))
var wavData = wavConverter.encodeWav(pcmData, {
numChannels: 1,
sampleRate: 8000,
byteRate: 16
})
fs.writeFileSync(path.resolve(__dirname, './audios/audio_wav', './16k.wav'), wavData)
pcm generated file is of almost 67KB but converted wav file is of 1KB.
if i change pcm to mp3 in polly params it works.
Any help?
I directly passed the stream to wav-converter
._convertPcmToWav(data.AudioStream, fileName);
and
static _convertPcmToWav(stream, fileName) {
const wavData = wavConverter.encodeWav(stream, {
numChannels: 1,
sampleRate: 8000,
byteRate: 2
});
const wavFileName = path.parse(fileName).name;
fs.writeFileSync(path.resolve(__dirname, './audios/audio_wav', `./${wavFileName}.wav`), wavData)
}
now the file is generated correctly and playable.
I am currently trying to send a microphone stream to Watson STT service but for some reason, the Watson service is not receiving the stream (I'm guessing) so I get the error "Error: No speech detected for 30s".
Note that I have streamed a .wav file to Watson and I have also tested piping micInputStream to my local files so I know both are at least set up correctly. I am fairly new to NodeJS / javascript so I'm hoping the error might be obvious.
const fs = require('fs');
const mic = require('mic');
var SpeechToTextV1 = require('watson-developer-cloud/speech-to-text/v1');
var speechToText = new SpeechToTextV1({
iam_apikey: '{key_here}',
url: 'https://stream.watsonplatform.net/speech-to-text/api'
});
var params = {
content_type: 'audio/l16; rate=44100; channels=2',
interim_results: true
};
const micParams = {
rate: 44100,
channels: 2,
debug: false,
exitOnSilence: 6
}
const micInstance = mic(micParams);
const micInputStream = micInstance.getAudioStream();
micInstance.start();
console.log('Watson is listening, you may speak now.');
// Create the stream.
var recognizeStream = speechToText.recognizeUsingWebSocket(params);
// Pipe in the audio.
var textStream = micInputStream.pipe(recognizeStream).setEncoding('utf8');
textStream.on('data', user_speech_text => console.log('Watson hears:', user_speech_text));
textStream.on('error', e => console.log(`error: ${e}`));
textStream.on('close', e => console.log(`close: ${e}`));
Conclusion: In the end, I am not entirely sure what was wrong with the code. I'm guessing it had something to do with the mic package. I ended up scrapping the package and using "Node-audiorecorder" instead for my audio stream https://www.npmjs.com/package/node-audiorecorder
Note: This module requires you to install SoX and it must be available in your $PATH. http://sox.sourceforge.net/
Updated Code: For anyone wondering what my final code looks like here you go. Also a big shoutout to NikolayShmyrev for trying to help me with my code!
Sorry for the heavy comments but for new projects I like to make sure I know what every line is doing.
// Import module.
var AudioRecorder = require('node-audiorecorder');
var fs = require('fs');
var SpeechToTextV1 = require('watson-developer-cloud/speech-to-text/v1');
/******************************************************************************
* Configuring STT
*******************************************************************************/
var speechToText = new SpeechToTextV1({
iam_apikey: '{your watson key here}',
url: 'https://stream.watsonplatform.net/speech-to-text/api'
});
var recognizeStream = speechToText.recognizeUsingWebSocket({
content_type: 'audio/wav',
interim_results: true
});
/******************************************************************************
* Configuring the Recording
*******************************************************************************/
// Options is an optional parameter for the constructor call.
// If an option is not given the default value, as seen below, will be used.
const options = {
program: 'rec', // Which program to use, either `arecord`, `rec`, or `sox`.
device: null, // Recording device to use.
bits: 16, // Sample size. (only for `rec` and `sox`)
channels: 2, // Channel count.
encoding: 'signed-integer', // Encoding type. (only for `rec` and `sox`)
rate: 48000, // Sample rate.
type: 'wav', // Format type.
// Following options only available when using `rec` or `sox`.
silence: 6, // Duration of silence in seconds before it stops recording.
keepSilence: true // Keep the silence in the recording.
};
const logger = console;
/******************************************************************************
* Create Streams
*******************************************************************************/
// Create an instance.
let audioRecorder = new AudioRecorder(options, logger);
//create timeout (so after 10 seconds it stops feel free to remove this)
setTimeout(function() {
audioRecorder.stop();
}, 10000);
// This line is for saving the file locally as well (Strongly encouraged for testing)
const fileStream = fs.createWriteStream("test.wav", { encoding: 'binary' });
// Start stream to Watson STT Remove .pipe(process.stdout) if you dont want translation printed to console
audioRecorder.start().stream().pipe(recognizeStream).pipe(process.stdout);
//Create another stream to save locally
audioRecorder.stream().pipe(fileStream);
//Finally pipe translation to transcription file
recognizeStream.pipe(fs.createWriteStream('./transcription.txt'));
I am writing a small node.js program that will be able to play wav sound files on a chosen audio device.
The sound starts well but it is stoped before the end of the file.
Here is my code :
const fs = require("fs");
const wav = require("wav");
const portAudio = require("naudiodon");
const ao = new portAudio.AudioIO({
outOptions: {
channelCount: 2,
sampleFormat: portAudio.SampleFormat24Bit,
sampleRate: 44100,
}
});
const name = "myfile.wav";
const file = fs.createReadStream(`./sounds/${name}`);
const reader = new wav.Reader();
reader.on("format", () => {
reader.pipe(ao);
ao.start();
});
file.pipe(reader);
process.on("SIGINT", ao.quit);
When I modify the highWaterMark option of fs.createReadStream, it slightly change the cut position in the sound but it never goes until the end of it.
I always get a portAudio status - output underflow log error.
Thanks for any help !
I have been experiencing a similar error, and my solution was to manually write to the AudioIO stream instead of using the pipe commands.
So instead of
reader.on("format", () => {
reader.pipe(ao);
ao.start();
});
You would use
ao.start();
reader.on("data",chunk=>ao.write(chunk));
Output underflow is generally not an issue, but to avoid it I initialised a new instance of PortAudio before playing every file, however that is only applicable if you don't care about slight latency.
Can I create video clips from an mp4 video with node js streams? I am sure there are npms for this task, but is this something that can be done with just streams?
When I create a server, I can pipe a brief clip from the beginning of a video to an HttpResponse stream and pipe it to a file Stream with the following code: (It works!)
const fs = require('fs');
const http= require('http');
http.createServer(async (req, res) => {
// Creating clip from the beginning to 5% of the video
var { size } = fs.statSync('./Fun.mp4');
var start = 0
var end = .05*size;
var videoClip = fs.createReadStream('./Fun.mp4', { start, end })
var fileCopy = fs.createWriteStream('./Fun-Copy.mp4')
res.writeHead(200, {'Content-Type': 'video/mp4'})
videoClip.pipe(res)
videoClip.pipe(fileCopy)
}).listen(3000);
However, if I change the start position to the middle of the video, it doesn't work. I don't see the clip in the web browser, and Qucktime cannot play the copy that was produced.
// Attempting to create a clip from 50% to 60% of the video
var { size } = fs.statSync('./Fun.mp4');
var start = 0.5*size;
var end = 0.6*size;
It seems like mp4 is incomplete without the beginning. Is there a way that I can create clips from a larger video file with streams. Is there some meta elements or something, or a specific number of bytes that need to be copied from the beginning of an mp4 file?
Does node js handle audio the same way? Can I build audio clips from a larger file with node js streams?
There is not a specific number of bytes needed. MP4 uses an index like structure to organize the files. If you modify the file at all, the index (called the moov box) needs to be rewritten in its entirety.
I'm using node.js and through the socket.io library I receive chunks of data that are actually jpeg images. These images are frames of a realtime video captured from a remote webcam. I'm forced to stream the video as jpeg frames. I'm looking for a way to convert on the fly these jpeg images in a video file (mpeg 4 or mjpeg file). Does node have a library that can do this? I already took a look at the Node-fluent-FFMPEG library but the only examples given were about conversions of jpeg files to a video and not a conversion on the fly from a stream of jpeg images. Or alternatively, does ffmpeg for windows support a stream of jpeg images as input?
FFMPEG supports streams as inputs, as stated in the docs.
You can add any number of inputs to an Ffmpeg command. An input can
be [...] a readable stream
So for instance it supports using
ffmpeg().input(fs.createReadStream('/path/to/input3.avi'));
which creates a Readable stream from the file at '/path/to/input3.avi'.
I don't know anything about FFMPEG, but you may pull your messages coming from socket.io (messages may be a Buffer already) and wrap it with your own implementation of Readable stream.
I think you should look at videofy
var exec = require("child_process").exec;
var escape = require("shell-escape");
var debug = require("debug")("videofy");
var mkdirp = require("mkdirp");
var uid = require("uid2");
/*
* Expose videofy
*/
module.exports = videofy;
/**
* Convert `input` file to `output` video with the given `opts`:
*
* - `rate` frame rate [10]
* - `encoders` the video codec format, default is libx264
*
* #param {String} input
* #param {String} output
* #return
* #api public
*/
function videofy(input, output, opts, fn) {
if (!input) throw new Error('input filename required');
if (!output) throw new Error('output filename required');
var FORMAT = '-%05d';
// options
if ('function' == typeof opts) {
fn = opts;
opts = {};
} else {
opts = opts || {};
}
opts.rate = opts.rate || 10;
opts.codec = opts.codec || 'libx264';
// tmpfile(s)
var id = uid(10);
var dir = 'tmp/' + id;
var tmp = dir + '/tmp' + FORMAT + '.jpg';
function gc(err) {
debug('remove %s', dir);
exec('rm -fr ' + dir);
fn(err);
}
debug('mkdirp -p %s', dir);
mkdirp(dir, function(error) {
if (error) return fn(error);
// convert gif to tmp jpg
var cmd = ['convert', input, tmp];
cmd = escape(cmd);
debug('exec %s', cmd);
// covert jpg collection to video
exec(cmd, function(err) {
if (err) return gc(err);
var cmd = ['ffmpeg'];
cmd.push('-f', 'image2');
cmd.push('-r', String(opts.rate));
cmd.push('-i', tmp);
cmd.push('-c:v', String(opts.codec));
cmd.push(output);
cmd = escape(cmd);
debug("exec %s", cmd);
exec(cmd, gc);
});
});
}
Using require("child_process") you can use ffmpeg, or there are probably npm modules to help with this. ffmpeg will allow you to first take a list of jpegs and convert that to a video, second you can add a list (or just one) jpegs to the beginning or end of videos.