Live streaming using FFMPEG to web audio api - node.js

I am trying to stream audio using node.js + ffmpeg to browsers connected in LAN only using web audio api.
Not using element because it's adding it's own buffer of 8 to 10 secs and I want to get maximum high latency possible (around 1 to 2 sec max).
Audio plays successfully but audio is very choppy and noisy.
Here is my node.js (server side) file:
var ws = require('websocket.io'),
server = ws.listen(3000);
var child_process = require("child_process");
var i = 0;
server.on('connection', function (socket)
{
console.log('New client connected');
var ffmpeg = child_process.spawn("ffmpeg",[
"-re","-i",
"A.mp3","-f",
"f32le",
"pipe:1" // Output to STDOUT
]);
ffmpeg.stdout.on('data', function(data)
{
var buff = new Buffer(data);
socket.send(buff.toString('base64'));
});
});
And here is my HTML:
var audioBuffer = null;
var context = null;
window.addEventListener('load', init, false);
function init() {
try {
context = new webkitAudioContext();
} catch(e) {
alert('Web Audio API is not supported in this browser');
}
}
var ws = new WebSocket("ws://localhost:3000/");
ws.onmessage = function(message)
{
var d1 = base64DecToArr(message.data).buffer;
var d2 = new DataView(d1);
var data = new Float32Array(d2.byteLength / Float32Array.BYTES_PER_ELEMENT);
for (var jj = 0; jj < data.length; ++jj)
{
data[jj] = d2.getFloat32(jj * Float32Array.BYTES_PER_ELEMENT, true);
}
var audioBuffer = context.createBuffer(2, data.length, 44100);
audioBuffer.getChannelData(0).set(data);
var source = context.createBufferSource(); // creates a sound source
source.buffer = audioBuffer;
source.connect(context.destination); // connect the source to the context's destination (the speakers)
source.start(0);
};
Can any one advise what is wrong?
Regards,
Nayan

I got it working !!
All I had to do is adjust the number of channel.
I've set FFMPEG to output mono audio and it worked like a charm. Here is my new FFMOEG command:
var ffmpeg = child_process.spawn("ffmpeg",[
"-re","-i",
"A.mp3",
"-ac","1","-f",
"f32le",
"pipe:1" // Output to STDOUT
]);

You are taking chunks of data, creating separate nodes from them, and starting them based on network timing. For audio to sound correct, the playback of buffers must be without break, and sample-accurate timing. You need to fundamentally change your method.
The way I do this is by creating a ScriptProcessorNode which manages its own buffer of PCM samples. On process, it reads the samples into to the output buffer. This guarantees smooth playback of audio.

Related

How to stream consecutive DYNAMIC audio files from NodeJS to HTML5 audio tag- as one stream

From an android device, the node server is receiving chunks of raw data file in arrayBuffer format.
The server is converting them individually (let's say 10 seconds worth of playable audio) into WAV format in order to stream to an HTML5 audio file.
The idea being that as soon as the android presses "send", the browser can immediately begin to hear the streamed audio file.
How can I stream these files consecutively to the HTML audio tag so that it sounds like one big file?
Currently I can stream one 10 second file to the tag and it plays. I also tried the audio tags "onended" attribute to try and fetch the next 10 second file, but you can hear that it is not connected as one file.
const audioStream = fs.createReadStream(audioPath);
audioStream.pipe(res);
Ideally I need to pipe all the files consecutively AS they come in from the android client, meaning I cannot first concatenate them into one file because the server is only receiving them as the browser is playing them (a latency of up to a few seconds is ok).
What can be done?
Edit:
Here is what I've tried:
var audio = document.getElementById('my');
var mediaSource = new MediaSource();
var SEGMENTS = 5;
var oReq = new XMLHttpRequest();
mediaSource.addEventListener('sourceopen', function() {
var sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
sourceBuffer.addEventListener('updateend', function() {
//GET('6.mp3', function(data) { onAudioLoaded(data, index); });
oReq.open("GET", '6.mp3');
oReq.responseType = "arraybuffer";
oReq.send();
});
function onAudioLoaded(data, index) {
sourceBuffer.appendBuffer(data);
}
function h(){
var arraybuffer = oReq.response; // not responseText
onAudioLoaded(arraybuffer, 0)
// console.log(arraybuffer);
}
currently, I am fetching the same mp3 file from the server with a desire to string them together to see if it works.
As I mentioned below, I get an "source buffer full" error.
in the long run i'd move this to a websocket so the each time the server receives a chunk, it will send to here, thus stringing the result into one "live stream".
Any thoughts?
Thank you
EDIT:
As it seems to have turned out, my whole approach here is wrong;
Because I am converting each received chunk of raw audio into a unique mp3 file (it cannot be WAV because of no "Media Source Extension" support)-- even though I am able to string them together using MSE, a gap between each file is heard. I am assuming this is because of "padding" in each mp3 file.
Does anyone have any alternatives so that I can seamlessly play my chunks of raw audio in the browser? Unless I am missing something in this "implementation".
Thank you.
Regarding the question "How to play consecutive dynamic audio files as one stream"...
Try a code setup like this below. Tested on Chrome.
(some names are changed, like your var arraybuffer is now input_MP3_Bytes) :
<!DOCTYPE html>
<html>
<body>
<audio id="audioMSE" controls></audio>
<script>
var input_MP3_Bytes = []; var mediaSource; var sourceBuffer;
var is_FirstAppend = true; //# for setting up MSE if first append.
var mimeCodec = 'audio/mpeg;'; //# tested on Chrome browser only
//var mimeCodec = 'audio/mpeg; codecs="mp3"'; //# untested.. maybe needed by other browsers?
//# for audio tag reference
var myAudio = document.getElementById('audioMSE');
myAudio.play(); //# start conveyor belt (to receive incoming bytes)
//# load some input MP3 bytes for MSE
var fileURL = "file_01.mp3"; //"6.mp3"
load_MP3( fileURL );
function load_MP3( filename )
{
var oReq = new XMLHttpRequest();
oReq.open("GET", filename, true);
oReq.responseType = "arraybuffer";
oReq.addEventListener('loadend', on_MP3isLoaded );
oReq.send(null);
}
function on_MP3isLoaded()
{
input_MP3_Bytes = new Uint8Array( this.response ); //# fill the byte array
if (is_FirstAppend == true)
{
console.log( "MSE support : " + MediaSource.isTypeSupported(mimeCodec) );
makeMSE(); is_FirstAppend = false;
}
else { sourceBuffer.appendBuffer( input_MP3_Bytes ); }
}
function makeMSE ( )
{
if ('MediaSource' in window && MediaSource.isTypeSupported(mimeCodec) )
{
mediaSource = new MediaSource;
myAudio.src = URL.createObjectURL( mediaSource );
mediaSource.addEventListener( "sourceopen", mse_sourceOpen );
}
else { console.log("## Unsupported MIME type or codec: " + mimeCodec); }
}
function mse_sourceOpen()
{
sourceBuffer = mediaSource.addSourceBuffer( mimeCodec );
sourceBuffer.addEventListener( "updateend", mse_updateEnd );
//sourceBuffer.mode = "sequence";
sourceBuffer.appendBuffer( input_MP3_Bytes );
};
function mse_updateEnd()
{
//# what to do after feeding-in the bytes...?
//# 1) Here you could update (overwrite) same "input_MP3_Bytes" array with new GET request
//# 2) Or below is just a quick test... endless looping
//# 1) load another MP3 file
fileURL = "file_25.mp3";
load_MP3( fileURL );
//# 2) re-feed same bytes for endless looping
//sourceBuffer.appendBuffer( input_MP3_Bytes );
//mediaSource.endOfStream(); //# close stream to close audio
//console.log("mediaSource.readyState : " + mediaSource.readyState); //# is ended
}
</script>
</body>
</html>

Web Audio Api: Proper way to play data chunks from a nodejs server via socket

I'm using the following code to decode audio chunks from nodejs's socket
window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new AudioContext();
var delayTime = 0;
var init = 0;
var audioStack = [];
var nextTime = 0;
client.on('stream', function(stream, meta){
stream.on('data', function(data) {
context.decodeAudioData(data, function(buffer) {
audioStack.push(buffer);
if ((init!=0) || (audioStack.length > 10)) { // make sure we put at least 10 chunks in the buffer before starting
init++;
scheduleBuffers();
}
}, function(err) {
console.log("err(decodeAudioData): "+err);
});
});
});
function scheduleBuffers() {
while ( audioStack.length) {
var buffer = audioStack.shift();
var source = context.createBufferSource();
source.buffer = buffer;
source.connect(context.destination);
if (nextTime == 0)
nextTime = context.currentTime + 0.05; /// add 50ms latency to work well across systems - tune this if you like
source.start(nextTime);
nextTime+=source.buffer.duration; // Make the next buffer wait the length of the last buffer before being played
};
}
But it has some gaps/glitches between audio chunks that I'm unable to figure out.
I've also read that with MediaSource it's possible to do the same and having the timing handled by the player instead of doing it manually. Can someone provide an example of handling mp3 data?
Moreover, which is the proper way to handle live streaming with web audio API? I've already read almost all questions os SO about this subject and none of them seem to work without glitches. Any ideas?
You can take this code as an example: https://github.com/kmoskwiak/node-tcp-streaming-server
It basically uses media source extensions. All you need to do is to change from video to audio
buffer = mediaSource.addSourceBuffer('audio/mpeg');
yes #Keyne is right,
const mediaSource = new MediaSource()
const sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg')
player.src = URL.createObjectURL(mediaSource)
sourceBuffer.appendBuffer(chunk) // Repeat this for each chunk as ArrayBuffer
player.play()
But do this only if you don't care about IOS support 🤔 (https://developer.mozilla.org/en-US/docs/Web/API/MediaSource#Browser_compatibility)
Otherwise please let me know how you do it !

Playing incoming ArrayBuffer audio binary data from binaryjs server simultaneously

Good day! I'm into video chat streaming this morning and I've bumped into a problem with the incoming ArrayBuffer which contains binary data of an audio.
Here is the code I found for playing binary audio data (Uint8Array):
function playByteArray(byteArray) {
var arrayBuffer = new ArrayBuffer(byteArray.length);
var bufferView = new Uint8Array(arrayBuffer);
for (i = 0; i < byteArray.length; i++) {
bufferView[i] = byteArray[i];
}
context.decodeAudioData(arrayBuffer, function(buffer) {
buf = buffer;
play();
});
}
// Play the loaded file
function play() {
// Create a source node from the buffer
var source = context.createBufferSource();
source.buffer = buf;
// Connect to the final output node (the speakers)
source.connect(context.destination);
// Play immediately
source.start(0);
}
Now below, I've used MediaStreamRecorder from https://github.com/streamproc/MediaStreamRecorder to record the stream from getUserMedia. This code will continuously send the recorded binary data to the server.
if (navigator.getUserMedia) {
navigator.getUserMedia({audio: true, video: true}, function(stream) {
video.src = (window.URL || window.webkitURL).createObjectURL(stream); //get this for video strewam url
video.muted = true;
multiStreamRecorder = new MultiStreamRecorder(stream);
multiStreamRecorder.canvas = {
width: video.width,
height: video.height
};
multiStreamRecorder.video = video;
multiStreamRecorder.ondataavailable = function(blobs) {
var audioReader = new FileReader();
audioReader.addEventListener("loadend", function() {
var arrBuf = audioReader.result;
var binary = new Uint8Array(arrBuf);
streamToServ.write(binary);
// streamToServ is the binaryjs client
});
audioReader.readAsArrayBuffer(blobs.audio);
};
multiStreamRecorder.start(1);
}, onVideoFail);
} else {
alert ('failed');
}
Convert the blobs produced (audio and video) to binary and send it to binaryjs which will be played on another client with this:
client.on('stream', function (stream, meta) {
stream.on('data', function(data) {
playByteArray(new Uint8Array(data));
});
});
I had no problems with transferring the binary data but the problem is there is a hiccup sound in the playback significantly on every binary data that was played. Is there something wrong on how I play the incoming ArrayBuffers? I'm also thinking of asking streamproc about this.
Thanks in advance!
Cheers.
I found a solution to this problem by making an audio buffer queueing. Most of the code is from here:
Choppy/inaudible playback with chunked audio through Web Audio API
Thanks.
Not sure if this is the problem, but perhaps instead of source.start(0), you should use source.start(time), where time is where you want to start the source. source.start(0) will start playing immediately. If your byte array comes in faster than real-time, the sources might overlap because you start them as soon as possible.

Socket.io with AudioContext send and receive audio Errors on receiving

I am trying to build something, where a user can send audio instantly to many people using, socket.io, audioContext, js for the front-end and Node.js,socket.io for the server.
I can record the audio, send it to the server and send it back to other users, but I cannot play the data. I guess it must be a problem of how I send them or how I process the buffer that receives them.
I get the following error: Update!
The buffer passed to decodeAudioData contains an unknown content type.
Audio is passed fine, the buffer is created with no errors but there is no sound feedback.
The User presses record and it started recording/streaming with he following functions:
This is how it all starts:
navigator.getUserMedia({audio: true,video: false}, initializeRecorder, errorCallback);
function initializeRecorder(stream) {
var bufferSize = 2048;
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var source = audioCtx.createMediaStreamSource(stream);
var recorder = audioCtx.createScriptProcessor(bufferSize, 1, 1);
recorder.onaudioprocess = recorderProcess;
source.connect(recorder);
recorder.connect(audioCtx.destination);
recording = true;
initialized = true;
play = false;
stop = true;
}
function recorderProcess(e) {
var left = e.inputBuffer.getChannelData(0);
socket.emit('audio-blod-send', convertFloat32ToInt16(left));
}
function convertFloat32ToInt16(buffer) {
l = buffer.length;
buf = new Int16Array(l);
while (l--) {
buf[l] = Math.min(1, buffer[l])*0x7FFF;
}
return buf.buffer;
}
Then the server uses the socket to broadcast what the original sender send:
socket.on('audio-blod-send',function(data){
socket.broadcast.to(roomName).emit('audio-blod-receive', data);
});
And then the data are played: Update!
I was using audioContext.decodeData which I found out that it is only used to read/decode audio from MP3 or WAV files not streaming. With the new code no errors appear however there is no Audio feedback.
socket.on('audio-blod-receive',function(data) {
playAudio(data);
});
function playAudio(buffer)
{
var audioCtx;
var started = false;
if(!audioCtx) {
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
}
source = audioCtx.createBufferSource();
audioBuffer = audioCtx.createBuffer( 1, 2048, audioCtx.sampleRate );
audioBuffer.getChannelData( 0 ).set( buffer );
source.buffer = audioBuffer;
source.connect( audioCtx.destination );
source.start(0);
console.log(buffer);
}
P.S If anyone is interested further in what I am trying to do, feel free to contact me.

Web audio API downsample 44.1 khz in Javascript

I'm using RecorderJS to record a microphone stream from the user. The default export is a WAV file at 44.1 kHz, 16bit. Is there anyway I can downsample this to 11kHz or 16kHz without it sounding weird?
Is there anyway I can get a 16bit 16khz WAV file out of a Web Audio API getUserMedia stream, by using only javascript?
I'm trying to reduce the file size, thus saving a lot of bandwidth for the users. Thanks.
Edit : one more thing, you can also, send only one channel instead of both...
I am not sure if this is the right way, but I did by doing interpolation of the data received from microphone,
I am guess, you are capturing your data from microphone something like this,
this.node.onaudioprocess = function(e){
if (!recording) return;
worker.postMessage({
command: 'record',
buffer: [
e.inputBuffer.getChannelData(0),
e.inputBuffer.getChannelData(1)
]
});
}
now modify it into
var oldSampleRate = 44100, newSampleRate = 16000;
this.node.onaudioprocess = function(e){
var leftData = e.inputBuffer.getChannelData(0);
var rightData = e.inputBuffer.getChannelData(1);
leftData = interpolateArray(leftData, leftData.length * (newSampleRate/oldSampleRate) );
rightData = interpolateArray(rightData, rightData.length * (newSampleRate/oldSampleRate) );
if (!recording) return;
worker.postMessage({
command: 'record',
buffer: [
leftData,
rightData
]
});
}
function interpolateArray(data, fitCount) {
var linearInterpolate = function (before, after, atPoint) {
return before + (after - before) * atPoint;
};
var newData = new Array();
var springFactor = new Number((data.length - 1) / (fitCount - 1));
newData[0] = data[0]; // for new allocation
for ( var i = 1; i < fitCount - 1; i++) {
var tmp = i * springFactor;
var before = new Number(Math.floor(tmp)).toFixed();
var after = new Number(Math.ceil(tmp)).toFixed();
var atPoint = tmp - before;
newData[i] = linearInterpolate(data[before], data[after], atPoint);
}
newData[fitCount - 1] = data[data.length - 1]; // for new allocation
return newData;
};

Resources