Web audio API downsample 44.1 khz in Javascript - audio

I'm using RecorderJS to record a microphone stream from the user. The default export is a WAV file at 44.1 kHz, 16bit. Is there anyway I can downsample this to 11kHz or 16kHz without it sounding weird?
Is there anyway I can get a 16bit 16khz WAV file out of a Web Audio API getUserMedia stream, by using only javascript?
I'm trying to reduce the file size, thus saving a lot of bandwidth for the users. Thanks.

Edit : one more thing, you can also, send only one channel instead of both...
I am not sure if this is the right way, but I did by doing interpolation of the data received from microphone,
I am guess, you are capturing your data from microphone something like this,
this.node.onaudioprocess = function(e){
if (!recording) return;
worker.postMessage({
command: 'record',
buffer: [
e.inputBuffer.getChannelData(0),
e.inputBuffer.getChannelData(1)
]
});
}
now modify it into
var oldSampleRate = 44100, newSampleRate = 16000;
this.node.onaudioprocess = function(e){
var leftData = e.inputBuffer.getChannelData(0);
var rightData = e.inputBuffer.getChannelData(1);
leftData = interpolateArray(leftData, leftData.length * (newSampleRate/oldSampleRate) );
rightData = interpolateArray(rightData, rightData.length * (newSampleRate/oldSampleRate) );
if (!recording) return;
worker.postMessage({
command: 'record',
buffer: [
leftData,
rightData
]
});
}
function interpolateArray(data, fitCount) {
var linearInterpolate = function (before, after, atPoint) {
return before + (after - before) * atPoint;
};
var newData = new Array();
var springFactor = new Number((data.length - 1) / (fitCount - 1));
newData[0] = data[0]; // for new allocation
for ( var i = 1; i < fitCount - 1; i++) {
var tmp = i * springFactor;
var before = new Number(Math.floor(tmp)).toFixed();
var after = new Number(Math.ceil(tmp)).toFixed();
var atPoint = tmp - before;
newData[i] = linearInterpolate(data[before], data[after], atPoint);
}
newData[fitCount - 1] = data[data.length - 1]; // for new allocation
return newData;
};

Related

How to convert from prism-media(#discordjs/opus) opus stream to format suitable for picovoice/porcupine?

I have created a discord bot using discord.js and am attempting to implement basic voice capabilities using porcupine.
I have a stream of audio per user and am trying to use the process(frame) method in porcupine on each chunk of data.
In order to get the data single channel and sample rate 16k I am manually decoding the stream using prism-media opus decoder then trying to pass in the chunks:
execute(connection, user, args) {
userHandlers[user] = new Porcupine([GRASSHOPPER, BUMBLEBEE], [0.5, 0.65]);
if (!receiver) {
receiver = connection.receiver;
}
userStreams[user] = receiver.createStream(user, {mode: 'opus', end: 'manual'});
const decoder = new prism.opus.Decoder({frameSize: 640, channels: 1, rate: 16000});
userStreams[user]
.pipe(decoder);
listeningToUsers[user] = true;
try {
console.log("Start utterance");
decoder.on('data', (chunk) => {//Need to make stream single channel, frame size 512
let keywordIndex = userHandlers[user].process(chunk);
if (keywordIndex != -1) {
meme.execute(connection, null, args);
}
});
} catch (error) {
console.error(error);
}
},
My issue, however, is that the size of the chunk is 640 whereas it needs to be 512 for the method to work. Changing the frameSize that is passed to the decoder doesn't work due to reasons explained in the answer here.
If anyone knows the best way to convert my data to the correct chunk size, or just a better way of doing this altogether, I'd appreciate it.
I ended up getting this working by using some of the code included in this demo file.
We include a chunkArray function:
function chunkArray(array, size) {
return Array.from({ length: Math.ceil(array.length / size) }, (v, index) =>
array.slice(index * size, index * size + size)
);
}
and change the code posted before to look like this:
execute(connection, user, args) {
userHandlers[user] = new Porcupine([GRASSHOPPER, BLUEBERRY], [0.7, 0.85]);
const frameLength = userHandlers[user].frameLength;
if (!receiver) {
receiver = connection.receiver;
}
userStreams[user] = receiver.createStream(user, {mode: 'opus', end: 'manual'});
userDecoders[user] = new prism.opus.Decoder({frameSize: 640, channels: 1, rate: 16000});
userStreams[user]
.pipe(userDecoders[user]);
listeningToUsers[user] = true;
userFrameAccumulators[user] = [];
try {
userDecoders[user].on('data', (data) => {
// Two bytes per Int16 from the data buffer
let newFrames16 = new Array(data.length / 2);
for (let i = 0; i < data.length; i += 2) {
newFrames16[i / 2] = data.readInt16LE(i);
}
// Split the incoming PCM integer data into arrays of size Porcupine.frameLength. If there's insufficient frames, or a remainder,
// store it in 'frameAccumulator' for the next iteration, so that we don't miss any audio data
userFrameAccumulators[user] = userFrameAccumulators[user].concat(newFrames16);
let frames = chunkArray(userFrameAccumulators[user], frameLength);
if (frames[frames.length - 1].length !== frameLength) {
// store remainder from divisions of frameLength
userFrameAccumulators[user] = frames.pop();
} else {
userFrameAccumulators[user] = [];
}
for (let frame of frames) {
let index = userHandlers[user].process(frame);
if (index !== -1) {
if (index == 0) {//GRASSHOPPER
play.execute(connection, null, args);
} else if (index == 1) {//BLUEBERRY
play.skip();
}
}
}
});
} catch (error) {
console.error(error);
}
}
This will take our decoded stream and then convert it and chunk it appropriately.
I imagine there are inefficiencies here and improvements to be had, but it is working well in the discord server and I wanted to post an answer in case anyone is trying to integrate discord.js with porcupine in the future.

How to pipe large file into mongodb using gridfsbucket, when receiving file in chunks?

I am using nodejs, mongodb, and gridfsbucket.
I am receiving a file into my server in 255 byte chunks, the files can be extremely large so creating a variable to store the chunks and then piping that into mongo using gridfsbucket is not a viable option.
Currently I have a working method of temporarily storing the file on disk and then piping that into mongo. This actually works pretty well, the only problem is I don't want to temporarily store the data before streaming into mongo using gridfsbucket.
Does anyone know how to take those chunks as they are coming into my server and immediately stream them into mongo using gridfsbucket? I am thinking that I would need to open the pipe and then constantly stream the chunks into the pipe, but I am not sure how to accomplish this.
This is my current code for storing to the disk:
fs.appendFileSync(this.tempFileName, Buffer.from(this.currfile.currdatachunk));
this.currfile.filename = this.currfile.filename.replace(")", Date.now() + ")");
var fileName = decodeURI(this.currfile.filename.replace("$(", "").replace(")", ""));
fileName = encodeURI(fileName);
var self = this;
var gb = new GridFSBucket(mongoCacheDb, { bucketName: this.cacheCollection });
var uploadStream = gb.openUploadStream(fileName);
uploadStream.options.metadata = {
'url': "/getFile/" + fileName,
'id': uploadStream.id
}
uploadStream.once("finish", function uploadStream_onceFinish() {
if (this.length > 0) {
var ms = new Message();
ms.data = self.cacheCollection + "/" + self.currfile.filename;
ms.datatype = "URL";
ms.hasdata = "yes";
ms.haserrors = "no";
ms.type = "APPXLOADURL";
sendMessage(ws, ms);
/*Send response to server indicating file receipt*/
var nameAB = Buffer.from(self.currfile.filename);
self.clientsocket.write(Buffer.from(hton32(nameAB.length)));
self.clientsocket.write(Buffer.from(nameAB));
self.clientsocket.write(Buffer.from([3, 1]));
console.log("Finished: " + Date.now());
} else {
var nameAB = Buffer.from(self.currfile.filename);
self.clientsocket.write(Buffer.from(hton32(nameAB.length)));
self.clientsocket.write(Buffer.from(nameAB));
self.clientsocket.write(Buffer.from([0, 0]));
}
fs.unlinkSync(self.tempFileName);
});
fs.createReadStream(this.tempFileName).pipe(uploadStream);
I guess I should have waited another day. Finally wrapped my head around it and stopped overthinking it. I created the pipe on the first run and then just pushed the buffered chunks into the pipe as they were received. Code is below:
if (this.chunksReceived == 0) {
this.rStream = new Readable({ read(size) { } });
this.currfile.filename = this.currfile.filename.replace(")", Date.now() + ")");
var fileName = decodeURI(this.currfile.filename.replace("$(", "").replace(")", ""));
fileName = encodeURI(fileName);
var self = this;
var gb = new GridFSBucket(mongoCacheDb, { bucketName: this.cacheCollection });
this.uploadStream = gb.openUploadStream(fileName);
this.uploadStream.options.metadata = {
'url': "/getFile/" + fileName,
'id': this.uploadStream.id
}
this.uploadStream.once("finish", function uploadStream_onceFinish() {
if (this.length > 0) {
var ms = new Message();
ms.data = self.cacheCollection + "/" + self.currfile.filename;
ms.datatype = "URL";
ms.hasdata = "yes";
ms.haserrors = "no";
ms.type = "APPXLOADURL";
sendMessage(ws, ms);
/*Send response to server indicating file receipt*/
var nameAB = Buffer.from(self.currfile.filename);
self.clientsocket.write(Buffer.from(hton32(nameAB.length)));
self.clientsocket.write(Buffer.from(nameAB));
self.clientsocket.write(Buffer.from([3, 1]));
console.log("Finished: " + Date.now());
} else {
var nameAB = Buffer.from(self.currfile.filename);
self.clientsocket.write(Buffer.from(hton32(nameAB.length)));
self.clientsocket.write(Buffer.from(nameAB));
self.clientsocket.write(Buffer.from([0, 0]));
}
});
this.rStream.pipe(this.uploadStream);
}
this.rStream.push(Buffer.from(this.currfile.currdatachunk));
this.chunksReceived++;

Web Audio Api: Proper way to play data chunks from a nodejs server via socket

I'm using the following code to decode audio chunks from nodejs's socket
window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new AudioContext();
var delayTime = 0;
var init = 0;
var audioStack = [];
var nextTime = 0;
client.on('stream', function(stream, meta){
stream.on('data', function(data) {
context.decodeAudioData(data, function(buffer) {
audioStack.push(buffer);
if ((init!=0) || (audioStack.length > 10)) { // make sure we put at least 10 chunks in the buffer before starting
init++;
scheduleBuffers();
}
}, function(err) {
console.log("err(decodeAudioData): "+err);
});
});
});
function scheduleBuffers() {
while ( audioStack.length) {
var buffer = audioStack.shift();
var source = context.createBufferSource();
source.buffer = buffer;
source.connect(context.destination);
if (nextTime == 0)
nextTime = context.currentTime + 0.05; /// add 50ms latency to work well across systems - tune this if you like
source.start(nextTime);
nextTime+=source.buffer.duration; // Make the next buffer wait the length of the last buffer before being played
};
}
But it has some gaps/glitches between audio chunks that I'm unable to figure out.
I've also read that with MediaSource it's possible to do the same and having the timing handled by the player instead of doing it manually. Can someone provide an example of handling mp3 data?
Moreover, which is the proper way to handle live streaming with web audio API? I've already read almost all questions os SO about this subject and none of them seem to work without glitches. Any ideas?
You can take this code as an example: https://github.com/kmoskwiak/node-tcp-streaming-server
It basically uses media source extensions. All you need to do is to change from video to audio
buffer = mediaSource.addSourceBuffer('audio/mpeg');
yes #Keyne is right,
const mediaSource = new MediaSource()
const sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg')
player.src = URL.createObjectURL(mediaSource)
sourceBuffer.appendBuffer(chunk) // Repeat this for each chunk as ArrayBuffer
player.play()
But do this only if you don't care about IOS support 🤔 (https://developer.mozilla.org/en-US/docs/Web/API/MediaSource#Browser_compatibility)
Otherwise please let me know how you do it !

Socket.io with AudioContext send and receive audio Errors on receiving

I am trying to build something, where a user can send audio instantly to many people using, socket.io, audioContext, js for the front-end and Node.js,socket.io for the server.
I can record the audio, send it to the server and send it back to other users, but I cannot play the data. I guess it must be a problem of how I send them or how I process the buffer that receives them.
I get the following error: Update!
The buffer passed to decodeAudioData contains an unknown content type.
Audio is passed fine, the buffer is created with no errors but there is no sound feedback.
The User presses record and it started recording/streaming with he following functions:
This is how it all starts:
navigator.getUserMedia({audio: true,video: false}, initializeRecorder, errorCallback);
function initializeRecorder(stream) {
var bufferSize = 2048;
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var source = audioCtx.createMediaStreamSource(stream);
var recorder = audioCtx.createScriptProcessor(bufferSize, 1, 1);
recorder.onaudioprocess = recorderProcess;
source.connect(recorder);
recorder.connect(audioCtx.destination);
recording = true;
initialized = true;
play = false;
stop = true;
}
function recorderProcess(e) {
var left = e.inputBuffer.getChannelData(0);
socket.emit('audio-blod-send', convertFloat32ToInt16(left));
}
function convertFloat32ToInt16(buffer) {
l = buffer.length;
buf = new Int16Array(l);
while (l--) {
buf[l] = Math.min(1, buffer[l])*0x7FFF;
}
return buf.buffer;
}
Then the server uses the socket to broadcast what the original sender send:
socket.on('audio-blod-send',function(data){
socket.broadcast.to(roomName).emit('audio-blod-receive', data);
});
And then the data are played: Update!
I was using audioContext.decodeData which I found out that it is only used to read/decode audio from MP3 or WAV files not streaming. With the new code no errors appear however there is no Audio feedback.
socket.on('audio-blod-receive',function(data) {
playAudio(data);
});
function playAudio(buffer)
{
var audioCtx;
var started = false;
if(!audioCtx) {
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
}
source = audioCtx.createBufferSource();
audioBuffer = audioCtx.createBuffer( 1, 2048, audioCtx.sampleRate );
audioBuffer.getChannelData( 0 ).set( buffer );
source.buffer = audioBuffer;
source.connect( audioCtx.destination );
source.start(0);
console.log(buffer);
}
P.S If anyone is interested further in what I am trying to do, feel free to contact me.

Live streaming using FFMPEG to web audio api

I am trying to stream audio using node.js + ffmpeg to browsers connected in LAN only using web audio api.
Not using element because it's adding it's own buffer of 8 to 10 secs and I want to get maximum high latency possible (around 1 to 2 sec max).
Audio plays successfully but audio is very choppy and noisy.
Here is my node.js (server side) file:
var ws = require('websocket.io'),
server = ws.listen(3000);
var child_process = require("child_process");
var i = 0;
server.on('connection', function (socket)
{
console.log('New client connected');
var ffmpeg = child_process.spawn("ffmpeg",[
"-re","-i",
"A.mp3","-f",
"f32le",
"pipe:1" // Output to STDOUT
]);
ffmpeg.stdout.on('data', function(data)
{
var buff = new Buffer(data);
socket.send(buff.toString('base64'));
});
});
And here is my HTML:
var audioBuffer = null;
var context = null;
window.addEventListener('load', init, false);
function init() {
try {
context = new webkitAudioContext();
} catch(e) {
alert('Web Audio API is not supported in this browser');
}
}
var ws = new WebSocket("ws://localhost:3000/");
ws.onmessage = function(message)
{
var d1 = base64DecToArr(message.data).buffer;
var d2 = new DataView(d1);
var data = new Float32Array(d2.byteLength / Float32Array.BYTES_PER_ELEMENT);
for (var jj = 0; jj < data.length; ++jj)
{
data[jj] = d2.getFloat32(jj * Float32Array.BYTES_PER_ELEMENT, true);
}
var audioBuffer = context.createBuffer(2, data.length, 44100);
audioBuffer.getChannelData(0).set(data);
var source = context.createBufferSource(); // creates a sound source
source.buffer = audioBuffer;
source.connect(context.destination); // connect the source to the context's destination (the speakers)
source.start(0);
};
Can any one advise what is wrong?
Regards,
Nayan
I got it working !!
All I had to do is adjust the number of channel.
I've set FFMPEG to output mono audio and it worked like a charm. Here is my new FFMOEG command:
var ffmpeg = child_process.spawn("ffmpeg",[
"-re","-i",
"A.mp3",
"-ac","1","-f",
"f32le",
"pipe:1" // Output to STDOUT
]);
You are taking chunks of data, creating separate nodes from them, and starting them based on network timing. For audio to sound correct, the playback of buffers must be without break, and sample-accurate timing. You need to fundamentally change your method.
The way I do this is by creating a ScriptProcessorNode which manages its own buffer of PCM samples. On process, it reads the samples into to the output buffer. This guarantees smooth playback of audio.

Resources