Filtering and mixing WebRTC sound in NodeJs - node.js

Few weeks ago I wrote WebRTC browser client using mediasoup library. Now I am in middle of rewriting it as a NodeJS client.
I am stuck with one thing. I want to receive multiple WebRTC audio souroces, mix them into single track then apply some filters(eg. biquad filter) and then resend this track via WebRTC. Like on this diagram:
With browser I could achieve this using Web Audio API this is the code I used:
this.audioContext = new AudioContext();
this.outgoingStream = this.audioContext.createMediaStreamDestination();
this.addSoundFilter();
this.mixedTrack = this.outgoingStream.stream.getAudioTracks()[0];
this.handleIncomingSound();
addSoundFilter() {
this.filter = this.audioContext.createBiquadFilter();
this.filter.type = "lowpass";
this.filter.frequency.value = this.mapFrequencyValue();
this.gainer = this.audioContext.createGain();
this.gainer.gain.value = this.mapGainValue();
}
handleIncomingSound() {
this.audios.forEach((audio, peerId) => {
this.filterAudio(audio);
});
}
filterAudio(audioConsumerId) {
let audio = this.audios.get(audioConsumerId);
const audioElement = document.getElementById(audio.id);
const incomingSource = this.audioContext.createMediaStreamSource(
audioElement.srcObject
);
incomingSource.connect(this.filter).connect(this.gainer).connect(this.outgoingStream);
}
And with this code I could then send this.mixedTrack via WebRTC
However in NodeJS there is no WebAudioApi.
So how this can be achieved or if it's even possible to do it?

Related

Google meet streaming through MediaRecorder APIs

I am trying to record google meet audio and video using getDisplayMedia using media options.I am able to record my screen through getDisplayMedia but unable to record voice coming in google meet. How can I use passive recording or is there any way where I can record screen, audio and video in google meet through any mechanism or code?
navigator.mediaDevices.getDisplayMedia({audio: true,video: true});
let mediaRecorder = new MediaRecorder(mediaStreamObj);
let chunks = [];
mediaRecorder.start();
console.log(mediaRecorder.state);
mediaRecorder.ondataavailable = function(ev) {
chunks.push(ev.data);
}
setTimeout(function() {
console.log("Stopping!");
mediaRecorder.stop();
console.log(mediaRecorder.state);
mediaRecorder.onstop = (ev) => {
let blob = new Blob(chunks, {
'type': 'video/mp4;'
});
chunks = [];
let videoURL = window.URL.createObjectURL(blob);
console.log(videoURL);
}
}, 40000);
i did it with puppeteer. i build a google meet bot who will join your meetings from google calendar and record audio + save all transcript . when meeting ends everything will be sent over email or you can see in app dashboard.

Unable to stream microphone audio to Google Speech to Text with NodeJS

I am going to develop a simple web based Speech to Text project. Develop with NodeJS, ws (WebSocket), and Google's Speech to Text API.
However, I have no luck to get the transcript from Google's Speech to Text API.
Below are my server side codes (server.js):
ws.on('message', function (message) {
if (typeof message === 'string') {
if(message == "connected") {
console.log(`Web browser connected postback.`);
}
}
else {
if (recognizeStream !== null) {
const buffer = new Int16Array(message, 0, Math.floor(message.byteLength / 2));
recognizeStream.write(buffer);
}
}
});
Below are my client side codes (ws.js):
function recorderProcess(e) {
var floatSamples = e.inputBuffer.getChannelData(0);
const ConversionFactor = 2 ** (16 - 1) - 1;
var floatSamples16 = Int16Array.from(floatSamples.map(n => n * ConversionFactor));
ws.send(floatSamples16);
}
function successCallback(stream) {
window.stream = stream;
var audioContext = window.AudioContext;
var context = new audioContext();
var audioInput = context.createMediaStreamSource(stream);
var recorder = context.createScriptProcessor(2048, 1, 1);
recorder.onaudioprocess = recorderProcess;
audioInput.connect(recorder);
recorder.connect(context.destination);
}
When I run the project, and open http://localhost/ in my browser, trying to speaking some sentences to the microphone. Unfortunately, there are no transcription returned, and no error messages returned in NodeJS console.
When I check the status in Google Cloud Console, it only display a 499 code in the dashboard.
Many thanks for helping!
I think the issue could be related to the stream process. Maybe some streaming process is stopped before the end of an operation. My suggestion is to review the callbacks in the JasvaScript code in order to find some “broken" promises.
Also, maybe its obvious but there is a different doc for audios than more than a minute:
https://cloud.google.com/speech-to-text/docs/async-recognize
CANCELLED - The operation was cancelled, typically by the caller.
HTTP Mapping: 499 Client Closed Request
Since the error message, this also could be related to the asynchronous and multithread features of node js.
Hope this works!

How to create a MySQL adapter for session.io in NodeJS?

I have a NodeJS project on 2 Google Cloud instances behind a load-balancer. I'm using socket.io. I want to share the sessions between the instances.
Usually developers doing it using socket.io-redies, but I don't want redis just for that. I have Cloud SQL (Aka: MySQL), and I want to use MySQL for sharing the sessions.
I have understand the whole index.js of the redis adapter file, except this function:
https://github.com/socketio/socket.io-redis/blob/master/index.js#L93
Redis.prototype.onmessage = function(channel, msg){
var args = msgpack.decode(msg);
var packet;
if (uid == args.shift()) return debug('ignore same uid');
packet = args[0];
if (packet && packet.nsp === undefined) {
packet.nsp = '/';
}
if (!packet || packet.nsp != this.nsp.name) {
return debug('ignore different namespace');
}
args.push(true);
this.broadcast.apply(this, args);
};
If I need to get events from the MySQL (subscribe) I think it is not possible. Am I right?
Do you know another solution of sharing socket.io between two machines, without using Redis?

Google TTS in Crossrider

i developed Google chrome extension that contains Google TTS
i rewrite it with Crossrider to make it work in different platforms (it works great untill it comes to TTS part)
here is the code :
function PlayGoogleTTS(EngWord){
voices = speechSynthesis.getVoices();
msg = new SpeechSynthesisUtterance();
msg.volume = 1; // 0 to 1
msg.rate = 10; // 0.1 to 10
msg.pitch = 2; //0 to 2
msg.text = EngWord;
msg.lang = 'en-US';
msg.voice = voices[1];
msg.voice = voices[1]; // Note: some voices don't support altering params
speechSynthesis.speak(msg);
}
// Fetch the list of voices and populate the voice options.
function loadVoices() {
// Fetch the available voices.
var voices = speechSynthesis.getVoices();
}
// Chrome loads voices asynchronously.
window.speechSynthesis.onvoiceschanged = function(e) {
loadVoices();
};
so how can i convert it to make it work on Crossrider?
It's not clear from your question which speechSynthesis library/api you are using. However, assuming it is based on Chrome's TTS API, the required "tts" permission is not available.
[Disclosure: I am a Crossrider employee]
it's considered workaround more than an answer
just used another TTS that able to generate ogg maves in firefox

Broadcasting with shoe sockjs

I am trying to setup a simple scenerio using shoe + dnode +sockjs and I do not know how to broadcast a message to all users connected to the web application.
Do you know if there is a function or method which manage this? or should it be make by "hand"?
AFAIK, you have to roll it by "hand" as you say. Here is what I do:
server.js:
var shoe = require('shoe')
var connectedClients = {}
var conCount = 0
var sock = shoe(function(clientStream) {
clienStream.id = conCount
connectedClients[clientStream.id] = clientStream
conCount += 1
})
somewhere else in your server-side program:
//write to all connected clients
Object.keys(connectedClients).forEach(function(cid) {
var clientStream = connectedClients[cid]
clientStream.write(yourData)
})
Note, you'll want to introduce additional logic to only write to connected clients, so you'll want to remove disconnected clients from connectedClients, something like delete connectedClients[id].
Hopefully that helps.

Resources