I'm capturing audio input from microphone with getUserMedia() function, works fine in chrome, but in firefox sound dies out after 5 seconds. If I send request for microphone again (without reloading the page) same thing happens. Here is the code (I used http://updates.html5rocks.com/2012/09/Live-Web-Audio-Input-Enabled as guidance):
//getting the function depending on browser
navigator.getMedia = ( navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia);
// success callback when requesting audio input stream
function gotAudioStream(stream) {
window.AudioContext = window.AudioContext || window.webkitAudioContext;
var audioContext = new AudioContext();
// Create an AudioNode from the stream.
var mediaStreamSource = audioContext.createMediaStreamSource( stream );
// Connect it to the destination to hear yourself (or any other node for processing!)
mediaStreamSource.connect( audioContext.destination );
}
function gotError(err) {
alert("An error occured! " + err);
}
//when button clicked, browser asks a permission to access microphone
jQuery("#sound_on").click(function()
{
navigator.getMedia({audio: true},gotAudioStream,gotError);
});
Any ideas?
EDIT/UPDATE
Thank you, csch, for the reference. Workaround by Karoun Kasraie worked!
context = new AudioContext();
navigator.getUserMedia({ audio: true }, function(stream) {
// the important thing is to save a reference to the MediaStreamAudioSourceNode
// thus, *window*.source or any other object reference will do
window.source = context.createMediaStreamSource(stream);
source.connect(context.destination);
}, alert);
It's a bug in Firefox, it can be found here:
https://bugzilla.mozilla.org/show_bug.cgi?id=934512
There's also a workaround:
context = new AudioContext();
navigator.getUserMedia({ audio: true }, function(stream) {
// the important thing is to save a reference to the MediaStreamAudioSourceNode
// thus, *window*.source or any other object reference will do
window.source = context.createMediaStreamSource(stream);
source.connect(context.destination);
}, alert);
source
Related
Please consider the code below:
navigator.mediaDevices.getUserMedia({audio: true}).then(function() {
navigator.mediaDevices.enumerateDevices().then((devices) => {
devices.forEach(function(device1, k1) {
if (device1.kind == 'audiooutput' && device1.deviceId == 'default') {
const speakersGroupId = device1.groupId;
devices.forEach(function(device2, k2) {
if (device2.groupId == speakersGroupId && ['default', 'communications'].includes(device2.deviceId) === false) {
const speakersId = device2.deviceId;
const constraints = {
audio: {
deviceId: {
exact: speakersId
}
}
};
console.log('Requesting stream for deviceId '+speakersId);
navigator.mediaDevices.getUserMedia(constraints).then((stream) => { // **this always fails**
console.log(stream);
});
}
});
}
});
});
});
The code asks for permissions via the first getUserMedia, then enumerates all devices, picks the default audio output then tries to get a stream for that output.
But it will always throw the error: OverconstrainedError { constraint: "deviceId", message: "", name: "OverconstrainedError" } when getting the audio stream.
There is nothing I can do in Chrome (don't care about other browsers, tested Chrome 108 and 109 beta) to get this to work.
I see a report here that it works, but not for me.
Please tell me that I'm doing something wrong, or if there's another way to get the speaker stream that doesn't involve chrome.tabCapture or chrome.desktopCapture.
Chrome MV3 extension ways are welcomed, not only HTML5.
.getUserMedia() is used to get input streams. So, when you tell it to use a speaker device, it can't comply. gUM's error reporting is, umm, confusing (to put it politely).
To use an output device, use element.setSinkId(deviceId). Make an audio or video element, then set its sink id. Here's the MDN example; it creates an audio element. You can also use a preexisting audio or video element.
const devices = await navigator.mediaDevices.enumerateDevices()
const audioDevice = devices.find((device) => device.kind === 'audiooutput')
const audio = document.createElement('audio')
await audio.setSinkId(audioDevice.deviceId)
console.log(`Audio is being played on ${audio.sinkId}`)
I've got webRTC to work on my express server but I want to be able to add the stream of the user dynamically. I looked up in the simple-peer docs and found this:
var Peer = require('simple-peer') // create peer without waiting for media
var peer1 = new Peer({ initiator: true }) // you don't need streams here
var peer2 = new Peer()
peer1.on('signal', data => {
peer2.signal(data)
})
peer2.on('signal', data => {
peer1.signal(data)
})
peer2.on('stream', stream => {
// got remote video stream, now let's show it in a video tag
var video = document.querySelector('video')
if ('srcObject' in video) {
video.srcObject = stream
} else {
video.src = window.URL.createObjectURL(stream) // for older browsers
}
video.play()
})
function addMedia (stream) {
peer1.addStream(stream) // <- add streams to peer dynamically
}
// then, anytime later...
navigator.mediaDevices.getUserMedia({
video: true,
audio: true
}).then(addMedia).catch(() => {})
Peer1 sends a stream to Peer2 dynamically, but it's in the same browser. I'm using socket.io so that people are able to join different rooms. I was using this example to get me started: https://github.com/Dirvann/webrtc-video-conference-simple-peer.
If I use the github example above I understand that I'd have to put:
navigator.mediaDevices.getUserMedia(constraints).then(stream => {
console.log('Received local stream');
localVideo.srcObject = stream;
localStream = stream;
}).catch(e => alert(`getusermedia error ${e.name}`))
In a function. Call init(); then call that function later.
But in the simple-peer example it called addMedia(stream) but how would peer2 receive the stream arg if it wasn't in the same browser? In the github code 'stream' is never sent via socket.emit.
Update:
This is based on the github link.
So I remove the getUserMedia from the beginningand made the init() run on its own.
// add my stream to all peers in the room dynamically
function addMyStreamDynamic(stream) {
for (let index in peers) {
peers[index].addStream(stream);
}
}
function addMyVideoStream() {
navigator.mediaDevices.getUserMedia(constraints).then(stream => {
localVideo.srcObject = stream;
localStream = stream;
addMyStreamDynamic(stream);
}).catch(e => alert(`getUserMedia error ${e.name}`))
}
When calling addMyVideoStream it adds the stream to other peers but it's not complete. When running it before a user joins, the stream does not get sent.
Update2: The code above works but only when the initiator calls it.
It seems that dynamically adding a stream as a non-initiator is much more involved. I just created a dummy stream and later replace the track.
I need to perform the following sequence of events in my alexa skill and I am currently unable to determine how to do this.
Play some audio clip (longer than 90 seconds).
Ask the user if they are ready for the next audio clip.
Play another audio clip if requested.
Repeat as needed.
I have the following intent for #1 :
"PlayWarmUp": function(){
console.log("play warm up intent fired");
if (this.event.context.System.device.supportedInterfaces.Display) {
//play warm up video
const builder = new Alexa.templateBuilders.BodyTemplate6Builder();
const template = builder.setTextContent(makeRichText('<font size="7">Are you ready?</font><br/><br/><br/><br/>')).build();
let meta = {
title: "Warmup",
subtitle: null
};
this.response.playVideo(videoURL, meta).renderTemplate(template).hint("'Yes', or 'No', or 'Ready'");
//switch state to workout since we want to handle their responses to the are you ready screen, which is in the WORKOUT state
this.handler.state = states.WORKOUT;
this.emit(":responseReady");
} else {
// Audio only
//
console.log("Audio only for play warm up");
this.response.shouldEndSession(false);
this.response.audioPlayerPlay('ENQUEUE', warmupAudioURL, 'warmup-audio-token', 'previous-token', '0');
this.handler.state = states.WORKOUT;
audioState = states.WARMUP;
this.emit(":responseReady");
}
}
Amazon provide built in intents for audio events like this:
const audioHandlers = {
'PlaybackStarted' : function() {
console.log('Alexa begins playing the audio stream');
this.emit(':responseReady');
},
'PlaybackFinished' : function() {
console.log('The stream comes to an end');
console.log("audioState: ", audioState);
console.log("this ", this);
console.log("context: ", this.context);
if (audioState === states.WARMUP) {
console.log("audio state is warm up...");
// This doesn't work. It does nothing.
//
this.handler.state = states.WORKOUT;
this.emitWithState("WORKOUT");
}
},
'PlaybackStopped' : function() {
console.log('Alexa stops playing the audio stream');
this.emit(':responseReady');
},
'PlaybackNearlyFinished' : function() {
console.log('The currently playing stream is nearly complate and the device is ready to receive a new stream');
this.emit(':responseReady');
},
'PlaybackFailed' : function() {
console.log('Alexa encounters an error when attempting to play a stream');
this.emit(':responseReady');
}
};
The above code doesn't work. I am unable to call the "WORKOUT" intent when the audio finishes.
The documentation states:
I've also looked at using SSML, but emitWithState() fires before the speech is finished and it only allows for 90 seconds of audio.
It doesn't make sense that you can't play and audio clip and then ask the user something. This should be a basic dialogue function. Has anyone solved this issue?
Good day! I'm into video chat streaming this morning and I've bumped into a problem with the incoming ArrayBuffer which contains binary data of an audio.
Here is the code I found for playing binary audio data (Uint8Array):
function playByteArray(byteArray) {
var arrayBuffer = new ArrayBuffer(byteArray.length);
var bufferView = new Uint8Array(arrayBuffer);
for (i = 0; i < byteArray.length; i++) {
bufferView[i] = byteArray[i];
}
context.decodeAudioData(arrayBuffer, function(buffer) {
buf = buffer;
play();
});
}
// Play the loaded file
function play() {
// Create a source node from the buffer
var source = context.createBufferSource();
source.buffer = buf;
// Connect to the final output node (the speakers)
source.connect(context.destination);
// Play immediately
source.start(0);
}
Now below, I've used MediaStreamRecorder from https://github.com/streamproc/MediaStreamRecorder to record the stream from getUserMedia. This code will continuously send the recorded binary data to the server.
if (navigator.getUserMedia) {
navigator.getUserMedia({audio: true, video: true}, function(stream) {
video.src = (window.URL || window.webkitURL).createObjectURL(stream); //get this for video strewam url
video.muted = true;
multiStreamRecorder = new MultiStreamRecorder(stream);
multiStreamRecorder.canvas = {
width: video.width,
height: video.height
};
multiStreamRecorder.video = video;
multiStreamRecorder.ondataavailable = function(blobs) {
var audioReader = new FileReader();
audioReader.addEventListener("loadend", function() {
var arrBuf = audioReader.result;
var binary = new Uint8Array(arrBuf);
streamToServ.write(binary);
// streamToServ is the binaryjs client
});
audioReader.readAsArrayBuffer(blobs.audio);
};
multiStreamRecorder.start(1);
}, onVideoFail);
} else {
alert ('failed');
}
Convert the blobs produced (audio and video) to binary and send it to binaryjs which will be played on another client with this:
client.on('stream', function (stream, meta) {
stream.on('data', function(data) {
playByteArray(new Uint8Array(data));
});
});
I had no problems with transferring the binary data but the problem is there is a hiccup sound in the playback significantly on every binary data that was played. Is there something wrong on how I play the incoming ArrayBuffers? I'm also thinking of asking streamproc about this.
Thanks in advance!
Cheers.
I found a solution to this problem by making an audio buffer queueing. Most of the code is from here:
Choppy/inaudible playback with chunked audio through Web Audio API
Thanks.
Not sure if this is the problem, but perhaps instead of source.start(0), you should use source.start(time), where time is where you want to start the source. source.start(0) will start playing immediately. If your byte array comes in faster than real-time, the sources might overlap because you start them as soon as possible.
I am trying to build something, where a user can send audio instantly to many people using, socket.io, audioContext, js for the front-end and Node.js,socket.io for the server.
I can record the audio, send it to the server and send it back to other users, but I cannot play the data. I guess it must be a problem of how I send them or how I process the buffer that receives them.
I get the following error: Update!
The buffer passed to decodeAudioData contains an unknown content type.
Audio is passed fine, the buffer is created with no errors but there is no sound feedback.
The User presses record and it started recording/streaming with he following functions:
This is how it all starts:
navigator.getUserMedia({audio: true,video: false}, initializeRecorder, errorCallback);
function initializeRecorder(stream) {
var bufferSize = 2048;
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var source = audioCtx.createMediaStreamSource(stream);
var recorder = audioCtx.createScriptProcessor(bufferSize, 1, 1);
recorder.onaudioprocess = recorderProcess;
source.connect(recorder);
recorder.connect(audioCtx.destination);
recording = true;
initialized = true;
play = false;
stop = true;
}
function recorderProcess(e) {
var left = e.inputBuffer.getChannelData(0);
socket.emit('audio-blod-send', convertFloat32ToInt16(left));
}
function convertFloat32ToInt16(buffer) {
l = buffer.length;
buf = new Int16Array(l);
while (l--) {
buf[l] = Math.min(1, buffer[l])*0x7FFF;
}
return buf.buffer;
}
Then the server uses the socket to broadcast what the original sender send:
socket.on('audio-blod-send',function(data){
socket.broadcast.to(roomName).emit('audio-blod-receive', data);
});
And then the data are played: Update!
I was using audioContext.decodeData which I found out that it is only used to read/decode audio from MP3 or WAV files not streaming. With the new code no errors appear however there is no Audio feedback.
socket.on('audio-blod-receive',function(data) {
playAudio(data);
});
function playAudio(buffer)
{
var audioCtx;
var started = false;
if(!audioCtx) {
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
}
source = audioCtx.createBufferSource();
audioBuffer = audioCtx.createBuffer( 1, 2048, audioCtx.sampleRate );
audioBuffer.getChannelData( 0 ).set( buffer );
source.buffer = audioBuffer;
source.connect( audioCtx.destination );
source.start(0);
console.log(buffer);
}
P.S If anyone is interested further in what I am trying to do, feel free to contact me.