Camera flipping issue -Twilio version 2.24 - node.js

The flip camera doesn't work on devices where they have more than 2 video inputs. In the first load, the video appears but when the flip camera button is clicked the application throws an error.
Expected behavior:
The camera should be flipped (environment)
Actual behavior:
It throws the following error:
error : call to getusermedia failed domexception could not start video source
Software versions:
Browser(s): Chrome
Operating System: Android (devices that I'm checking & it's not working eq. Samsung M31, Redmi note 11 T, One Plus 7T)
twilio-video.js: 2.24.0
Third-party libraries (e.g., Angular, nodejs, etc.):
Code used to start twilio stream
async startTwilioStream(twilioToken: string, localVideo: ElementRef, remoteVideo: ElementRef): Promise<void> {
console.log('startTwilioStream');
this.localVideoElement = localVideo;
this.remoteVideoElement = remoteVideo;
await this.startLocalVideo(this.localVideoElement);
this.connectOptions = {
video: false,
audio: false,
tracks: [this.localAudioTrack, this.localVideoTrack],
audioConstraints: {
mandatory: {
googAutoGainControl: false,
},
},
region: 'in1',
preferredAudioCodecs: ['opus'],
preferredVideoCodecs: ['H264'],
};
connect(twilioToken, this.connectOptions).then((twilioRoom: any) => {
console.log('twilioRoom.localParticipant ================== ', twilioRoom.localParticipant);
setTimeout(() => {
if (this.remoteVideoElement?.nativeElement) {
this.remoteVideoElement.nativeElement.muted = false;
}
}, 5000);
this.twilioRoom = twilioRoom;
console.log('this.twilioRoom vvvv', this.twilioRoom);
twilioRoom.localParticipant.setNetworkQualityConfiguration({
local: 2,
remote: 1,
});
// flip.addEventListener('change', this.updateVideoDevice);
twilioRoom.on('participantConnected', participant => {
console.log('participant Connected===============', participant);
participant.tracks.forEach((publication) => {
console.log('publication', publication);
if (publication.isSubscribed) {
const track = publication.track;
this.attachTracks([track]);
}
});
this.twilioRoom = twilioRoom;
});
twilioRoom.on('participantDisconnected', participant => {
console.log('participantDisconnected', participant);
console.log('SOME PARTICIPANT DISCONNECTED');
if ((participant.identity === 'agent-screen-share' && this.serviceUserType !== 'agent') || (participant.identity === 'consumer-screen-share' && this.serviceUserType !== 'consumer')) {
this.changeDetectionEmitter.emit('remoteScreenShareStopped');
this.isRemoteScreenShareOn = false;
} else if (participant.identity !== 'agent-screen-share' && participant.identity !== 'consumer-screen-share') {
console.log('real participant dced');
this.remoteMediaStream = null;
this.detachTracks(participant);
this.isRemoteVideoOn = false;
}
this.twilioRoom = twilioRoom;
});
twilioRoom.participants.forEach((participant) => {
participant.tracks.forEach((publication) => {
if (publication.track) {
const track = publication.track;
this.attachTracks([track]);
}
});
participant.on('trackSubscribed', (track) => {
console.log('trackSubscribed', track);
this.attachTracks([track]);
});
this.twilioRoom = twilioRoom;
});
twilioRoom.on('trackAdded', (track, participant) => {
console.log('trackAdded', track, participant);
this.attachTracks([track]);
this.twilioRoom = twilioRoom;
});
// When a Participant adds a Track, attach it to the DOM.
twilioRoom.on('trackSubscribed', (track, err, participant) => {
console.log('trackSubscribed', track);
this.sendLoaderStatus('ringing');
if ((participant.identity === 'agent-screen-share' && this.serviceUserType !== 'agent') || (participant.identity === 'consumer-screen-share' && this.serviceUserType !== 'consumer')) {
this.attachScreenShareTrack([track]);
} else if (participant.identity === 'agent-screen-share' || participant.identity === 'consumer-screen-share') {
} else {
this.attachTracks([track]);
}
this.twilioRoom = twilioRoom;
});
// When a Participant removes a Track, detach it from the DOM.
twilioRoom.on('trackRemoved', (track, participant) => {
console.log('trackRemoved', track);
this.detachTracks([track]);
this.twilioRoom = twilioRoom;
});
}, err => {
});
}
Start local video and local audio track
async startLocalVideo(localVideo: ElementRef, deviceId = 'user'): Promise<void> {
this.localVideoElement = localVideo;
const localAudioTrack = await createLocalAudioTrack({
audio: true
});
const localVideoTrack = await createLocalVideoTrack({
facingMode: deviceId
});
this.localAudioTrack = localAudioTrack;
this.localVideoTrack = localVideoTrack;
if (!this.localAudioTrack) {
alert('Audio source not found, do you hava a mic connected ?');
}
if (!this.localVideoTrack) {
alert('Video source not found, do you hava a videocam connected ?');
}
console.log('this.localVideoTrack to check', this.localVideoTrack);
this.localDisplayMediaStream = new MediaStream();
console.log('this.localVideoTrack.mediaStreamTrack to check', this.localVideoTrack.mediaStreamTrack);
this.localDisplayMediaStream.addTrack(this.localVideoTrack.mediaStreamTrack);
console.log('this.localDisplayMediaStream to check', this.localDisplayMediaStream);
this.localVideoElement.nativeElement.srcObject = this.localDisplayMediaStream;
}
Flip event listener calls on the click of switch button
const flip = document.querySelector('#flip');
flip.addEventListener('click', (e) => {
if (this.facingMode == "user") {
this.facingMode = "environment";
this.twilioService.switch(this.facingMode)
} else {
this.facingMode = "user";
this.twilioService.switch(this.facingMode)
}
});
Switch camera function calls in flip event listener
async switch(facingMode) {
console.log(this.localDisplayMediaStream);
if (this.localDisplayMediaStream) {
this.localDisplayMediaStream.getTracks().forEach(track => {
track.stop();
});
if (this.twilioRoom) {
await this.twilioRoom.localParticipant.videoTracks.forEach((track: any) => {
console.log('track', track);
track.track.stop();
});
}
}
const localVideoTrack = await createLocalVideoTrack({
facingMode: facingMode
});
this.localVideoTrack = localVideoTrack;
this.localDisplayMediaStream = new MediaStream();
this.localDisplayMediaStream.addTrack(this.localVideoTrack.mediaStreamTrack);
this.localVideoElement.nativeElement.srcObject = this.localDisplayMediaStream;
}

Related

WebDriverError: unknown error: DevToolsActivePort file doesn't exist - no fixes have worked

This seems to be a common error, and I have read through all the search results on google.
Chrome options are set as suggested in other posts
Installed Xvfb libXfont Xorg
and a half dozen other suggestions from the many other similar posts about this issue.
Ubuntu 22.04 LTS on AWS EC2
chromedriver 101.0.4951.41
selenium-webdriver 4.2.0
This is a new EC2 instance just for this project, if I could uninstall and install over again to get it working I wouldn't care..
theres a docker file for this, and I haven't used docker before but I'm now considering it as I can't get passed this issue- I've overcome the handful of issues before this but theres no fix yet that I've seen that works.
(author of this bot im trying to run suggests using chromedriver 88, so if anyone knows how to uninstall and install just that version that would be great tip)
const { exec } = require("child_process")
const webdriver = require('selenium-webdriver')
const chrome = require('selenium-webdriver/chrome')
const YoutubeDlWrap = require("youtube-dl-wrap")
const youtubeDlWrap = new YoutubeDlWrap()
class Video {
async load(url, youtube_dl, msg) {
if (this.in_loading) return
this.in_loading = true
this.driver.executeScript('video.innerHTML = null')
if (youtube_dl) {
await msg.edit("Downloading...")
.then(async msg => {
console.log("Downloading...")
const fileName = await this.download(url, msg)
url = __dirname + "/client/tmp/" + fileName
})
}
await this.driver.executeScript(`video.src='${url}'`)
.then(_ => {
console.log('Loading...')
msg.edit("Loading...")
.then(_ => {
var int1 = setInterval(() => {
is_error && clearInterval(int1)
if (this.killed) {
msg.edit(":no_entry_sign: Loading stopped")
this.in_loading = false
this.killed = false
clearInterval(int1)
clearInterval(int2)
clearInterval(int3)
}
this.driver.getCurrentUrl()
.then(url => {
if (!this.init && url === "file:///channels/#me") {
this.init = true
this.open_guild()
this.join(msg)
clearInterval(int1)
}
else if(this.init)
clearInterval(int1)
})
}, 10)
})
})
// Wait until video load
let is_load
var int2 = setInterval(() => {
this.driver.executeScript("return video.duration")
.then(result => {
if (result) {
is_load = true
this.duration = result
this.in_loading = false
msg.edit("Done, Type `*play` to start playing.")
clearInterval(int2)
}
else if (is_error)
clearInterval(int2)
})
}, 10)
// Error event
let is_error
var int3 = setInterval(() => {
this.driver.executeScript('return video_error')
.then(error_msg => {
if (error_msg) {
msg.edit(":no_entry_sign: " + error_msg)
is_error = true
this.in_loading = false
this.driver.executeScript('video_error = ""')
clearInterval(int3)
return
}
else if (is_load)
clearInterval(int3)
})
}, 10)
}
download(url, msg) {
return new Promise((resolve, reject) => {
const fileName = Date.now()
const path = "./client/tmp"
exec(`rm -rf ${path}/*`, _ => {
this.download_process = youtubeDlWrap.exec([url, "-o", `${path}/video`])
.on("progress", progress => {
//console.log(progress.percent)
})
.on("error", err => {
msg.edit(":no_entry_sign: " + err.message)
.then(_ => {
this.in_loading = false
})
})
.on("close", () => {
if (this.killed) {
msg.edit(":no_entry_sign: Downloading process killed")
this.killed = false
}
else
exec(`mv ${path}/* ${path}/${fileName}`, _ => {
resolve(fileName)
})
}).youtubeDlProcess
})
})
}
play() {
console.log("Play")
this.start()
this.driver.executeScript('video.play()')
}
pause() {
console.log("Pause")
this.driver.executeScript('video.pause()')
}
current(time=null) {
if (time) {
if (time[0] === '+' || time[0] === '-') {
this.current().then(c => {
if (!c) return
let r
c = parseFloat(c)
const s = parseInt(time.slice(1))
time[0] === '+' ?
r = c + s :
r = c - s
this.driver.executeScript(`video.currentTime = ${r}`)
})
}
else
this.driver.executeScript(`video.currentTime = ${time}`)
}
else
return this.driver.executeScript("return video.currentTime")
}
hms(sec) {
if (sec)
return new Date(sec * 1000).toISOString().substr(11, 8)
return sec
}
}
class Stream extends Video {
client_url = `file://${__dirname}/client/index.html`
constructor(token, headless=true) {
super()
const chrome_options = new chrome.Options()
headless && chrome_options.addArguments('--headless')
chrome_options.addArguments('--no-sandbox')
chrome_options.addArguments('--window-size=1920,1080')
chrome_options.addArguments('--disable-web-security')
chrome_options.addArguments('--disable-dev-shm-usage')
chrome_options.addArguments('--autoplay-policy=no-user-gesture-required')
chrome_options.addArguments('user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36')
console.log("Webdriver started")
this.driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(chrome_options).build()
this.driver.get(this.client_url)
this.driver.executeScript(`localStorage.setItem("token", '"${token}"')`)
}
open_guild() {
this.driver.executeScript(`document.querySelector('[data-list-item-id="guildsnav___${this.guild_id}"]').click()`)
}
is_full() {
return this.driver.executeScript(`
return document.querySelector("[aria-label='Channel is full']")
`)
}
is_locked() {
return this.driver.executeScript(`
return document.querySelector("[data-list-item-id='channels___${this.channel_id}']").innerHTML.includes("Voice (Locked)")
`)
}
scroll() {
this.driver.executeScript(`
var c_inject = document.getElementById("channels");
if( c_inject.scrollTop === (c_inject.scrollHeight - c_inject.offsetHeight))
c_inject.scroll(0, 0)
else
c_inject.scroll(0, c_inject.scrollTop + 250)
`)
}
join(msg) {
var intJoin = setInterval(() => {
this.driver.executeScript(`document.querySelector("[data-list-item-id='channels___${this.channel_id}']").click()`)
.then(() => {
// this.is_locked()
// .then(result => {
// if (result) {
// msg.channel.send(":no_entry_sign: Channel is locked")
// return
// }
// })
// this.is_full()
// .then(result => {
// if (result) {
// msg.channel.send(":no_entry_sign: Channel is full")
// return
// }
// })
setTimeout(() => {
this.start()
}, 1000)
clearInterval(intJoin)
})
.catch(() => this.scroll())
}, 10)
}
start() {
this.driver.executeScript(`
var streamBtn_inject = document.querySelector('[aria-label="Share Your Screen"]')
!streamBtn_inject.className.includes('buttonActive-3FrkXp') &&
streamBtn_inject.click()
`).catch(e => e)
}
stop() {
console.log("Stop")
this.init = false
this.driver.get(this.client_url)
}
}
exports.Stream = Stream

Node.js webRTC screen shareing

I have got problem with webRTC screen sharing. When one person shares his screen another person can`t see shared screen stream, and asks him for again sharing screen.I am useing node.js express server with socket.io. I am useing Google chrome. It requires HTTPS connection if it is not local.
This is web application code`
(function() {
const socket = io.connect(window.location.origin);
const localVideo = document.querySelector('.localVideo');
const remoteVideos = document.querySelector('.remoteVideos');
const peerConnections = {};
var url_string =window.location.href
var url = new URL(url_string);
var de = url.searchParams.get("key");
let room = de
let getUserMediaAttempts = 5;
let gettingUserMedia = false;
let getdisplaymedia=true;
const config = {
'iceServers': [{
'urls': ['stun:stun.l.google.com:19302']
}]
};
/** #type {MediaStreamConstraints} */
const constraints = {
audio: true,
video: { facingMode: "user" }
};
socket.on('bye', function(id) {
handleRemoteHangup(id);
});
if (room && !!room) {
socket.emit('join', room);
}
window.onunload = window.onbeforeunload = function() {
socket.close();
};
socket.on('ready', function (id) {
if (!(localVideo instanceof HTMLVideoElement) || !localVideo.srcObject) {
return;
}
const peerConnection = new RTCPeerConnection(config);
peerConnections[id] = peerConnection;
if (localVideo instanceof HTMLVideoElement) {
peerConnection.addStream(localVideo.srcObject);
}
peerConnection.createOffer()
.then(sdp => peerConnection.setLocalDescription(sdp))
.then(function () {
socket.emit('offer', id, peerConnection.localDescription);
});
peerConnection.onaddstream = event => handleRemoteStreamAdded(event.stream, id);
peerConnection.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('candidate', id, event.candidate);
}
};
});
socket.on('offer', function(id, description) {
const peerConnection = new RTCPeerConnection(config);
peerConnections[id] = peerConnection;
if (localVideo instanceof HTMLVideoElement) {
peerConnection.addStream(localVideo.srcObject);
}
peerConnection.setRemoteDescription(description)
.then(() => peerConnection.createAnswer())
.then(sdp => peerConnection.setLocalDescription(sdp))
.then(function () {
socket.emit('answer', id, peerConnection.localDescription);
});
peerConnection.onaddstream = event => handleRemoteStreamAdded(event.stream, id);
peerConnection.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('candidate', id, event.candidate);
}
};
});
socket.on('candidate', function(id, candidate) {
peerConnections[id].addIceCandidate(new RTCIceCandidate(candidate))
.catch(e => console.error(e));
});
socket.on('answer', function(id, description) {
peerConnections[id].setRemoteDescription(description);
});
function getUserMediaSuccess(stream) {
gettingUserMedia = false;
if (localVideo instanceof HTMLVideoElement) {
!localVideo.srcObject && (localVideo.srcObject = stream);
}
socket.emit('ready');
}
function handleRemoteStreamAdded(stream, id) {
const remoteVideo = document.createElement('video');
remoteVideo.srcObject = stream;
remoteVideo.setAttribute("id", id.replace(/[^a-zA-Z]+/g, "").toLowerCase());
remoteVideo.setAttribute("playsinline", "true");
remoteVideo.setAttribute("autoplay", "true");
remoteVideos.appendChild(remoteVideo);
if (remoteVideos.querySelectorAll("video").length === 1) {
remoteVideos.setAttribute("class", "one remoteVideos");
} else {
remoteVideos.setAttribute("class", "remoteVideos");
}
}
function getUserMediaError(error) {
console.error(error);
gettingUserMedia = false;
(--getUserMediaAttempts > 0) && setTimeout(getUserMediaDevices, 1000);
}
function getUserMediaDevices() {
if (localVideo instanceof HTMLVideoElement) {
if (localVideo.srcObject) {
getUserMediaSuccess(localVideo.srcObject);
} else if (!gettingUserMedia && !localVideo.srcObject) {
gettingUserMedia = true;
navigator.mediaDevices.getDisplayMedia(constraints)
.then(getUserMediaSuccess)
.catch(getUserMediaError);
}
}
}
function handleRemoteHangup(id) {
peerConnections[id] && peerConnections[id].close();
delete peerConnections[id];
document.querySelector("#" + id.replace(/[^a-zA-Z]+/g, "").toLowerCase()).remove();
if (remoteVideos.querySelectorAll("video").length === 1) {
remoteVideos.setAttribute("class", "one remoteVideos");
} else {
remoteVideos.setAttribute("class", "remoteVideos");
}
}
getUserMediaDevices();
})();
This is node.js code`
const credentials = require('./credentials');
const express = require('express');
const app = express();
let server;
let port;
if (credentials.key && credentials.cert) {
const https = require('https');
server = https.createServer(credentials, app);
port = 443;
} else {
const http = require('http');
server = http.createServer(app);
port = 1517;
}
const io = require('socket.io')(server);
const RoomService = require('./RoomService')(io);
io.sockets.on('connection', RoomService.listen);
io.sockets.on('error', e => console.log(e));
app.use(express.static(__dirname + '/public'));
app.get('*', function(req, res) {
res.sendFile(${__dirname}/public/index.html);
});
server.listen(port, () => console.log(Server is running on port ${port}));
ok man this is fixed code you can use yuu need just stream getDisplayMedia value
(function() {
const socket = io.connect(window.location.origin);
const localVideo = document.querySelector('.localVideo');
const remoteVideos = document.querySelector('.remoteVideos');
const peerConnections = {};
var url_string =window.location.href
var url = new URL(url_string);
var de = url.searchParams.get("key");
let room = de
let getUserMediaAttempts = 5;
let gettingUserMedia = false;
let getdisplaymedia=true;
/** #type {RTCConfiguration} */
const config = {
'iceServers': [{
'urls': ['stun:stun.l.google.com:19302']
}]
};
/** #type {MediaStreamConstraints} */
const constraints = {
audio: true,
video: { facingMode: "user" }
};
socket.on('full', function(room) {
alert('Room ' + room + ' is full');
});
socket.on('bye', function(id) {
handleRemoteHangup(id);
});
if (room && !!room) {
socket.emit('join', room);
}
window.onunload = window.onbeforeunload = function() {
socket.close();
};
socket.on('ready', function (id) {
if (!(localVideo instanceof HTMLVideoElement) || !localVideo.srcObject) {
return;
}
const peerConnection = new RTCPeerConnection(config);
peerConnections[id] = peerConnection;
if (localVideo instanceof HTMLVideoElement) {
peerConnection.addStream(localVideo.srcObject);
}
peerConnection.createOffer()
.then(sdp => peerConnection.setLocalDescription(sdp))
.then(function () {
socket.emit('offer', id, peerConnection.localDescription);
});
peerConnection.onaddstream = event => handleRemoteStreamAdded(event.stream, id);
peerConnection.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('candidate', id, event.candidate);
}
};
});
socket.on('offer', function(id, description) {
const peerConnection = new RTCPeerConnection(config);
peerConnections[id] = peerConnection;
if (localVideo instanceof HTMLVideoElement) {
peerConnection.addStream(localVideo.srcObject);
}
peerConnection.setRemoteDescription(description)
.then(() => peerConnection.createAnswer())
.then(sdp => peerConnection.setLocalDescription(sdp))
.then(function () {
socket.emit('answer', id, peerConnection.localDescription);
});
peerConnection.onaddstream = event => handleRemoteStreamAdded(event.stream, id);
peerConnection.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('candidate', id, event.candidate);
}
};
});
socket.on('candidate', function(id, candidate) {
peerConnections[id].addIceCandidate(new RTCIceCandidate(candidate))
.catch(e => console.error(e));
});
socket.on('answer', function(id, description) {
peerConnections[id].setRemoteDescription(description);
});
function getUserMediaSuccess(stream) {
gettingUserMedia = false;
if (localVideo instanceof HTMLVideoElement) {
!localVideo.srcObject && (localVideo.srcObject = stream);
}
socket.emit('ready');
}
function handleRemoteStreamAdded(stream, id) {
const remoteVideo = document.createElement('video');
remoteVideo.srcObject = stream;
remoteVideo.setAttribute("id", id.replace(/[^a-zA-Z]+/g, "").toLowerCase());
remoteVideo.setAttribute("playsinline", "true");
remoteVideo.setAttribute("autoplay", "true");
remoteVideos.appendChild(remoteVideo);
if (remoteVideos.querySelectorAll("video").length === 1) {
remoteVideos.setAttribute("class", "one remoteVideos");
} else {
remoteVideos.setAttribute("class", "remoteVideos");
}
}
function getUserMediaError(error) {
console.error(error);
gettingUserMedia = false;
(--getUserMediaAttempts > 0) && setTimeout(getUserMediaDevices, 1000);
}
function getUserMediaDevices() {
var constraints = { audio: true, video: { width: 1280, height: 720 } };
navigator.mediaDevices.getDisplayMedia(constraints)
.then(function(mediaStream) {
var video = document.querySelector('video');
video.srcObject = mediaStream;
video.onloadedmetadata = function(e) {
video.play();
getUserMediaSuccess(video.srcObject)
};
})
.catch(function(err) { console.log(err.name + ": " + err.message); }); // always check for errors at the end.
}
function handleRemoteHangup(id) {
peerConnections[id] && peerConnections[id].close();
delete peerConnections[id];
document.querySelector("#" + id.replace(/[^a-zA-Z]+/g, "").toLowerCase()).remove();
if (remoteVideos.querySelectorAll("video").length === 1) {
remoteVideos.setAttribute("class", "one remoteVideos");
} else {
remoteVideos.setAttribute("class", "remoteVideos");
}
}
getUserMediaDevices();
})();

WebRTC P2P with browser and Android (Flutter) not working although everything seems fine

I tried to set up a WebRTC P2P video communication with flutter, a node backend as signaling server and kurento media server.
When I run the app in combination with the Kurento Demo everything works fine but as soon as I try to implement my own backend the video stream doesn't start although the log messages indicate that everything is ok.
Please let me know if more input is required to find a solution.
Relevant code snippets
Web Frontend:
call(username) {
const wrapper = this;
const remoteVideo = this.videoOutputFactory();
if (!remoteVideo) {
console.error('videoOutput not found');
}
const options = {
'remoteVideo': document.getElementById('testVideo'),
onicecandidate(candidate) {
console.debug('onicecandidate', candidate);
wrapper.rpc.onIceCandidate(candidate);
},
mediaConstraints: wrapper.remoteMediaConstraints
};
console.debug('WebRtcWrapper.call: options', options);
return new Promise((resolve, reject) => {
console.log('Creating WebRtcPeer');
this.webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerSendrecv(options, (error) => {
if (error) {
console.error('Error while creating WebRtcPeer', error);
reject(error);
return;
}
console.log('Generating WebRtcPeer offer');
this.webRtcPeer.generateOffer((offerError, offerSdp) => {
if (offerError) {
console.error('Error while generating WebRtcPeer offer', error);
reject(error);
return;
}
this.rpc.call(username, offerSdp).then((res) => {
console.log("Got call answer - Generated-SDPOffer: " + offerSdp);
if (res.response === 'rejected') {
console.log('Call rejected by peer');
reject(res.rejectionMessage);
return;
}
console.log('Processing peer SDP answer', res.sdpAnswer);
this.webRtcPeer.processAnswer(res.sdpAnswer);
});
});
});
});
}
App
TestController._()
: _channel = IOWebSocketChannel.connect('wss://server.marcostephan.at:443') {
_peer = jsonrpc.Peer(_channel.cast<String>());
_peer.registerMethod(
'rtc.incomingCall', (jsonrpc.Parameters message) async => await _onIncomingCall(message));
_peer.registerMethod(
'rtc.offerIceCandidate', (jsonrpc.Parameters message) => _onOfferIceCandidate(message));
_peer.registerMethod(
'rtc.startCommunication', (jsonrpc.Parameters message) => _onStartCommunication(message));
_peer.registerMethod('conn.heartbeat', (jsonrpc.Parameters message) => "");
_peer.registerFallback((jsonrpc.Parameters params) =>
print('Unknown request [${params.method}]: ${params.value}'));
_peer.listen();
_peer.sendRequest("auth.login", {'username': 'john.doe', 'role': 'actor'});
_peer.sendNotification("disp.helpMe", {'category': 'spareParts'});
}
_onIncomingCall(jsonrpc.Parameters message) async {
try{
print('Incoming call from ${message['username'].value}');
if (this.onStateChange != null) {
this.onStateChange(SignalingState.CallStateNew);
}
await _createPeerConnection();
RTCSessionDescription s = await _peerConnection
.createOffer(_constraints);
_peerConnection.setLocalDescription(s);
return {
'from': message['username'].value,
'callResponse': 'accept',
'sdpOffer': s.sdp
};
}
catch(e){
print('TestController._onIncomingCall: ERROR: $e');
}
}
_onOfferIceCandidate(jsonrpc.Parameters message) {
try{
var candidateMap = message['candidate'].value;
print('Received IceCandidate $candidateMap');
if (_peerConnection != null) {
RTCIceCandidate candidate = new RTCIceCandidate(candidateMap['candidate'],
candidateMap['sdpMid'], candidateMap['sdpMLineIndex']);
_peerConnection.addCandidate(candidate);
}
}
catch(e){
print('TestController._onOfferIceCandidate: ERROR: $e');
}
}
_onStartCommunication(jsonrpc.Parameters message) {
try{
_peerConnection.setRemoteDescription(
RTCSessionDescription(message['sdpAnswer'].value, 'answer'));
}
catch(e){
print('TestController._onStartCommunication: ERROR: $e');
}
}
_createPeerConnection() async {
_localStream = await _createStream();
RTCPeerConnection pc = await createPeerConnection(_iceServers, _config);
_peerConnection = pc;
pc.addStream(_localStream);
pc.onAddStream = (stream) {
if (this.onRemoteStream != null) this.onRemoteStream(stream);
//_remoteStreams.add(stream);
};
pc.onIceConnectionState = (state) {
print(
'TestController._createPeerConnection: onIceConnectionState: $state');
};
pc.onIceCandidate = (candidate) {
_peer.sendNotification("rtc.onIceCandidate", {
'candidate': {
'sdpMLineIndex': candidate.sdpMlineIndex,
'sdpMid': candidate.sdpMid,
'candidate': candidate.candidate
}
});
};
}
Future<MediaStream> _createStream() async {
final Map<String, dynamic> mediaConstraints = {
'audio': true,
'video': {
'mandatory': {
'minWidth': '1980',
'minHeight': '1020',
'minFrameRate': '30',
},
'facingMode': 'environment',
'optional': [],
}
};
MediaStream stream = await navigator.getUserMedia(mediaConstraints);
if (this.onLocalStream != null) {
this.onLocalStream(stream);
}
return stream;
}
final Map<String, dynamic> _iceServers = {
'iceServers': [
{'url': 'stun:stun.l.google.com:19302'},
]
};
final Map<String, dynamic> _config = {
'mandatory': {},
'optional': [
{'DtlsSrtpKeyAgreement': true},
],
};
final Map<String, dynamic> _constraints = {
'mandatory': {
'OfferToReceiveAudio': true,
'OfferToReceiveVideo': true,
},
'optional': [],
};
Logs
Web Frontend
Pastebin
App
Pastebin

Failed to set remote offer sdp: Session error code: ERROR_CONTENT

I have a problem connecting peer-to-peer video from native android browser to safari 11 desktop (vice versa), here is the error:
Unhandled Promise Rejection: OperationError (DOM Exception 34): Failed to set remote offer sdp: Session error code: ERROR_CONTENT. Session error description: Failed to set remote video description send parameters..
I'm currently stuck in this issue
and here is my whole client videochat code, thanks.
import app from '../../../config';
const videoChatService = app.service('participants/video-chat');
let localVid;
let remoteVid;
let localstream;
let rtcPeerConn;
let conversationId;
let userId;
let isSdpSent = false;
let hasAddTrack;
const configuration = {
iceServers: [{
urls: 'stun:stun.l.google.com:19302',
},
{
urls: 'stun:stun.services.mozilla.com',
username: 'louis#mozilla.com',
credential: 'webrtcdemo',
},
// {
// urls: 'turn:mdn-samples.mozilla.org',
// username: 'webrtc',
// credential: 'turnserver' }
] };
function closeVideoCall() {
if (rtcPeerConn) {
rtcPeerConn.onaddstream = null;
rtcPeerConn.ontrack = null;
rtcPeerConn.onremovestream = null;
rtcPeerConn.onicecandidate = null;
if (remoteVid.srcObject) {
remoteVid.srcObject.getTracks().forEach(track => track.stop());
remoteVid.srcObject = null;
}
if (localVid.srcObject) {
localVid.srcObject.getTracks().forEach(track => track.stop());
localVid.srcObject = null;
}
rtcPeerConn.close();
rtcPeerConn = null;
}
}
// set local sdp
function setLocalSDP(desc) {
console.log('>>> setLocalSDP', rtcPeerConn);
return rtcPeerConn.setLocalDescription(desc);
}
function logError(error) {
console.log(`>>>>logError, ${error.name}: ${error.message}`);
}
function handleNegotiatedNeededEvent() {
console.log('>>>>> on negotiation called');
console.log('query >>>', conversationId, userId);
if (!isSdpSent) {
rtcPeerConn.createOffer()
.then(setLocalSDP)
.then(() => {
isSdpSent = true;
videoChatService.patch(null, {
data: {
type: 'video-offer',
message: JSON.stringify(rtcPeerConn.localDescription),
},
}, {
query: {
conversation_id: conversationId,
user_id: userId,
},
}).then().catch(e => { console.log('patch error', e); });
})
.catch(logError);
}
}
function handleRemoveStreamEvent() {
closeVideoCall();
}
function handleTrackEvent (evt) {
console.log('>>>>> going to add their stream...', evt);
remoteVid = document.getElementById('remoteStream');
if (!remoteVid.srcObject) {
remoteVid.srcObject = evt.streams[0];
}
}
function handleAddStreamEvent(evt) {
console.log('>>>>> stream added');
remoteVid = document.getElementById('remoteStream');
remoteVid.srcObject = event.stream;
}
function handleICECandidateEvent(evt) {
console.log('>>>> onicecandidate', evt);
console.log('query >>>', conversationId, userId);
if (evt.candidate) {
videoChatService.patch(null, {
data: {
type: 'new-ice-candidate',
message: JSON.stringify(evt.candidate),
},
}, {
query: {
conversation_id: conversationId,
user_id: userId,
},
});
}
}
function handleICEConnectionStateChangeEvent() {
console.log(`>>>>> ICE connection state changed to ${rtcPeerConn.iceConnectionState}`);
switch (rtcPeerConn.iceConnectionState) {
case 'closed':
case 'failed':
case 'disconnected':
console.log('>>>> disconnected');
closeVideoCall();
break;
}
}
function handleSignalingStateChangeEvent() {
console.log(`>>>>> WebRTC signaling state changed to: ${rtcPeerConn.signalingState}`);
switch (rtcPeerConn.signalingState) {
case 'closed':
console.log('>>>> closed');
closeVideoCall();
break;
}
}
function createPeerConnection() {
rtcPeerConn = new RTCPeerConnection(configuration);
console.log('>>>>> create peer connection', rtcPeerConn);
hasAddTrack = (rtcPeerConn.addTrack !== undefined);
rtcPeerConn.onicecandidate = handleICECandidateEvent;
rtcPeerConn.onnegotiationneeded = handleNegotiatedNeededEvent;
rtcPeerConn.oniceconnectionstatechange = handleICEConnectionStateChangeEvent;
rtcPeerConn.onsignalingstatechange = handleSignalingStateChangeEvent;
rtcPeerConn.onremovestream = handleRemoveStreamEvent;
if (hasAddTrack) {
rtcPeerConn.ontrack = handleTrackEvent;
} else {
rtcPeerConn.onaddstream = handleAddStreamEvent;
}
}
function handleGetUSerMediaError(e) {
switch (e.name) {
case 'NotFoundError':
alert('Unable to open your call because no camera and/or microphone were found.');
break;
case 'SecurityError':
case 'PermissionDeniedError':
// Do nothing; this is the same as the user canceling the call.
break;
default:
alert(`Error opening your camera and/or microphone: ${e.message}`);
break;
}
}
// add video to local and add to track
function gotStream(stream) {
console.log('>>>> gotStream', stream);
localVid.srcObject = stream;
localstream = stream;
if (hasAddTrack) {
stream.getTracks().forEach(track => rtcPeerConn.addTrack(track, localstream));
} else {
rtcPeerConn.addStream(localstream);
}
}
// start signaling
export function startSignaling(conversation_id, user_id) {
localVid = document.getElementById('localStream');
remoteVid = document.getElementById('remoteStream');
console.log('>>>>> startSignaling');
conversationId = conversation_id;
userId = user_id;
return () => {
if (!rtcPeerConn) {
createPeerConnection();
navigator.mediaDevices.getUserMedia({
audio: true,
video: {
facingMode: 'user',
},
})
.then(gotStream)
.catch(handleGetUSerMediaError);
}
};
}
export function handleVideoOfferMsg(conversation_id, user_id, message) {
console.log('>>>>> handleVideoOfferMsg');
localstream = null;
conversationId = conversation_id;
userId = user_id;
localVid = document.getElementById('localStream');
remoteVid = document.getElementById('remoteStream');
return () => {
createPeerConnection();
console.log('query >>>', conversationId, userId);
const sdp = new RTCSessionDescription(message);
// sdp.sdp = sdp.replace('a=setup:active', 'a=setup:passive');
rtcPeerConn.setRemoteDescription(sdp)
.then(() => (
navigator.mediaDevices.getUserMedia({
audio: true,
video: {
facingMode: 'user',
},
})
))
.then(gotStream)
.then(() => (
rtcPeerConn.createAnswer()
))
.then(setLocalSDP)
.then(() => {
videoChatService.patch(null, {
data: {
type: 'video-answer',
message: JSON.stringify(rtcPeerConn.localDescription),
},
}, {
query: {
conversation_id: conversationId,
user_id: userId,
},
}).then().catch(e => { console.log('patch error', e); });
});
};
}
export function handleVideoAnswerMsg(message) {
console.log('>>>>> handle video answer message', message);
return () => {
const sdp = new RTCSessionDescription(message);
rtcPeerConn.setRemoteDescription(sdp)
.catch(logError);
};
}
// Adding ice candidate
export function addIceCandidate(message) {
console.log('>>>> addIceCandidate', message);
return () => {
const candidate = new RTCIceCandidate(message);
rtcPeerConn.addIceCandidate(candidate)
.then(() => {
console.log('>>> candidate added ');
})
.catch(e => {
console.log('Error candidate', e);
});
};
}
There are two different issues making a WebRTC connection between Chrome on Android and iOS/Safari not working:
1) No H.264 implementation on device
Chrome for Android has only a hardware implementation for H.264 and there is no software implementation. At this moment H.264 works only with devices with a processor of Qualcomm (Kitkat and later) or Samsung Exynos (Lollipop and later). Since Apple only supports H.264 other Android devices can't connect with iOS and Safari.
2) There is bug in Chrome for Android:
Chrome Android does not offer/answer H.264 Constrained Baseline Profile
Because Apple only supports H.264, Android/Chrome can not connect with iOS at this moment.
This problem will be solved in Chrome Android 65 (now Canary). See this for more information.
I see your error message that is exactly this bug so I am pretty sure this is the problem. But in the end it doesn't matter. But you should be aware of both problems.
This might be related to this issue.
Chrome on Android does not always support H264, Safari does only support H264.
To verify it is a video codec issue as mentioned by the other answers you can adjust the stream constraints:
navigator.mediaDevices.getUserMedia({
video: false,
audio: true
})
If the ICE handling succeeds after this change it is likely the SDP contained a codec that is not supported on the other side.
Then you should i.e. extend your code to fallback to audio only after displaying an error message in your GUI.

WebRTC: One to one Audio call not working in different machine

I am trying to implement a one to one audio call using webRTC (signalling using websockets) . But it works when i try it in one system using multiple tabs of chrome (localhost). When I try to hit my server from another machine it does initial handshakes , but call doesn't happen.
But when i try to change the tag to and changed the constraints to video constraints . it works even if the we try to access from other machine (i.e video call works ).
I initially thought it was because if firewall but when video call worked I was puzzled .
Here is my code:
// Constraints to get audio stream only
$scope.constraints = {
audio: {
mandatory: {
googEchoCancellation: true
},
optional: []
},
video:false
};
navigator.getUserMedia = navigator.getUserMedia ||
navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
// success Callback of getUserMedia(), stream variable is the audio stream.
$scope.successCallback = function (stream) {
if (window.URL) {
myVideo.src = window.URL.createObjectURL(stream); // converting media stream to Blob URL.
} else {
myVideo.src = stream;
}
//attachMediaStream(audioTag, stream);
localStream = stream;
if (initiator)
maybeStart();
else
doAnswer();
};
// failure Callback of getUserMedia()
$scope.failureCallback = function (error) {
console.log('navigator.getUserMedia Failed: ', error);
};
var initiator, started = false;
$("#call").click(function () {
socket.emit("message", undefined);
initiator = true;
navigator.getUserMedia($scope.constraints, $scope.successCallback, $scope.failureCallback);
});
var channelReady = false;
socket.on('message', function (data) {
channelReady = true;
if (data) {
if (data.type === 'offer') {
if (!initiator) {
$("#acceptCall").show();
$("#acceptCall").click(function(){
if (!initiator && !started) {
var pc_config = {
iceServers: [
{ url: "stun:stun.l.google.com:19302" },
{ url: "turn:numb.viagenie.ca", credential: "drfunk", username: "toadums#hotmail.com"}
]
};
pc = new webkitRTCPeerConnection(pc_config);
pc.onicecandidate = onIceCandidate;
pc.onaddstream = onRemoteStreamAdded;
}
pc.setRemoteDescription(new RTCSessionDescription(data));
$scope.acceptCall();
});
}
} else if (data.type === 'answer' && started) {
pc.onaddstream = onRemoteStreamAdded;
pc.setRemoteDescription(new RTCSessionDescription(data));
} else if (data.type === 'candidate' && started) {
var candidate = new RTCIceCandidate({
sdpMLineIndex: data.label,
candidate: data.candidate
});
pc.addIceCandidate(candidate);
} else if (data.type === 'bye' && started) {
console.log("Bye");
}
}
});
function onRemoteStreamAdded(event) {
othersVideo.src = URL.createObjectURL(event.stream);
};
var sdpConstraints = {
'mandatory': {
'OfferToReceiveAudio': true,
'OfferToReceiveVideo': false
}
};
function doAnswer() {
pc.addStream(localStream);
pc.createAnswer(gotDescription,null,sdpConstraints);
}
function gotDescription(desc) {
pc.setLocalDescription(desc);
socket.send(desc);
}
function maybeStart() {
if (!started && localStream && channelReady)
createPeerConnection();
pc.addStream(localStream);
started = true;
if (initiator)
doCall();
}
$scope.acceptCall = function () {
navigator.getUserMedia($scope.constraints, $scope.successCallback, $scope.failureCallback);
}
function createPeerConnection() {
var pc_config = {
iceServers: [
{ url: "stun:stun.l.google.com:19302" },
{ url: "turn:numb.viagenie.ca", credential: "drfunk", username: "toadums#hotmail.com"}
]
};
pc = new webkitRTCPeerConnection(pc_config);
pc.onicecandidate = onIceCandidate;
console.log("Created RTCPeerConnnection with config:\n" + " \"" +
JSON.stringify(pc_config) + "\".");
};
function doCall() {
$scope.caller = true;
pc.createOffer(setLocalAndSendMessage,null,sdpConstraints);
};
function setLocalAndSendMessage(sessionDescription) {
pc.setLocalDescription(sessionDescription);
socket.send(sessionDescription);
}
function onIceCandidate(event) {
if (event.candidate) {
socket.emit('message', {
type: 'candidate',
label: event.candidate.sdpMLineIndex,
id: event.candidate.sdpMid,
candidate: event.candidate.candidate
});
} else {
console.log("End of candidates.");
}
}
If navigator.mediaDevices is undefined, this because work only in secure context (https)
see:
https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia

Resources