We Keep Coding

WebRTC Multiple Peer connection video is not working

So I found a way to make a peer connection multiple times by a lot..but I can't make the video work although there is no error shown...
UPDATE: DEMO but this demo is not allow working localStream so try it in your own browser index.html
First let say we have this html file
// This one is for multiple videos
<div class="item-videos">
//This first video is a start video
<video id="video1" playsinline autoplay muted></video>
//This is join videos
</div>
<div>
<button id="start"> Start </button>
<button id="join"> Join </button>
<button id="hangup"> Hang Up </button>
</div>
First I will takes the initial inputs for starter in script.js
let containers = document.querySelector('.item-videos');
const startButton = document.querySelector('#start')
const joinButton = document.querySelector("#join")
const video1 = document.querySelector('video#video1');
let localStream;
// This is the RTCPeerConnections arrays.
let pcLocals = [];
let pcRemotes = [];
const offerOptions = {
offerToReceiveAudio: 1,
offerToReceiveVideo: 1
};
const servers = {
iceServers: [
{
urls: ['stun:stun1.l.google.com:19302', 'stun:stun2.l.google.com:19302'],
},
],
iceCandidatePoolSize: 10,
};
And then let say first we will start our call server..which will be created.
So now we will make a start click then our code
...
function gotStream(stream) {
console.log('Received local stream');
video1.srcObject = stream;
localStream = stream;
joinButton.disabled = false;
}
function start() {
console.log('Requesting local stream');
startButton.disabled = true;
navigator.mediaDevices
.getUserMedia({
audio: true,
video: true
})
.then(gotStream)
.catch(e => console.log('getUserMedia() error: ', e));
}
startButton.addEventListener("click",start)
Now this is for join button in the server...let say I have let count = 0
and I will createElement each video I click the button
So our code for the join button click is
let count = 0;
joinButton.addEventListener("click",() => {
count += 1
//Creating Video Element
const addVideo = document.createElement('video')
addVideo.setAttribute('id',`video${count + 1}`)
addVideo.setAttribute('class',`try-${count + 1}`)
// Here I believe this part was my error where in the video is set up yet for the RTCPeerConnection functions.
containers.appendChild(addVideo)
const videoCall = containers.querySelectorAll('video')[count]
// Here I will create RTCPeerConnections and push it in the pcLocals and pcRemotes;
const init_localStreams = new RTCPeerConnection(servers);
const init_remoteStreams = new RTCPeerConnection(servers);
pcLocals.push(init_localStreams)
pcRemotes.push(init_remoteStreams)
console.log(pcLocals)
console.log(pcRemotes)
//Here I'm passing the stream videos in RTCPeer Arrays...
pcRemotes[count - 1].ontrack = (ev) => {
function gotRemoteStream(e,video,idx) {
if (video.srcObject !== e.streams[0]) {
video.srcObject = e.streams[0]
console.log(`pc${idx+1}: received remote stream`);
}
}
gotRemoteStream(ev,videoCall,count - 1)
}
//Here I'm passing the tracks of the video in each locals
localStream.getTracks().forEach((track) =>
{
pcLocals[count - 1].addTrack(track, localStream)
});
function onAddIceCandidateSuccess() {
console.log('AddIceCandidate success.');
}
function onAddIceCandidateError(error) {
console.log(`Failed to add ICE candidate: ${error.toString()}`);
}
function handleCandidate(candidate, dest, prefix, type) {
dest.addIceCandidate(candidate)
.then(onAddIceCandidateSuccess, onAddIceCandidateError);
console.log(`${prefix}New ${type} ICE candidate: ${candidate ? candidate.candidate : '(null)'}`);
}
function iceCallbackRemote(e,local_) {
handleCandidate(e.candidate,local_,`pc${count}: `, 'remote')
}
function iceCallbackLocal(e,remote_) {
handleCandidate(e.candidate,remote_,`pc${count}: `, 'local')
}
pcLocals[count - 1].onicecandidate = (ev) => {
iceCallbackRemote(ev,pcLocals[count - 1])
}
pcRemotes[count - 1].onicecandidate = (ev) => {
iceCallbackLocal(ev,pcRemotes[count - 1])
}
function gotDescriptionRemote(desc) {
pcRemotes[count-1].setLocalDescription(desc);
// console.log(`Answer from pc1Remote\n${desc.sdp}`);
pcLocals[count-1].setRemoteDescription(desc);
}
function gotDescriptionLocal(desc) {
pcLocals[count-1].setLocalDescription(desc);
// console.log(`Answer from pc1Remote\n${desc.sdp}`);
pcRemotes[count-1].setRemoteDescription(desc);
pcRemotes[count-1].createAnswer().then(gotDescriptionRemote,onCreateSessionDescriptionError)
}
function onCreateSessionDescriptionError(error) {
console.log(`Failed to create session description: ${error.toString()}`);
}
pcLocals[count - 1].
createOffer(offerOptions)
.then(gotDescriptionLocal, onCreateSessionDescriptionError)
})
I'm somehow doubt my if my video was not to pass yet before the RTCPeerConnection operations happening..I don't know if where my errors here... I just tried to make a multiple peerconnection that was documentary here at WEBRTC TUTORIAL

I've looked at you codesandbox code and found two issues:
The playback of your created video is never started
Your icecandidates are set on the wrong peerconnection
To adress the first problem, you need to start your playback either with autoplay or by starting it with addVideo.play(). For the autoplay solution, you can simply add:
addVideo.setAttribute("autoplay", true);
addVideo.setAttribute("playsinline", true);
To address the second problem, you need to change the passed peerconnections in the onicecandidate event handlers:
pcLocals[count - 1].onicecandidate = (ev) => {
//old: iceCallbackRemote(ev, pcLocals[count - 1]);
//new:
iceCallbackRemote(ev, pcRemotes[count - 1]);
};
pcRemotes[count - 1].onicecandidate = (ev) => {
//old: iceCallbackRemote(ev, pcRemotes[count - 1]);
//new:
iceCallbackLocal(ev, pcLocals[count - 1]);
};
The ice candidates need to be exchanged, meaning, ice candidates gathered by local must be passed to the remote and vice versa. Before, you added the ice candidates from local to your local connection, thats why it didn't work. This why the connectionState was "connecting", and never changed to "connected", as the connection was never fully connected and was still expecting ice candidate exchange.

webRTC datachannel's onopen event is not firing even with twilio TURN server

I am using twilio TURN server for webRTC peer connecting two browsers located on different sides of the world, still the connection does not open.
Log shows the local and remote descriptions are set on both sides. Audio/video tracks are also pushed and received, but the "onopen" method on either of the data channels are not firing. Below is the code extract.
create offer code
async createOffer(){
this.initiated = true
this.conn = new RTCPeerConnection(this.servers);
if (this.conn){
this.conn.ontrack = e => {
e.streams[0].getTracks().forEach(track => {
this.calleeStream?.addTrack(track);
this.logs.push('received track:' + track.label);
})
}
if (this.callerStream)
{
const s = this.callerStream;
this.callerStream.getTracks().forEach(track =>
{
this.conn?.addTrack(track,s);
this.logs.push('pushed track:' + track.label);
});
}
}
this.channel = this.conn.createDataChannel('channelX');
this.channel.onmessage = e => this.logs.push('received =>'+ e.data);
this.channel.onopen = e => {
this.logs.push('connection OPENED!!!');
this.enabletestmessage = true;
};
this.conn.onicecandidate = async e=> {
if (e.candidate===null && !this.iceCandiSent){
this.iceCandiSent = true;
this.logs.push('new ICE candidate received- reprinting SDP'+JSON.stringify(this.conn?.localDescription));
await this.dataService.createOffer(this.data.callerid,this.data.calleeid,JSON.stringify(this.conn?.localDescription));
this.logs.push('offer set in db');
this.logs.push('waiting for answer...');
}
}
const offer = await this.conn.createOffer();
await this.conn?.setLocalDescription(offer);
this.logs.push('local description (offer) set');
}
create answer code
async createAnswer(offerSDP:string){
this.initiated = true;
this.conn = new RTCPeerConnection(this.servers);
if (this.conn)
{
this.conn.ontrack = e => {
e.streams[0].getTracks().forEach(track => {
this.callerStream?.addTrack(track);
this.logs.push('received track:' + track.label);
})
}
if (this.calleeStream)
{
const s = this.calleeStream;
this.calleeStream.getTracks().forEach(track =>
{
this.conn?.addTrack(track,s);
this.logs.push('pushed track:' + track.label);
});
}
}
await this.conn.setRemoteDescription(JSON.parse(offerSDP));
this.logs.push('remote description (offer) set');
this.conn.onicecandidate = async e => {
if (e.candidate === null && !this.iceCandiSent){
this.iceCandiSent=true;
this.logs.push('new ICE candidate received- reprinting SDP'+JSON.stringify(this.conn?.localDescription));
await this.dataService.updateAnswer(this.data.callerid,this.data.calleeid,JSON.stringify(this.conn?.localDescription));
this.logs.push('answer set in db');
}
}
this.conn.ondatachannel = e => {
this.channel = e.channel;
this.channel.onmessage = e => this.logs.push('received =>'+ e.data);
this.channel.onopen = e => {
this.logs.push('connection RECEIVED!!!');
this.enabletestmessage = true;
};
}
const answer = await this.conn.createAnswer();
await this.conn.setLocalDescription(answer);
this.logs.push('local description (answer) set');
}
server side code for retrieving ice servers from Twillio
const twilio = require('twilio');
const client = twilio(<MY ACCOUNT SID>,<MY AUTH TOKEN>);
const result = await client.tokens.create();
return result.iceServers; //this is set to this.servers in the code above
Everything works when I run on two browser windows in my local machine. However even afer implementing TURN they dont work between browsers in Nepal and USA. The onopen event handlers on data channel does notfire even though local and remote descriptions are set on both sides. What am I missing ?
NOTE: signalling is done inside the onicecandidate event handler ( the line that calls dataService createOffer/updateAnswer methods)

How do I play sound in my node.js chatroom?

I have been looking up how to play sound with node.js all day, I can't use "document.getElementById" and I can't use "new Audio" either. I want it to be able to play sound when I do #everyone in my chatroom. The audio file is name "ping.mp3" and is in the same path as my main node.js file. I need some recommendations or code snippets. Thanks!
This is a code snippet of where the ping code is.
function highlight(message){
if(message == "") {
return message
}
let mentions = message.match(/#\b([A-Za-z0-9]+)\b/g)
let urlCheck1 = message.split(` `)
if (mentions === null ) { return message }
for (i = 0; i < mentions.length; i++) {
let urlCheck = urlCheck1[i].includes(`http`)
let mention = mentions[i].substring(1)
if(sesskx.has(mention) && !urlCheck) {
message = message.replace(mentions[i], `<span class="name-color">#${mention}</span>`)
} else if (mention == 'everyone') {
ping.play();
message = message.replace(mentions[i], `<span class="name-color">#${mention}</span>`)
} else if (mention == 'here') {
ping.play();
message = message.replace(mentions[i], `<span class="name-color">#${mention}</span>`)
}
else {
return message;
}
}
return message
};
I want "ping.play();" to make the sound.

This is possible but kind of tricky. You can use play-sound to take care of it for you:
var player = require('play-sound')(opts = {})
player.play('foo.mp3', function(err){
if (err) throw err
});
What it basically does is look for sound players in the environment and try to use one of them to play the mp3:
const { spawn, execSync } = require('child_process');
// List of players used
const players = [
'mplayer',
'afplay',
'mpg123',
'mpg321',
'play',
'omxplayer',
'aplay',
'cmdmp3'
];
// find a player by seeing what command doesn't error
let player;
for(const p of players) {
if (isExec(p)) {
player = p;
break;
}
}
function isExec(command) {
try{
execSync(command)
return true
}
catch {
return false
}
}
spawn(player, ['ping.mp3']);

If you are creating a node.js server, node.js is back-end/server side, so users won't be able to hear anything that happens if a sound is "played" there. So you'll have to do it client side using "new Audio()", when the client receives a message including "#everyone" (which I don't understand why you cant use, you can send the mp3 file to the client with the page.).
I also made an example for you on repl.

WebRTC manual signal DOMException unknown ufrag

I'm trying to learn about webrtc and I have some code that is meant to be run in two different firefox windows on my laptop with me copying relevant info between the developer consoles, but I run into the error DOMException: "Unknown ufrag (..fragsnippet..)" and I can't tell what I'm messing up. Anyone know what causes this?
I know that signaling servers can be setup to do what I'm doing, but I wanted to break the steps down to understand the order and inspect the result of each step. I included the source for anyone to see if there's any clear issues happening. For explanation each window will be called pc1 or pc2.
steps
pc1 calls initConnection which creates the variable localcon of
type RTCPeerConnection using no stunServers because the example is
supposed to use local ice candidates
then we copy the locOffer variable which already contains
icecandidate entries
in window pc2 i execute reactConnection which makes a localcon
for this window and then I call ingestOffer providing the copied
locOffer from the pc1 window this sets remoteDescription for the
localcon of pc2 and then creates an answer that I would like to
copy back to pc1,
the steps end there because I run into the dom exception error and
localcon for pc2 changes to iceConnectionState:fail
Any ideas about how to fix this would be very helpful!
/* new process
create a single pc, and setup binding of candidate stuff and offer stuff to be set, and trigger other parts of the process to continue,
hopefully no servers is going to work
*/
'use strict'
var localcon, remotecon, datachannel, pcconstraint, dataconstraint, recchannel, config
export function setup() {
var servers = null
pcconstraint = null
dataconstraint = null
config = {
iceServers: [],
iceTransportPolicy: "all",
iceCandidatePoolSize: "0"
}
window.ingestOffer = ingestOffer
window.handleAnswer = handleAnswer
window.desc1 = desc1
window.setAnswer = setAnswer
window.initConnection = initConnection
window.reactConnection = reactConnection
// answer will be created in desc1
}
function reactConnection() {
window.localcon = localcon = new RTCPeerConnection(config)
localcon.onicecandidate = iceCB2
// bind different channel events
localcon.ondatachannel = handleDC
}
function initConnection() {
window.localcon = localcon = new RTCPeerConnection(config)
window.datachannel = datachannel = localcon.createDataChannel('sendDataChannel', dataconstraint)
// bind the ice events
// attach the datachannel events
datachannel.onopen = openDC
datachannel.onclose = closeDC
localcon.onicecandidate = iceCB1
localcon.oniceconnectionstatechange = handleConnectionChange
window.locCanInfo = []
// start the offers
localcon.createOffer().then(
desc1,
descriptionErr
)
}
function handleConnectionChange(event) {
console.log("connection change",event);
console.log("peer connection ",event.target.iceConnectionState);
}
function openDC () {
console.log("opening first data channel");
}
function closeDC() {
console.log("closing first data channel");
}
function handleDC(event) {
console.log("handling rec channel connect")
recchannel = event.channel
recchannel.onmessage = recmessage
recchannel.onopen = recopen
recchannel.onclose = recclose
}
function recmessage(event) {
console.log("rec channel got",event.data);
}
function recopen() {
console.log("rec channel open");
}
function recclose() {
console.log("rec channel close");
}
function setAnswer(desc) {
localcon.setRemoteDescription(new RTCSessionDescription(desc)).then(()=> console.log("success"))
}
function ingestCandidate(msg) {
console.log("ingesting candidate")
let candidate = new RTCIceCandidate({
sdpMLineIndex: msg.label,
candidate: msg.candidate
})
localcon.addIceCandidate(candidate)
}
function ingestOffer(desc,ocand) {
// ?? what part of desc should be provided?
window.theirOffer = desc
console.log("setting remote desc")
localcon.setRemoteDescription(new RTCSessionDescription(desc)).then(()=> console.log("ingest success")).catch(e=> console.log("ingest error",e))
// set window otherCands and use them at add time
window.otherCands = ocand
// create an answer
localcon.createAnswer().then(handleAnswer)
}
function handleAnswer(desc) {
window.locAnswer = desc
localcon.setLocalDescription(desc).then(()=> console.log("handle answer success")).catch(e=> console.log("handle answer error",e))
// now copy and use in other window
}
function descriptionErr(e) {
console.log("error ", e)
}
// no send Data func, because we will do this from the window
function closeChannels() {
console.log("closing send");
datachannel.close()
console.log("closing rec");
recchannel.close()
localcon.close()
remotecon.close()
}
function desc1(desc) {
// assign desc to remote, as remote desc and local to local
localcon.setLocalDescription(desc)
console.log("local offer", desc.sdp)
// put the offer in text
window.locOffer = desc
}
function desc2(desc) {
// this is using the answer
console.log("answer text", desc.sdp)
remotecon.setLocalDescription(desc)
localcon.setRemoteDescription(desc)
}
function iceCB1(event) {
// this is local dealing with ice candidate
console.log("lcal ice callbac", event.candidate);
if (event.candidate != null && event.candidate.candidate != "") {
window.locCanInfo.push({
type: 'candidate',
label: event.candidate.sdpMLineIndex,
id: event.candidate.sdpMid,
candidate: event.candidate.candidate
})
//localcon.addIceCandidate(event.candidate)
}
}
function iceCB2(event) {
console.log("remote ice callback");
if (event.candidate) {
localcon.addIceCandidate(
event.candidate
).then(
iceSuccess,
iceFail
)
console.log("remote ice candidate", event.candidate, "and", event.candidate.candidate)
}
}
function iceSuccess() {
console.log("addice success");
}
function iceFail(e) {
console.log("ice fail", e);
}
function onChannelStateChange() {
let readystate = datachannel.readyState
if (readystate === "open") {
console.log("og channel ope");
} else {
console.log("og channel is not open");
}
}
function onRecChannelStateChange() {
console.log("rec channel state is ", recchannel.readyState);
}

How would I fix this code to allow me to leave one channel when switching to the next

This is what I have to switch between channels in the chat, but it does not leave the channel even when calling the .leave() function
new Twilio.Chat.Client.create(data.token).then(function(chatClient) {
$(".change-channel").on("click", function(){
if ($(this).text() == "General Button"){
if(currentChannel != "general"){
chatClient.getSubscribedChannels().then(joinChannel(chatClient, 'general','General Chat Channel'));
currentChannel.leave();
}
// if not current channel general then:
}
if ($(this).text() == "Specific Button"){
if(currentChannel != "generals"){
chatClient.getSubscribedChannels().then(joinChannel(chatClient, 'generals','Generals Chat Channel'));
currentChannel.leave();
}
// if not currentchannel generals then:
}
});
});

Twilio developer evangelist here.
I think the channels may be getting confused as you join and leave them and promises and callbacks are resolved. I would try making sure that you have left the channel first, before joining the next one and disallowing the switching of channels while the process is going on.
Something like this:
new Twilio.Chat.Client.create(data.token).then(function(chatClient) {
let currentChannel, nextChannel, nextChannelName;
let changingChannel = false;
// fired when a channel has been left completely
chatClient.on('channelLeft', channel => {
chatClient.getSubscribedChannels()
.then(joinChannel(chatClient, nextChannel, nextChannelName));
})
// fired when a channel has been successfully joined
chatClient.on('channelJoined', channel => {
currentChannel = channel;
changingChannel = false;
nextChannel = nextChannelName = null;
})
$(".change-channel").on("click", function(){
if (changingChannel) {
// already switching channels, don't do anything
return;
}
if ($(this).text() == "General Button"){
if(currentChannel.uniqueName != "general"){
changingChannel = true;
nextChannel = 'general';
nextChannelName = 'General Chat Channel';
// start the process to leave the channel
currentChannel.leave();
currentChannel = null;
}
// if not current channel general then:
}
if ($(this).text() == "Specific Button"){
if(currentChannel.uniqueName != "generals"){
changingChannel = true;
nextChannel = 'generals';
nextChannelName = 'Generals Chat Channel';
// start the process to leave the channel
currentChannel.leave();
currentChannel = null;
}
// if not currentchannel generals then:
}
});
});
Let me know if that helps at all.