I'm developing a video conference with WebRTC in a local network, so I use only one signaling server to exchage SDP data. As I understand, I also need to exchange ICE candidates, but I don't know how to generate them. Thanks.
You can get the generated iceCandidate by setting the peerConnection.onicecandidate event.
(async () => {
const pc = new RTCPeerConnection();
pc.onicecandidate = evt => {
console.log(evt.candidate?.candidate);
};
const stream = await navigator.mediaDevices.getUserMedia({video:true});
stream.getTracks().forEach(track => pc.addTrack(track, stream));
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
})();
Do you create a configuration which contains at least one ice server url, and then use this configuration to create your RTCPeerConnection instance? When you set an ice server url, the 'icecandidate' event should be fired.
const configuration = {
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
};
const pc = new RTCPeerConnection(configuration);
pc.addEventListener('icecandidate', event => {
if (event.candidate) {
console.log('icecandidate received: ', event.candidate);
}
});
Related
I created a discord bot with discord.js v13, I get trouble with converting the opus packet to other file types, even the discord.js official examples haven't updated for discord.js v13, I got no idea to deal with it, here is part of my code
async function record(interaction, opts = {}) {
//get voice connection, if there isn't one, create one
let connection = getVoiceConnection(interaction.guildId);
if (!connection) {
if (!interaction.member.voice.channel) return false;
connection = joinVoice(interaction.member.voice.channel, interaction)
}
const memberId = interaction.member.id;
//create the stream
const stream = connection.receiver.subscribe(memberId, {
end: {
behavior: EndBehaviorType.Manual
}
});
//create the file stream
const writableStream = fs.createWriteStream(`${opts.filename || interaction.guild.name}.${opts.format || 'opus'}`);
console.log('Created the streams, started recording');
//todo: set the stream into client and stop it in another function
return setTimeout(() => {
console.log('Creating the decoder')
let decoder = new prism.opus.Decoder();
console.log('Created');
stream.destroy();
console.log('Stopped recording and saving the stream');
stream
.pipe(writableStream)
stream.on('close', () => {
console.log('Data Stream closed')
});
stream.on('error', (e) => {
console.error(e)
});
}, 5000);
}
Try setting frameSize, channels and rate for the Decoder:
const opusDecoder = new prism.opus.Decoder({
frameSize: 960,
channels: 2,
rate: 48000,
})
Also not sure if it is intended, but you seem to destroy the stream just before you pipe it into writable stream.
Here is my example that gives stereo 48kHz signed 16-bit PCM stream:
const writeStream = fs.createWriteStream('samples/output.pcm')
const listenStream = connection.receiver.subscribe(userId)
const opusDecoder = new prism.opus.Decoder({
frameSize: 960,
channels: 2,
rate: 48000,
})
listenStream.pipe(opusDecoder).pipe(writeStream)
You can then use Audacity to play the PCM file. Use File -> Import -> Raw Data...
I am trying to send images captured from a canvas to my NodeJS backend server using the WebRTC data channel. That is I am trying to make my server a peer. But for some reason, I am unable to establish a connection.
Client Side
async function initChannel()
{
const offer = await peer.createOffer();
await peer.setLocalDescription(offer);
const response = await fetch("/connect", {
headers: {
'Content-Type': 'application/json',
},
method: 'post',
body: JSON.stringify({ sdp: offer, id: Math.random() })
}).then((res) => res.json());
peer.setRemoteDescription(response.sdp);
const imageChannel = peer.createDataChannel("imageChannel", { ordered: false, maxPacketLifeTime: 100 });
peer.addEventListener("icecandidate", console.log);
peer.addEventListener("icegatheringstatechange",console.log);
// drawCanvas function draws images got from the server.
imageChannel.addEventListener("message", message => drawCanvas(remoteCanvasCtx, message.data, imageChannel));
// captureImage function captures and sends image to server using imageChannel.send()
imageChannel.addEventListener("open", () => captureImage(recordCanvasCtx, recordCanvas, imageChannel));
}
const peer = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.stunprotocol.org:3478" }] });
initChannel();
Here both captureImage and drawCanvas are not being invoked.
Server Side
import webrtc from "wrtc"; // The wrtc module ( npm i wrtc )
function handleChannel(channel)
{
console.log(channel.label); // This function is not being called.
}
app.use(express.static(resolve(__dirname, "public")))
.use(bodyParser.json())
.use(bodyParser.urlencoded({ extended: true }));
app.post("/connect", async ({ body }, res) =>
{
console.log("Connecting to client...");
let answer, id = body.id;
const peer = new webrtc.RTCPeerConnection({ iceServers: [{ urls: "stun:stun.stunprotocol.org:3478" }] });
await peer.setRemoteDescription(new webrtc.RTCSessionDescription(body.sdp));
await peer.setLocalDescription(answer = await peer.createAnswer());
peer.addEventListener("datachannel",handleChannel)
return res.json({ sdp: answer });
});
app.listen(process.env.PORT || 2000);
Here the post request is handled fine but handleChannel is never called.
When I run this I don't get any errors but when I check the connection status it shows "new" forever. I console logged remote and local description and they seem to be all set.
What am I doing wrong here?
I am pretty new to WebRTC and I am not even sure if this is the correct approach to continuously send images (frames of user's webcam feed) to and back from the server, if anyone can tell me a better way please do.
And one more thing, how can I send image blobs ( got from canvas.toBlob() ) via the data channel with low latency.
I finally figured this out with the help of a friend of mine. The problem was that I have to create DataChannel before calling peer.createOffer(). peer.onnegotiationneeded callback is only called once the a channel is created. Usually this happens when you create a media channel ( either audio or video ) by passing a stream to WebRTC, but here since I am not using them I have to to this this way.
Client Side
const peer = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.l.google.com:19302" }] });
const imageChannel = peer.createDataChannel("imageChannel");
imageChannel.onmessage = ({ data }) =>
{
// Do something with received data.
};
imageChannel.onopen = () => imageChannel.send(imageData);// Data channel opened, start sending data.
peer.onnegotiationneeded = initChannel
async function initChannel()
{
const offer = await peer.createOffer();
await peer.setLocalDescription(offer);
// Send offer and fetch answer from the server
const { sdp } = await fetch("/connect", {
headers: {
"Content-Type": "application/json",
},
method: "post",
body: JSON.stringify({ sdp: peer.localDescription }),
})
.then(res => res.json());
peer.setRemoteDescription(new RTCSessionDescription(sdp));
}
Server
Receive offer from client sent via post request. Create an answer for it and send as response.
app.post('/connect', async ({ body }, res) =>
{
const peer = new webrtc.RTCPeerConnection({
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
});
console.log('Connecting to client...');
peer.ondatachannel = handleChannel;
await peer.setRemoteDescription(new webrtc.RTCSessionDescription(body.sdp));
await peer.setLocalDescription(await peer.createAnswer());
return res.json({ sdp: peer.localDescription });
});
The function to handle data channel.
/**
* This function is called once a data channel is ready.
*
* #param {{ type: 'datachannel', channel: RTCDataChannel }} event
*/
function handleChannel({ channel })
{
channel.addEventListener("message", {data} =>
{
// Do something with data received from client.
});
// Can use the channel to send data to client.
channel.send("Hi from server");
}
So here is what happens :
Client creates a Data-Channel.
Once data channel is created onnegotiationneeded callback is called.
The client creates an offer and sends it to the server (as post request).
Server receives the offer and creates an answer.
Server sends the answer back to the client (as post response).
Client completes the initialization using the received answer.
ondatachannel callback gets called on the server and the client.
I have used post request here to exchange offer and answer but it should be fairly easy to do the same using Web Socket if that is what you prefer.
I have a basic webRTC application that supports video/audio communication and file sharing between two peers, The app runs as intended when I open it on Mozilla Firefox but when I run it on Google Chrome the onicecandidate returns null
My RTCPeerConnection
myConnection = new RTCPeerConnection();
Setting up the peer connection
myConnection.createOffer().then(offer => {
currentoffer = offer
myConnection.setLocalDescription(offer);
})
.then(function () {
myConnection.onicecandidate = function (event) {
console.log(event.candidate);
if (event.candidate) {
send({
type: "candidate",
candidate: event.candidate
});
}
};
send({
type: "offer",
offer: currentoffer
});
})
.catch(function (reason) {
alert("Problem with creating offer. " + reason);
});
On Mozilla Firefox you can see in the console log all the ICE candidates that are collected on each "onicecandidate" event
On Chrome the output is null
You should pass options object when calling createOffer() method, e.g.:
myConnection = new RTCPeerConnection();
var mediaConstraints = {
'offerToReceiveAudio': true,
'offerToReceiveVideo': true
};
myConnection.createOffer(mediaConstraints).then(offer => {
currentoffer = offer
myConnection.setLocalDescription(offer);
})
...// the rest of you code goes here
Alternatively, you can specify RTCRtpTransceiver before creating an offer:
myConnection = new RTCPeerConnection();
myConnection.addTransceiver("audio");
myConnection.addTransceiver("video");
myConnection.createOffer().then(offer => {
currentoffer = offer
myConnection.setLocalDescription(offer);
})
...// the rest of you code goes here
Sources:WebRTC 1.0MDN RTCPeerConnection.createOffer()MDN RTCPeerConnection.addTransceiver()Example -- GitHub
You have to pass STUN/TURN servers when create a peer connection.
Otherwise you will only local candidates and hence will be able to connect locally only
var STUN = {
'url': 'stun:stun.l.google.com:19302',
};
var iceServers =
{
iceServers: [STUN]
};
var pc = new RTCPeerConnection(iceServers);
For a cross-platform app project using Meteor framework, I'd like to record microphone inputs and extract speech thanks to Google Speech API
Following Google documentation, I'm more specifically trying to build an audio stream to feed the Google Speech client.
On client side, a recording button triggers the following startCapture function (based on cordova audioinput plugin):
export var startCapture = function () {
try {
if (window.audioinput && !audioinput.isCapturing()) {
setTimeout(stopCapture, 20000);
var captureCfg = {
sampleRate: 16000,
bufferSize: 2048,
}
audioinput.start(captureCfg);
}
}
catch (e) {
}
}
audioinput events allow me to get chunks of audio data as it is recorded:
window.addEventListener('audioinput', onAudioInputCapture, false);
var audioDataQueue = [];
function onAudioInputCapture(evt) {
try {
if (evt && evt.data) {
// Push the data to the audio queue (array)
audioDataQueue.push(evt.data);
// Here should probably be a call to a Meteor server method?
}
}
catch (e) {
}
}
I'm struggling to convert the recorded audio data to some ReadableStream, that I would pipe to Google Speech API client on server side.
const speech = require('#google-cloud/speech');
const client = new speech.SpeechClient();
const request = {
config: {
encoding: "LINEAR16",
sampleRateHertz: 16000,
languageCode: 'en-US',
},
interimResults: true,
};
export const recognizeStream = client
.streamingRecognize(request)
.on('error', console.error)
.on('data', data =>
console.log(data.results)
);
I tried the following approach, but it doesn't feel like the right way to proceed:
const Stream = require('stream')
var serverAudioDataQueue = [];
const readable = new Stream.Readable({
objectMode: true,
});
readable._read = function(n){
this.push(audioDataQueue.splice(0, audioDataQueue.length))
}
readable.pipe(recognizeStream);
Meteor.methods({
'feedGoogleSpeech': function(data){
data.forEach(item=>serverAudioDataQueue.push(item));
},
...
});
Any insight on this?
What I want to achieve is the same real-time transcript process as Web Speech API but using Google Cloud Speech API.
The main goal is to transcribe live recording through an Electron app with Speech API using gRPC protocol.
This is a simplified version of what I implemented:
const { desktopCapturer } = window.require('electron');
const speech = require('#google-cloud/speech');
const client = speech.v1({
projectId: 'my_project_id',
credentials: {
client_email: 'my_client_email',
private_key: 'my_private_key',
},
});
desktopCapturer.getSources({ types: ['window', 'screen'] }, (error, sources) => {
navigator.mediaDevices
.getUserMedia({
audio: true,
})
.then((stream) => {
let fileReader = new FileReader();
let arrayBuffer;
fileReader.onloadend = () => {
arrayBuffer = fileReader.result;
let speechStreaming = client
.streamingRecognize({
config: {
encoding: speech.v1.types.RecognitionConfig.AudioEncoding.LINEAR16,
languageCode: 'en-US',
sampleRateHertz: 44100,
},
singleUtterance: true,
})
.on('data', (response) => response);
speechStreaming.write(arrayBuffer);
};
fileReader.readAsArrayBuffer(stream);
});
});
The error response from Speech API is that the audio stream is too slow and we are not sending it in real-time.
I feel that the reason is that I passed the stream without any formatting or object initialization so the streaming recognition cannot be performed.
This official sample project on Github appears to match what you're looking for: https://github.com/googleapis/nodejs-speech/blob/master/samples/infiniteStreaming.js
This application demonstrates how to perform infinite streaming using the streamingRecognize operation with the Google Cloud Speech API.
See also my comment for an alternative in Electron, using OtterAI's transcription service. (it's the approach I'm going to try soon)
You may use node-record-lpcm16 module to record audio and pipe directly to a speech recognition system like Google.
In the repository, there is an example using wit.ai.
For Google Speech recognition, you may use something like that:
'use strict'
const { SpeechClient } = require('#google-cloud/speech')
const recorder = require('node-record-lpcm16')
const RECORD_CONFIG = {
sampleRate: 44100,
recorder: 'arecord'
}
const RECOGNITION_CONFIG = {
config: {
sampleRateHertz: 44100,
language: 'en-US',
encoding: 'LINEAR16'
},
interimResults: true
}
const client = new SpeechClient(/* YOUR CREDENTIALS */)
const recognize = () => {
client
.streamingRecognize(RECOGNITION_CONFIG)
.on('error', err => {
console.error('Error during recognition: ', err)
})
.once('writing', data => {
console.log('Recognition started!')
}
.on('data', data => {
console.log('Received recognition data: ', data)
}
}
const recording = recorder.record(RECORD_CONFIG)
recording
.stream()
.on('error', err => {
console.error('Error during recognition: ', err)
.pipe(recognize)