I'm streaming video file from server to client. On the client side i receive Uint8Array and i don't know as use this array for streaming video.
what data format to convert it to and how to send it as a video tag so that the file can be played ?
The task is to stream an rtsp stream, but first I want to try streaming one video file
server
io.on('connection', socket => {
console.log('conn')
let stream = ss.createStream();
ss(socket).emit('stream', stream);
let readStream = fs.createReadStream(mp4Path);
readStream.pipe(stream);
socket.on('disconnect', () => {
console.log('user disconnect')
})
})
client
const socket = io();
ss(socket).on('stream', function(stream) {
var binaryString = "";
stream.on('data', function(data) {
for(var i = 0; i < data.length; i++) {
binaryString += String.fromCharCode(data[i]);
}
});
stream.on('end', function(data) {
binaryString = "";
});
});
Related
I have a buffer that contain the voice of the person that in media stream, i and I send it from JavaScript to NodeJS using socket.io
I need to convert that buffer to text (like speech to text, but the voice stored as buffer coming from media stream)
There is a helper function I used (in nodejs see below) that convert from/to buffer/arrayBuffer
and there is a package called node-blob that convert buffer to audio blob
but I search a lot how convert audio or even buffer to text, but I failed
any help, code or package that may help to convert it to text ?
JavaScript
navigator.mediaDevices
.getUserMedia({
video: true,
audio: true,
})
.then((stream) => {
setSrcVideo(stream);
const mediasStream = new MediaStream();
mediasStream.addTrack(stream.getVideoTracks()[0]);
mediasStream.addTrack(stream.getAudioTracks()[0]);
const mediaRecorder = new MediaRecorder(mediasStream);
socket.emit('ready');
mediaRecorder.addEventListener('dataavailable', (event) => {
if (event.data && event.data.size > 0) {
socket.emit('send-chunks', event.data);
}
});
socket.on('start-recording', () => {
mediaRecorder.start(1000);
});
});
and I receive that buffer bysocket.on('send-chunks') in NodeJS like this
NodeJS
// connection to socket.io
io.on('connection', (socket) => {
socket.on('ready', () => {
socket.emit('start-recording');
});
socket.on('send-chunks', (chunks) => {
// covert to text
});
});
// helper functions
const toArrayBuffer = (buffer) => {
const arrayBuffer = new ArrayBuffer(buffer.length);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return arrayBuffer;
};
const toBuffer = (arrayBuffer) => {
const buffer = Buffer.alloc(arrayBuffer.byteLength);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < buffer.length; ++i) {
buffer[i] = view[i];
}
return buffer;
};
I have wav file and i went to convert to AudioBuffer because i have some function that only take audio buffer as input how can i get audio buffer from audio file? i've already searched the internet and all i got was converting from audio buffer to wav not what i wanted, and i try to feed the function with Base64 data of the audio, no luck
i've already write this function but it look like it doesn't gives me what i wanted
const getbuffer = async (url) => {
try {
var result = await axios
.get(url, {
responseType: 'arraybuffer'
})
.then(response => new Buffer.from(response.data, 'binary'))
return result
}catch (e) {
return {error: e};
}
}
the buffer i want is to feed to this function so it can change the audio for me
async function alienSoundTransform(audioBuffer) {
let ctx = offlineContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);
let source = ctx.createBufferSource();
source.buffer = audioBuffer;
let oscillator = ctx.createOscillator();
oscillator.frequency.value = 5;
oscillator.type = 'sine';
let oscillatorGain = ctx.createGain();
oscillatorGain.gain.value = 0.05;
let delay = ctx.createDelay();
delay.delayTime.value = 0.05;
// source --> delay --> ctx.destination
// oscillator --> oscillatorGain --> delay.delayTime --> ctx.destination
source.connect(delay);
delay.connect(ctx.destination);
oscillator.connect(oscillatorGain);
oscillatorGain.connect(delay.delayTime);
oscillator.start();
source.start();
let outputAudioBuffer = await ctx.startRendering();
return outputAudioBuffer;
}
In my function connect(id): I start a call using var call = this.peer.call(id, this.mediastream); then I use
call.on('stream', function(stream)
{
// `stream` is the MediaStream of the remote peer.
// Here you'd add it to an HTML video/canvas element.
peerVideo.srcObject = stream;
console.log("Call stream has started! CLinet end!!!!!" + stream);
output.innerHTML = "Streaming audio from other client...";
});
To get the audio from the peer. This call.on stream is never executed so that means the call is not returning anything.
On the peer side of things they can hear the caller just fine. I'm not sure what is wrong!
Here is the full code:
const video = document.getElementById("video");
const peerVideo = document.getElementById("peervideo");
const output = document.getElementById("output");
const peerText = document.getElementById("peerid");
const peerButton = document.getElementById("peersubmit");
var promise = null;
var mediastream = null;
var peer = null;
var myId = null;
async function init()
{
//init the media devices
initMediaDevices();
//init peer
initPeer();
//Add timeout to ensure the peer.on method had time to get id from server
setTimeout(() => { console.log("My ID is: " + this.myId); output.innerHTML = "My ID: " + formatString(this.myId)}, 2000);
}
function initMediaDevices()
{
//Setup stream for usermedia
try
{
this.promise = navigator.mediaDevices.getUserMedia({audio: true})
promise.then((stream) =>
{
setStream(stream);
//video.srcObject = stream;
console.log(stream);
})
output.innerHTML = "Audio was established!";
}
catch(err)
{
console.log(err)
output.innerHTML = "Audio Failed!"
}
}
function initPeer()
{
this.peer = new Peer();
this.peer.on('open', function(id)
{
setID(id)
});
peer.on('call', function(call)
{
// Answer the call, providing our mediaStream
call.answer(this.mediastream);
console.log(call + "-------------------- got here peer call");
output.innerHTML = "Connected to Peer";
call.on('stream', function(stream)
{
// `stream` is the MediaStream of the remote peer.
// Here you'd add it to an HTML video/canvas element.
peerVideo.srcObject = stream;
console.log("Call stream has started! Peer end");
output.innerHTML = "Streaming audio from other client...";
});
});
}
function setID(id)
{
this.myId = id;
}
function setStream(stream)
{
this.mediastream = stream;
console.log("Media Stream Set! " + this.mediastream);
}
function connect(id)
{
var call = this.peer.call(id, this.mediastream);
call.on('stream', function(stream)
{
// `stream` is the MediaStream of the remote peer.
// Here you'd add it to an HTML video/canvas element.
peerVideo.srcObject = stream;
console.log("Call stream has started! CLinet end!!!!!" + stream);
output.innerHTML = "Streaming audio from other client...";
});
console.log(call + "----------------------" + this.mediastream + " This is the person that connected");
}
init();
//Event listeners
peerButton.addEventListener("click", () =>
{
let id = peerText.value;
console.log("Button Pressed!")
connect(id);
});
//unrelated
function formatString(string)
{
var newString = "";
for (var i = 0; i < string.length; i++)
{
var letter = string.charAt(i);
if(isNaN(letter))
{
newString += letter.fontcolor("red");
}
else
{
newString += letter;
}
}
return newString;
}
And the html
<video id="video" autoplay>Video Stream no available</video>
<video id="peervideo" autoplay>Video Stream no available</video>
<h3 id="output"></h3>
<input type="text" id="peerid">
<input type="submit" value="Submit" id="peersubmit">
I have same issue.
When you decide to set a debugger and make breaking points around
var call = this.peer.call(id, this.mediastream);
call.on('stream', function(s
You will receive the stream, sounds like you need to put some setTimeout to prevent this behaviour, and I know it works... but its bad practice
Add timeout to connect function it will work
your connect(id) is triggered before the new user completes navigator.getUserMedia();
Actually it's because you executed call.on before navigator.mediaDevices.getUserMedia.
so try use await initMediaDevices to make sure initPeer();
execute afterwards.
I am trying to send audio file (or chunk) via socket.io.
I tried:
// server
socket.on('client-get-audio', function (data) {
//data.fname - name of requested file
var buffer = fs.readFileSync(data.fname, null);
// try 2 - read arraybuffer
//var buffer = fs.readFileSync(data.fname, null).buffer;
buffer = new Uint8Array(buffer);
socket.emit('audio-data', {count: 1, buff: buffer});
});
I can not decode data:
// client
// ctx - AudioContext
// source - buffer source
socket.on('audio-data', function(data){
ctx.decodeAudioData(data.buff, decoded => {
source.buffer = decoded;
source.start(0);
});
I will not describe all my attempts, there were a lot of them.
update
working variant
// server
socket.on('client-get-audio', function (data) {
//data.fname - name of requested file
var buffer = fs.readFileSync(data.fname, null);
socket.emit('audio-data', {count: 1, buff: buffer});
});
// client
function toArrayBuffer(buf) {
var ab = new ArrayBuffer(buf.length);
var view = new Uint8Array(ab);
for (var i = 0; i < buf.length; ++i) {
view[i] = buf[i];
}
return ab;
}
// ctx - AudioContext
// source - buffer source
socket.on('audio-data', function(data){
ctx.decodeAudioData(toArrayBuffer(data.buff.data), decoded => {
source.buffer = decoded;
source.start(0);
});
Maybe someone can suggest a more correct variant?
Try to build an API using the socket.
It's more convenient.
I think.
I am trying to stream MP3 file from a nodeJS server using BinaryJS - http://binaryjs.com/
But, when I am decoding the buffers on the client side they are seems to be overlapping, Meaning that the new chunk of data is being played few milliseconds before the previous one ended, causing the audio to lag.
is there any way to make the client wait until the current buffer is finished before starting the new one?
Server:
var BinaryServer = require('binaryjs').BinaryServer;
var fs = require('fs');
var server = BinaryServer({port: 9000});
server.on('connection', function(client){
var file = fs.createReadStream(__dirname + '/Song.mp3', {
'flags': 'r',
'bufferSize': 4 * 1024
});
});
client.send(file);
});
Client:
var client = new BinaryClient('ws://localhost:9000');
window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new AudioContext();
client.on('stream', function (stream, meta) {
var parts = [];
var last = 0;
stream.on('data', function (data) {
var source = context.createBufferSource();
context.decodeAudioData(data, function (buf) {
source.buffer = buf;
source.connect(context.destination);
source.loop = false;
source.start(last);
last += buf.duration;
source.onended = function() {
console.log('Your audio has finished playing');
};
},
function (e) {
"Error with decoding audio data" + e.err
});
parts.push(data);
});
stream.on('end', function () {
console.log(parts);
});
});
Not sure about this, but instead of initializing last to 0, you might want to initialize it to context.currentTime.