We Keep Coding

Adding real time Voice filter with web audio api

I am trying to implement real time voice changer, I have tried some methods but I could not get any results.
I have tried to implement my codes according to following tutorial link
According to the link above, instead of using bufferSource, I created the source from the stream with createMediaStreamSource and sent it to the demonBeastTransform method
here is my final codes:
const video = document.querySelector("video");
const range = document.querySelector("#gain");
navigator.mediaDevices
.getUserMedia({
audio: true,
video: true
})
.then((stream) => {
video.srcObject = stream;
video.onloadedmetadata = (e) => {
video.play();
video.muted = true;
};
// Create a MediaStreamAudioSourceNode
// Feed the HTMLMediaElement into it
const audioCtx = new AudioContext();
const source = audioCtx.createMediaStreamSource(stream);
demonBeastTransform(source, audioCtx)
})
.catch((err) => {
console.error(`The following error occured: ${err}`);
});
async function demonBeastTransform(source, ctx, distortionAmount = 100) {
/* let ctx = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate); */
// Source
/* let source = ctx.createBufferSource();
source.buffer = audioBuffer; */
// Reverb
let convolver = ctx.createConvolver();
convolver.buffer = await ctx.decodeAudioData(await (await fetch("https://voicechanger.io/audio/impulse-responses/voxengo/Large Wide Echo Hall.wav")).arrayBuffer());
// Fire
let fire = ctx.createBufferSource();
fire.buffer = await ctx.decodeAudioData(await (await fetch("https://voicechanger.io/audio/backgrounds/brush_fire-Stephan_Schutze-55390065.mp3")).arrayBuffer());
fire.loop = true;
// Compressor
let compressor = ctx.createDynamicsCompressor();
compressor.threshold.value = -50;
compressor.ratio.value = 16;
// Wobble
let oscillator = ctx.createOscillator();
oscillator.frequency.value = 50;
oscillator.type = 'sawtooth';
// ---
let oscillatorGain = ctx.createGain();
oscillatorGain.gain.value = 0.004;
// ---
let delay = ctx.createDelay();
delay.delayTime.value = 0.01;
// ---
let fireGain = ctx.createGain();
fireGain.gain.value = 0.2;
// ---
let convolverGain = ctx.createGain();
convolverGain.gain.value = 2;
// Filter
let filter = ctx.createBiquadFilter();
filter.type = "highshelf";
filter.frequency.value = 1000;
filter.gain.value = 10;
// Create graph
oscillator.connect(oscillatorGain);
oscillatorGain.connect(delay.delayTime);
// ---
source.connect(delay)
delay.connect(convolver);
//waveShaper.connect(convolver);
fire.connect(fireGain);
convolver.connect(convolverGain);
convolverGain.connect(filter);
filter.connect(compressor);
fireGain.connect(ctx.destination);
compressor.connect(ctx.destination);
let filter2 = ctx.createBiquadFilter();
filter2.type = "lowpass";
filter2.frequency.value = 2000;
let noConvGain = ctx.createGain();
noConvGain.gain.value = 0.9;
delay.connect(filter2);
filter2.connect(filter);
filter.connect(noConvGain);
noConvGain.connect(compressor);
// Render
oscillator.start(0);
source.start(0);
fire.start(0);
/* let outputAudioBuffer = await ctx.startRendering();
return outputAudioBuffer; */
}
<h1>Web Audio API examples: MediaStreamAudioSourceNode</h1>
<video controls></video>
<br />

I took the liberty of tinkering a bit with your script and made the following modifications:
Stream
The biggest change is how the stream is handled. Now the demonBeastTransform passes the stream object in to the function as its only argument.
The reason for this is that we need both the audio and video tracks from the stream. And the idea behind it is to split the audio and video tracks, modify the audio track and then combine them back together into a single stream that is passed as the srcObject of the video.
CORS
The first issue I ran into is that the fetch requests weren't working because of a Same Origin Policy error. However, I was able to download the files and fetch them from my local server, so I'd recommend that you do the same.
Add a try / catch block around any awaited requests to handle them better in case of an error.
const video = document.querySelector("video");
const range = document.querySelector("#gain");
navigator.mediaDevices
.getUserMedia({
audio: true,
video: true
})
.then(demonBeastTransform)
.then((stream) => {
video.onloadedmetadata = () => video.play()
video.srcObject = stream;
})
.catch((err) => {
console.error(`The following error occured: ${err}`);
});
async function demonBeastTransform(stream) {
const audioCtx = new AudioContext();
const source = audioCtx.createMediaStreamSource(stream);
const streamDestination = audioCtx.createMediaStreamDestination();
// Get the video tracks and add them to the stream destination.
const videoTracks = stream.getVideoTracks();
for (const videoTrack of videoTracks) {
streamDestination.stream.addTrack(videoTrack);
}
// Reverb
let convolver = audioCtx.createConvolver();
try {
const convolerResponse = await fetch("Large Wide Echo Hall.wav");
const convolverBuffer = await convolerResponse.arrayBuffer();
convolver.buffer = await audioCtx.decodeAudioData(convolverBuffer);
} catch (error) {
return Promise.reject(error);
}
// Fire
let fire = audioCtx.createBufferSource();
try {
const fireResponse = await fetch("brush_fire-Stephan_Schutze-55390065.mp3");
const fireBuffer = await fireResponse.arrayBuffer();
fire.buffer = await audioCtx.decodeAudioData(fireBuffer);
fire.loop = true;
} catch (error) {
return Promise.reject(error);
}
// Compressor
let compressor = audioCtx.createDynamicsCompressor();
compressor.threshold.value = -50;
compressor.ratio.value = 16;
// Wobble
let oscillator = audioCtx.createOscillator();
oscillator.frequency.value = 50;
oscillator.type = 'sawtooth';
// ---
let oscillatorGain = audioCtx.createGain();
oscillatorGain.gain.value = 0.004;
// ---
let delay = audioCtx.createDelay();
delay.delayTime.value = 0.01;
// ---
let fireGain = audioCtx.createGain();
fireGain.gain.value = 0.2;
// ---
let convolverGain = audioCtx.createGain();
convolverGain.gain.value = 2;
// Filter
let filter = audioCtx.createBiquadFilter();
filter.type = "highshelf";
filter.frequency.value = 1000;
filter.gain.value = 10;
// Create graph
oscillator.connect(oscillatorGain);
oscillatorGain.connect(delay.delayTime);
// ---
source.connect(delay)
delay.connect(convolver);
//waveShaper.connect(convolver);
fire.connect(fireGain);
convolver.connect(convolverGain);
convolverGain.connect(filter);
filter.connect(compressor);
// Instead of audioCtx.destination we pass the audio into the new stream.
fireGain.connect(streamDestination);
compressor.connect(streamDestination);
let filter2 = audioCtx.createBiquadFilter();
filter2.type = "lowpass";
filter2.frequency.value = 2000;
let noConvGain = audioCtx.createGain();
noConvGain.gain.value = 0.9;
delay.connect(filter2);
filter2.connect(filter);
filter.connect(noConvGain);
noConvGain.connect(compressor);
// Render
oscillator.start(0);
fire.start(0);
return streamDestination.stream;
}

Can covert buffer or arrayBuffer of voice to Text in nodejs?

I have a buffer that contain the voice of the person that in media stream, i and I send it from JavaScript to NodeJS using socket.io
I need to convert that buffer to text (like speech to text, but the voice stored as buffer coming from media stream)
There is a helper function I used (in nodejs see below) that convert from/to buffer/arrayBuffer
and there is a package called node-blob that convert buffer to audio blob
but I search a lot how convert audio or even buffer to text, but I failed
any help, code or package that may help to convert it to text ?
JavaScript
navigator.mediaDevices
.getUserMedia({
video: true,
audio: true,
})
.then((stream) => {
setSrcVideo(stream);
const mediasStream = new MediaStream();
mediasStream.addTrack(stream.getVideoTracks()[0]);
mediasStream.addTrack(stream.getAudioTracks()[0]);
const mediaRecorder = new MediaRecorder(mediasStream);
socket.emit('ready');
mediaRecorder.addEventListener('dataavailable', (event) => {
if (event.data && event.data.size > 0) {
socket.emit('send-chunks', event.data);
}
});
socket.on('start-recording', () => {
mediaRecorder.start(1000);
});
});
and I receive that buffer bysocket.on('send-chunks') in NodeJS like this
NodeJS
// connection to socket.io
io.on('connection', (socket) => {
socket.on('ready', () => {
socket.emit('start-recording');
});
socket.on('send-chunks', (chunks) => {
// covert to text
});
});
// helper functions
const toArrayBuffer = (buffer) => {
const arrayBuffer = new ArrayBuffer(buffer.length);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return arrayBuffer;
};
const toBuffer = (arrayBuffer) => {
const buffer = Buffer.alloc(arrayBuffer.byteLength);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < buffer.length; ++i) {
buffer[i] = view[i];
}
return buffer;
};

Javascript Array for Signal Processing

I am trying to record and edit my voice in javascript. Specifically, I am trying to record it in an array that looks like this for my boss [0,102, 301,...] where the values are samples of my voice.
When I record my voice in javascript, I get a Blob type. Is there any way to transform a Blob into the [x, y, z,...] array? Or how is javascript signal processing normally completed?
This is code from this medium article that is how we are doing things. I just can't share the actual company code.
const recordAudio = () =>
new Promise(async resolve => {
const stream = await navigator.mediaDevices.getUserMedia({ audio:true});
const mediaRecorder = new MediaRecorder(stream);
const audioChunks = [];
mediaRecorder.addEventListener("dataavailable", event => {
audioChunks.push(event.data);
});
const start = () => mediaRecorder.start();
const stop = () =>
new Promise(resolve => {
mediaRecorder.addEventListener("stop", () => {
console.log(audioChunks);
console.log(audioChunks)
const audioBlob = new Blob (audioChunks);
const audioURL = URL.createObjectURL(audioBlob);
const audio = new Audio(audioURL);
const play = () => audio.play();
resolve({ audioBlob, audioURL, play });
});
mediaRecorder.stop();
});
resolve({ start, stop});
});
const sleep = time => new Promise(resolve => setTimeout(resolve, time));
const handleAction = async () => {
const recorder = await recordAudio();
const actionButton = document.getElementById('action');
actionButton.disabled = true;
recorder.start();
await sleep(3000);
const audio = await recorder.stop();
audio.play();
await sleep(3000);
actionButton.disabled = false;
}

you can use AudioContext and provide userMediaStream to it, then you can pick up an UInt8Array() that you want with the raw time domain signal, or already transformed frequency domain signal.
Here you can check more details.
https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode
//initialize your signal catching system
let audioContext = new AudioContext();
let analyser = audioContext.createAnalyser();
navigator.mediaDevices.getUserMedia({audio: true}).then(stream => {
let source = audioContext.createMediaStreamSource(stream);
source.connect(analyser);
})
//then update the array with signal every milisecond
setInterval(() => {
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
//get time domain signal
analyser.getByteTimeDomainData(dataArray);
//get frequency domain signal
analyser.getByteFrequencyData(dataArray)
console.log(dataArray)
}, 1)
as for visualization it works ok, with recording there might be a problem with repeating signal if you pick it up couple times before change, or there will be holes in data, but i cant figure out how to read directly from the stream.

Take a photo of the webcam in JavaScript and send it to the database

** I take a photo of the user using the webcam. And I convert this photo to a file so I can send it to the server. I use form data to send this file and it gives me a 400 warning **
async function postImage(url, _image) {
let response = await fetch(url, {
method: 'POST',
body: _image
});
let uploadImage = await response.json();
return uploadImage;
}
let video = document.getElementById('video');
let canvas = document.getElementById('canvas');
let cameraStream = null;
if (null == cameraStream) {
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia){
navigator.mediaDevices.getUserMedia({video: true})
.then(function (stream) {
video.srcObject = stream;
video.play();
cameraStream = stream;
})
}
}
function dataURItoBlob(dataURI) {
// convert base64/URLEncoded data component to raw binary data held in a string
var byteString;
if (dataURI.split(',')[0].indexOf('base64') >= 0)
byteString = atob(dataURI.split(',')[1]);
else
byteString = unescape(dataURI.split(',')[1]);
// separate out the mime component
var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
// write the bytes of the string to a typed array
var ia = new Uint8Array(byteString.length);
for (var i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
return new Blob([ia], {type:mimeString});
}
document.getElementById("snap").addEventListener('click', () => {
video.style.display = 'none';
if (null != cameraStream) {
let context = canvas.getContext('2d');
let img = canvas.toDataURL('image/png', 0.5);
let file = dataURItoBlob(img);
context.drawImage(video, 0, 0, 640, 480);
let track = cameraStream.getTracks()[ 0 ];
track.stop();
video.load();
cameraStream = null;
let formData = new FormData();
formData.append("objectable",'attendance_times')
formData.append("filePath", file, "image.png");
for (var value of formData.values()) {
console.log(value);
}
setTimeout(()=>{
postImage(base_url+'/api/media', formData)
.then(data => console.log(data))
.catch(err => console.log(err));
},2000)
}
})
<video id="video" width="400" height="400" autoplay></video>
<canvas id="canvas" width="640" height="480"></canvas>
<button type="submit" class="btn btn-success btn-lg btn-block" id="snap">snap photo</button>
Unfortunately, I have trouble sending this file
I don't know where the problem with this code is. Please help if this problem can be fixed

I don't think your post is going to work that way. Perhaps have a look at the formdata object.

Send buffer for audiocontext via socket.io (node.js)

I am trying to send audio file (or chunk) via socket.io.
I tried:
// server
socket.on('client-get-audio', function (data) {
//data.fname - name of requested file
var buffer = fs.readFileSync(data.fname, null);
// try 2 - read arraybuffer
//var buffer = fs.readFileSync(data.fname, null).buffer;
buffer = new Uint8Array(buffer);
socket.emit('audio-data', {count: 1, buff: buffer});
});
I can not decode data:
// client
// ctx - AudioContext
// source - buffer source
socket.on('audio-data', function(data){
ctx.decodeAudioData(data.buff, decoded => {
source.buffer = decoded;
source.start(0);
});
I will not describe all my attempts, there were a lot of them.
update
working variant
// server
socket.on('client-get-audio', function (data) {
//data.fname - name of requested file
var buffer = fs.readFileSync(data.fname, null);
socket.emit('audio-data', {count: 1, buff: buffer});
});
// client
function toArrayBuffer(buf) {
var ab = new ArrayBuffer(buf.length);
var view = new Uint8Array(ab);
for (var i = 0; i < buf.length; ++i) {
view[i] = buf[i];
}
return ab;
}
// ctx - AudioContext
// source - buffer source
socket.on('audio-data', function(data){
ctx.decodeAudioData(toArrayBuffer(data.buff.data), decoded => {
source.buffer = decoded;
source.start(0);
});
Maybe someone can suggest a more correct variant?

Try to build an API using the socket.
It's more convenient.
I think.