We Keep Coding

Recording browser audio using navigator.mediaDevices.getUserMedia

I am recording browser audio input from the microphone, and sending it via websocket to a nodeJs service that writes the stream to a .wav file.
My problem is that the first recording comes out fine, but any subsequent recordings come out sounding very slow, about half the speed and are therefore unusable.
If I refresh the browser the first recording works again, and subsequent recordings are slowed down which is why I am sure the problem is not in the nodeJs service.
My project is an Angular 5 project.
I have pasted the code I am trying below.
I am using binary.js ->
https://cdn.jsdelivr.net/binaryjs/0.2.1/binary.min.js
this.client = BinaryClient(`ws://localhost:9001`)
createStream() {
window.Stream = this.client.createStream();
window.navigator.mediaDevices.getUserMedia({ audio: true }).then(stream => {
this.success(stream);
})
}
stopRecording() {
this.recording = false;
this.win.Stream.end();
}
success(e) {
var audioContext = window.AudioContext || window.webkitAudioContext;
var context = new audioContext();
// the sample rate is in context.sampleRate
var audioInput = context.createMediaStreamSource(e);
var bufferSize = 2048;
var recorder = context.createScriptProcessor(bufferSize, 1, 1);
}
recorder.onaudioprocess = (e) => {
if (!this.recording) return;
console.log('recording');
var left = e.inputBuffer.getChannelData(0);
this.win.Stream.write(this.convertoFloat32ToInt16(left));
}
audioInput.connect(recorder)
recorder.connect(context.destination);
}
convertoFloat32ToInt16(buffer) {
var l = buffer.length;
var buf = new Int16Array(l)
while (l--) {
buf[l] = buffer[l] * 0xFFFF; //convert to 16 bit
}
return buf.buffer
}
I am stumped as to what can be going wrong so if anyone has experience using this browser tech I would appreciate any help.
Thanks.

I've had this exact problem - your problem is the sample rate you are writing your WAV file with is incorrect.
You need to pass the sample rate used by the browser and the microphone to the node.js which writes the binary WAV file.
Client side:
After a successfull navigator.mediaDevices.getUserMedia (in your case, success function), get the sampleRate variable from the AudioContext element:
var _smapleRate = context.sampleRate;
Then pass it to the node.js listener as a parameter. In my case I used:
binaryClient.createStream({ SampleRate: _smapleRate });
Server (Node.js) side:
Use the passed SampleRate to set the WAV file's sample rate. In my case this is the code:
fileWriter = new wav.FileWriter(wavPath, {
channels: 1,
sampleRate: meta.SampleRate,
bitDepth: 16
});
This will prevent broken sounds, low pitch sounds, low or fast WAV files.
Hope this helps.

Stream live audio to Node.js server

I'm working on a project and I require to send an audio stream to a Node.js server. I'm able to capture microphone sound with this function:
function micCapture(){
'use strict';
navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
var constraints = {
audio: true,
video: false
};
var video = document.querySelector('video');
function successCallback(stream) {
window.stream = stream; // stream available to console
if (window.URL) {
video.src = window.webkitURL.createObjectURL(stream);
} else {
video.src = stream;
}
//Send audio stream
//server.send(stream);
}
function errorCallback(error) {
console.log('navigator.getUserMedia error: ', error);
}
navigator.getUserMedia(constraints, successCallback, errorCallback);
}
As you can see, I'm able to capture audio and play it on the website.
Now I want to send that audio stream to a Node.js server, and send it back to other clients. Like a voicechat, but I don't want to use WebRTC as I need the stream in the server. How can I achieve this? Can I use socket.io-stream to do this? In the examples I saw, they recorded the audio, and sent a file, but I need "live" audio.

I have recently done live audio upload using socket.io from browser to server. I am going to answer here in case someone else needs it.
var stream;
var socket = io();
var bufferSize = 1024 * 16;
var audioContext = new AudioContext();
// createScriptProcessor is deprecated. Let me know if anyone find alternative
var processor = audioContext.createScriptProcessor(bufferSize, 1, 1);
processor.connect(audioContext.destination);
navigator.mediaDevices.getUserMedia({ video: false, audio: true }).then(handleMicStream).catch(err => {
console.log('error from getUserMedia', err);
});
handleMicStream will run when user accepts the permission to use microphone.
function handleMicStream(streamObj) {
// keep the context in a global variable
stream = streamObj;
input = audioContext.createMediaStreamSource(stream);
input.connect(processor);
processor.onaudioprocess = e => {
microphoneProcess(e); // receives data from microphone
};
}
function microphoneProcess(e) {
const left = e.inputBuffer.getChannelData(0); // get only one audio channel
const left16 = convertFloat32ToInt16(left); // skip if you don't need this
socket.emit('micBinaryStream', left16); // send to server via web socket
}
// Converts data to BINARY16
function convertFloat32ToInt16(buffer) {
let l = buffer.length;
const buf = new Int16Array(l / 3);
while (l--) {
if (l % 3 === 0) {
buf[l / 3] = buffer[l] * 0xFFFF;
}
}
return buf.buffer;
}
Have your socket.io server listen to micBinaryStream and you should get the data. I needed the data as a BINARY16 format for google api if you do not need this you can skip the function call to convertFloat32ToInt16().
Important
When you need to stop listening you MUST disconnect the the processor and end the stream. Run the function closeAll() below.
function closeAll() {
const tracks = stream ? stream.getTracks() : null;
const track = tracks ? tracks[0] : null;
if (track) {
track.stop();
}
if (processor) {
if (input) {
try {
input.disconnect(processor);
} catch (error) {
console.warn('Attempt to disconnect input failed.');
}
}
processor.disconnect(audioContext.destination);
}
if (audioContext) {
audioContext.close().then(() => {
input = null;
processor = null;
audioContext = null;
});
}
}

it's an old time question, i see. I'm doing the same thing (except my server doesn't run node.js and is written in C#) and stumbled upon this.
Don't know if someone is still interested but i've elaborated a bit. The current alternative to the deprecated createScriptProcessor is the AudioWorklet interface.
From: https://webaudio.github.io/web-audio-api/#audioworklet
1.32.1. Concepts
The AudioWorklet object allows developers to supply scripts (such as JavaScript or >WebAssembly code) to process audio on the rendering thread, supporting custom >AudioNodes. This processing mechanism ensures synchronous execution of the script >code with other built-in AudioNodes in the audio graph.
You cannot implement interfaces in Javascript as far as i know but you can extend a class derived from it.
And the one we need is: https://developer.mozilla.org/en-US/docs/Web/API/AudioWorkletProcessor
So i did write a processor that just mirrors the output with the input values and displays them.
class CustomAudioProcessor extends AudioWorkletProcessor {
process (inputs, outputs, parameters) {
const input = inputs[0];
const output = output[0];
for (let channel = 0; channel < input.length; ++channel) {
for (let i = 0; i < input[channel].length; ++i) {
// Just copying all the data from input to output
output[channel][i] = input[channel][i];
// The next one will make the app crash but yeah, the values are there
// console.log(output[channel][i]);
}
}
}
}
The processor must then be placed into the audio pipeline, after the microphone and before the speakers.
function record() {
constraints = { audio: true };
navigator.mediaDevices.getUserMedia(constraints)
.then(function(stream) {
audioCtx = new AudioContext();
var source = audioCtx.createMediaStreamSource(stream);
audioCtx.audioWorklet.addModule("custom-audio-processor.js").then(() => {
customAudioProcessor = new AudioWorkletNode(audioCtx, "custom-audio-processor");
source.connect(customAudioProcessor);
customAudioProcessor.connect(audioCtx.destination);
})
audioCtx.destination.play();
Works! Good luck! :)

Playing RTP Video Stream with HTML5-Video-Tag

I am sending RTP video data from an Android phone to my computer. The RTP streaming works, proven with ffplay.
Now I'm trying to receive and display the stream inside Chrome (63.0.3239.84, Win10, 64Bit). I receive the RTP packets with node.js and pass them to all connected users with help of socket.io.
To display the video in an HTML5 <video>-tag, I tried a similar approach as the example in the docs of the MediaSource API. This is, what i've tried to append the video data on the fly:
var socket = io();
var ms = new MediaSource();
var mimeCodec = 'video/mp4; codecs="avc1.42C01E"';
var queue = [];
var video = document.querySelector('video');
// receiving the rtp-packets from android, forwarded by nodes socket.io
socket.on('video-fragment', function(fragment) {
queue.push(fragment.slice(11)); // cutting of rtp header to get plain NAL units
});
video.src = URL.createObjectURL(ms);
var timeout = 500;
var appendChunk = function() {
if(queue.length > 0) {
timeout = 500;
var append = function() {
var sb = ms.addSourceBuffer(mimeCodec);
sb.appendBuffer(queue.shift());
sb.addEventListener('updateend', function() {
if(video.paused) {
console.log("try to start video");
video.play();
}
appendChunk();
});
ms.removeSourceBuffer(sb);
};
if(ms.readyState == "open") {
append();
} else {
ms.addEventListener('sourceopen', function() {
appendChunk();
});
}
} else {
setTimeout(appendChunk, timeout); // try again after some timeout...
timeout *= 2;
}
}
appendChunk(); // start recursive call
<video></video>
However, I don't get any errors from my video-object. If I close the MediaSource at some point with ms.endOfStream() and try to start the video after that, I get a MediaError with code 4: DEMUXER_ERROR_COULD_NOT_OPEN.
Any suggestions? Am I somehow on the right way? Or is this kind of video streaming not possible in Chrome and the only way of "live streaming" would be with using a playlist and video-file-chunks?

The browser only supports video in an mp4 (or webm) container. It does not support the rtp protocol. It must be repackaged to a format the browser supports.

Failed to set local answer sdp: Called in wrong state: STATE_INPROGRESS

I have two clients :
1) Chrome (version 50.0.2661.102 m) on Windows 7 PC
2) Chrome (version 50.0.2661.89) on Android tablet
Both are in the same network (so no need for STUN/TURN server).
I use my own signal server built with node.js (webSocket) on a VirtualBox VM with Centos 6.
The communication with video/sound between the clients works fine. Now I want to transfer a file from one client to another. As base of my code i use the code of this example
here
As this code suggess, I create the dataChannnel exactly after the creation of PeerConnection.
function createPeerConnection() {
....
myPeerConnection = new RTCPeerConnection(iceServers, optional);
myDataChannel = myPeerConnection.createDataChannel('myDataChannel');
// Set up event handlers for the ICE negotiation process.
myPeerConnection.onicecandidate = handleICECandidateEvent;
myPeerConnection.onaddstream = handleAddStreamEvent;
myPeerConnection.onnremovestream = handleRemoveStreamEvent;
myPeerConnection.oniceconnectionstatechange = handleICEConnectionStateChangeEvent;
myPeerConnection.onicegatheringstatechange = handleICEGatheringStateChangeEvent;
myPeerConnection.onsignalingstatechange = handleSignalingStateChangeEvent;
myPeerConnection.onnegotiationneeded = handleNegotiationNeededEvent;
myPeerConnection.ondatachannel = handleDataChannel;
myDataChannel.onmessage = handleDataChannelMessage;
myDataChannel.onopen = handleDataChannelOpen;
}
...
...
function invite(peerId) {
...
createPeerConnection();
...
}
...
...
function handleVideoOfferMsg(msg) {
thereIsNegotiation = true;
targetUsername = msg.name;
// Call createPeerConnection() to create the RTCPeerConnection.
log("Starting to accept invitation from " + targetUsername);
createPeerConnection();
// We need to set the remote description to the received SDP offer
// so that our local WebRTC layer knows how to talk to the caller.
var desc = new RTCSessionDescription(msg.sdp);
myPeerConnection.setRemoteDescription(desc)
.then(function(stream) {
log("-- Calling myPeerConnection.addStream()");
return myPeerConnection.addStream(localStream);
})
.then(function() {
log("------> Creating answer");
// Now that we've successfully set the remote description, we need to
// start our stream up locally then create an SDP answer. This SDP
// data describes the local end of our call, including the codec
// information, options agreed upon, and so forth.
return myPeerConnection.createAnswer();
})
.then(function(answer) {
log("------> Setting local description after creating answer");
// We now have our answer, so establish that as the local description.
// This actually configures our end of the call to match the settings
// specified in the SDP.
return myPeerConnection.setLocalDescription(answer);
})
.then(function() {
var msg = {
name: clientId,
room: roomId,
target: targetUsername,
type: "video-answer",
sdp: myPeerConnection.localDescription
};
// We've configured our end of the call now. Time to send our
// answer back to the caller so they know that we want to talk
// and how to talk to us.
log("Sending answer packet back to other peer");
sendToServer(msg);
})
.catch(handleGetUserMediaError);
}
When the second client makes the offer, the first client when tries to make the answer, I get the error
Error opening your camera and / or microphone : failed to set local answer
spd: Failed to push down transport description: Local fingerprint provided
but no identity available.
or
Error opening your camera and / or microphone : failed to set local answer
spd: Called in wrong state : STATE_INPROGRESS
Only one time the creation was successful.
Do I have to create DataChannel in other place? Like here :
function handleICEConnectionStateChangeEvent {
switch(myPeerConnection.iceConnectionState) {
...
case "connected":
createDataChannel();
break;
}
}
function createDataChannel(){
myDataChannel = myPeerConnection.createDataChannel('myDataChannel');
myPeerConnection.ondatachannel = handleDataChannel;
myDataChannel.onmessage = handleDataChannelMessage;
myDataChannel.onopen = handleDataChannelOpen;
}
Any suggestions?

The error in this code is that both sender and receiver create new datachannel. The right thing is, one to create the datachannel
myDataChannel = myPeerConnection.createDataChannel('myDataChannel')
and the other to wait for the creation of dataChannel:
myPeerConnection.ondatachannel = handleDataChannel;

Access microphone from a browser - Javascript

Is it possible to access the microphone (built-in or auxiliary) from a browser using client-side JavaScript?
Ideally, it would store the recorded audio in the browser. Thanks!

Here we capture microphone audio as a Web Audio API event loop buffer using getUserMedia() ... time domain and frequency domain snippets of each audio event loop buffer are printed (viewable in browser console just hit key F12 or ctrl+shift+i )
<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>capture microphone audio into buffer</title>
<script type="text/javascript">
var webaudio_tooling_obj = function () {
var audioContext = new AudioContext();
console.log("audio is starting up ...");
var BUFF_SIZE = 16384;
var audioInput = null,
microphone_stream = null,
gain_node = null,
script_processor_node = null,
script_processor_fft_node = null,
analyserNode = null;
if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia || navigator.msGetUserMedia;
if (navigator.getUserMedia){
navigator.getUserMedia({audio:true},
function(stream) {
start_microphone(stream);
},
function(e) {
alert('Error capturing audio.');
}
);
} else { alert('getUserMedia not supported in this browser.'); }
// ---
function show_some_data(given_typed_array, num_row_to_display, label) {
var size_buffer = given_typed_array.length;
var index = 0;
var max_index = num_row_to_display;
console.log("__________ " + label);
for (; index < max_index && index < size_buffer; index += 1) {
console.log(given_typed_array[index]);
}
}
function process_microphone_buffer(event) { // invoked by event loop
var i, N, inp, microphone_output_buffer;
microphone_output_buffer = event.inputBuffer.getChannelData(0); // just mono - 1 channel for now
// microphone_output_buffer <-- this buffer contains current gulp of data size BUFF_SIZE
show_some_data(microphone_output_buffer, 5, "from getChannelData");
}
function start_microphone(stream){
gain_node = audioContext.createGain();
gain_node.connect( audioContext.destination );
microphone_stream = audioContext.createMediaStreamSource(stream);
microphone_stream.connect(gain_node);
script_processor_node = audioContext.createScriptProcessor(BUFF_SIZE, 1, 1);
script_processor_node.onaudioprocess = process_microphone_buffer;
microphone_stream.connect(script_processor_node);
// --- enable volume control for output speakers
document.getElementById('volume').addEventListener('change', function() {
var curr_volume = this.value;
gain_node.gain.value = curr_volume;
console.log("curr_volume ", curr_volume);
});
// --- setup FFT
script_processor_fft_node = audioContext.createScriptProcessor(2048, 1, 1);
script_processor_fft_node.connect(gain_node);
analyserNode = audioContext.createAnalyser();
analyserNode.smoothingTimeConstant = 0;
analyserNode.fftSize = 2048;
microphone_stream.connect(analyserNode);
analyserNode.connect(script_processor_fft_node);
script_processor_fft_node.onaudioprocess = function() {
// get the average for the first channel
var array = new Uint8Array(analyserNode.frequencyBinCount);
analyserNode.getByteFrequencyData(array);
// draw the spectrogram
if (microphone_stream.playbackState == microphone_stream.PLAYING_STATE) {
show_some_data(array, 5, "from fft");
}
};
}
}(); // webaudio_tooling_obj = function()
</script>
</head>
<body>
<p>Volume</p>
<input id="volume" type="range" min="0" max="1" step="0.1" value="0.5"/>
</body>
</html>
Since this code exposes microphone data as a buffer you could add ability to stream using websockets or simply aggregate each event loop buffer into a monster buffer then download the monster to a file
Notice the call to
var audioContext = new AudioContext();
which indicates its using the Web Audio API which is baked into all modern browsers (including mobile browsers) to provide an extremely powerful audio platform of which tapping into the mic is but a tiny fragment ... NOTE the CPU usage jumps up due to this demo writing each event loop buffer into browser console log which is for testing only so actual use is far less resource intensive even when you mod this to stream audio to elsewhere
Links to some Web Audio API documentation
Basic concepts behind Web Audio API
SO wiki on Web Audio API
nice Web Audio API demos ... some with github links

Yes you can.
Using the getUserMedia() API, you can capture raw audio input from your microphone.

In a secure context, to query the devices.
getUserMedia() is a powerful feature which can only be used in secure
contexts; in insecure contexts, navigator.mediaDevices is undefined,
preventing access to getUserMedia(). A secure context is, in short, a
page loaded using HTTPS or the file:/// URL scheme, or a page loaded
from localhost.
async function getMedia(constraints) {
let stream = null;
try {
stream = await navigator.mediaDevices.getUserMedia(constraints);
console.log(stream)
} catch(err) {
document.write(err)
}
}
getMedia({ audio: true, video: true })
https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia

This is a simple way:
//event:
const micButtonClicked = () => {
//check the access:
isMicrophoneAllowed(isAllowed => {
if(isAllowed)
record();
else
navigator.mediaDevices.getUserMedia({audio: true})
.then(stream => record())
.catch(err => alert('need permission to use microphone'));
});
}
//isMicrophoneAllowed:
const isMicrophoneAllowed = callback => {
navigator.permissions.query({name: 'microphone'})
.then(permissionStatus => Strings.runCB(callback, permissionStatus.state === 'granted'));
}
//record:
const record = () => {
// start recording...
}