Can covert buffer or arrayBuffer of voice to Text in nodejs? - javascript

I have a buffer that contain the voice of the person that in media stream, i and I send it from JavaScript to NodeJS using socket.io
I need to convert that buffer to text (like speech to text, but the voice stored as buffer coming from media stream)
There is a helper function I used (in nodejs see below) that convert from/to buffer/arrayBuffer
and there is a package called node-blob that convert buffer to audio blob
but I search a lot how convert audio or even buffer to text, but I failed
any help, code or package that may help to convert it to text ?
JavaScript
navigator.mediaDevices
.getUserMedia({
video: true,
audio: true,
})
.then((stream) => {
setSrcVideo(stream);
const mediasStream = new MediaStream();
mediasStream.addTrack(stream.getVideoTracks()[0]);
mediasStream.addTrack(stream.getAudioTracks()[0]);
const mediaRecorder = new MediaRecorder(mediasStream);
socket.emit('ready');
mediaRecorder.addEventListener('dataavailable', (event) => {
if (event.data && event.data.size > 0) {
socket.emit('send-chunks', event.data);
}
});
socket.on('start-recording', () => {
mediaRecorder.start(1000);
});
});
and I receive that buffer bysocket.on('send-chunks') in NodeJS like this
NodeJS
// connection to socket.io
io.on('connection', (socket) => {
socket.on('ready', () => {
socket.emit('start-recording');
});
socket.on('send-chunks', (chunks) => {
// covert to text
});
});
// helper functions
const toArrayBuffer = (buffer) => {
const arrayBuffer = new ArrayBuffer(buffer.length);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return arrayBuffer;
};
const toBuffer = (arrayBuffer) => {
const buffer = Buffer.alloc(arrayBuffer.byteLength);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < buffer.length; ++i) {
buffer[i] = view[i];
}
return buffer;
};

Related

Uint8Array stream buffer to video

I'm streaming video file from server to client. On the client side i receive Uint8Array and i don't know as use this array for streaming video.
what data format to convert it to and how to send it as a video tag so that the file can be played ?
The task is to stream an rtsp stream, but first I want to try streaming one video file
server
io.on('connection', socket => {
console.log('conn')
let stream = ss.createStream();
ss(socket).emit('stream', stream);
let readStream = fs.createReadStream(mp4Path);
readStream.pipe(stream);
socket.on('disconnect', () => {
console.log('user disconnect')
})
})
client
const socket = io();
ss(socket).on('stream', function(stream) {
var binaryString = "";
stream.on('data', function(data) {
for(var i = 0; i < data.length; i++) {
binaryString += String.fromCharCode(data[i]);
}
});
stream.on('end', function(data) {
binaryString = "";
});
});

javascript fetch an image and create base64

I'm using CloudFlare service workers and I want to fetch an image and then generate a base64 representation of it.
So something like this:
const res = await fetch('https://cdn.cnn.com/cnnnext/dam/assets/211010073527-tyson-fury-exlarge-169.jpg')
const blob = await res.blob();
console.log(blob)
console.log(btoa(blob))
this of course doesn't work, any ideas how to get this resolved?

complete worker script with cloudflare using btao
addEventListener('fetch', event => {
event.respondWith(handleRequest(event.request))
})
const imageUrl =
'https://upload.wikimedia.org/wikipedia/commons/thumb/7/72/' +
'Cat_playing_with_a_lizard.jpg/1200px-Cat_playing_with_a_lizard.jpg';
function base64Encode (buf) {
let string = '';
(new Uint8Array(buf)).forEach(
(byte) => { string += String.fromCharCode(byte) }
)
return btoa(string)
}
function base64Decode (string) {
string = atob(string);
const
length = string.length,
buf = new ArrayBuffer(length),
bufView = new Uint8Array(buf);
for (var i = 0; i < length; i++) { bufView[i] = string.charCodeAt(i) }
return buf
}
async function handleRequest(request) {
const
upstreamResponse = await fetch(imageUrl),
text = base64Encode(await upstreamResponse.arrayBuffer()),
bin = base64Decode(text);
return new Response(bin, {status: 200, headers: {
'Content-type': upstreamResponse.headers.get('Content-type')
}})
}
can refer this discussion as well

Web Serial API - problem when stream consists of mix of ASCII and raw data

I'm trying to parse the following serial input raw data:
[51,45,51,30,33,30,33,00,00,00,C0,50,17,FE,05,00,00,58,02,9A,0E,00,00,00,00,02,00,0D,0A]
It consist of some chars(0X51,0x45,0x51...) and some raw data (0x00,0x00,0xC0...) and it ends with '\r\n' (0x0D,0x0A).
The end result I'm getting is ('\r\n' is not included):
['51','45','51','30','33','30','33','0','0','0','fffd','50','17','fffd','5','0','0','58','2','fffd','e','0','0','0','0','2','0']
The values in index 10,13,19 (C0,FE,9A) are not displayed correctly (because they are not part of the ASCII table?).
To get the data I use TextDecoderStream, and I'm not sure it's the right decoder, although the data is partially comprised of ASCII chars and ends with '\r\n'.
Is there more suitable decoder or there is an error in my code?
Thanks in advance.
// request & open port here.
port = await navigator.serial.requestPort();
await port.open({
baudrate: 115200,
baudRate: 115200,
dataBits: 8,
stopBits: 1,
parity: "none",
flowControl: "none",
});
// setup the output stream here.
const encoder = new TextEncoderStream();
outputDone = encoder.readable.pipeTo(port.writable);
outputStream = encoder.writable;
/**********************************/
// code to read the stream here.
let decoder = new TextDecoderStream();
inputDone = port.readable.pipeTo(decoder.writable);
inputStream = decoder.readable.pipeThrough(
new TransformStream(new LineBreakTransformer())
);
reader = inputStream.getReader();
readLoop();
}
/**
* Reads data from the input stream and displays it on screen.
*/
async function readLoop() {
let i = 0;
while (true) {
const { value, done } = await reader.read();
if (value) {
let arrValue = [...value];
ascii_to_hexa(arrValue);
console.log(arrValue);
if (done) {
console.log("[readLoop] DONE", done);
reader.releaseLock();
break;
}
}
}
}
class LineBreakTransformer {
constructor() {
// A container for holding stream data until a new line.
this.container = "";
}
transform(chunk, controller) {
// Handle incoming chunk
this.container += chunk;
const lines = this.container.split("\r\n");
this.container = lines.pop();
lines.forEach((line) => controller.enqueue(line));
}
flush(controller) {
// Flush the stream.
controller.enqueue(this.container);
}
}
function ascii_to_hexa(arr) {
for (let i = 0; i < arr.length; i++) {
arr[i] = Number(arr[i].charCodeAt(0)).toString(16);
}
}

nodejs convert wav, mp3 to AudioBuffer

I have wav file and i went to convert to AudioBuffer because i have some function that only take audio buffer as input how can i get audio buffer from audio file? i've already searched the internet and all i got was converting from audio buffer to wav not what i wanted, and i try to feed the function with Base64 data of the audio, no luck
i've already write this function but it look like it doesn't gives me what i wanted
const getbuffer = async (url) => {
try {
var result = await axios
.get(url, {
responseType: 'arraybuffer'
})
.then(response => new Buffer.from(response.data, 'binary'))
return result
}catch (e) {
return {error: e};
}
}
the buffer i want is to feed to this function so it can change the audio for me
async function alienSoundTransform(audioBuffer) {
let ctx = offlineContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);
let source = ctx.createBufferSource();
source.buffer = audioBuffer;
let oscillator = ctx.createOscillator();
oscillator.frequency.value = 5;
oscillator.type = 'sine';
let oscillatorGain = ctx.createGain();
oscillatorGain.gain.value = 0.05;
let delay = ctx.createDelay();
delay.delayTime.value = 0.05;
// source --> delay --> ctx.destination
// oscillator --> oscillatorGain --> delay.delayTime --> ctx.destination
source.connect(delay);
delay.connect(ctx.destination);
oscillator.connect(oscillatorGain);
oscillatorGain.connect(delay.delayTime);
oscillator.start();
source.start();
let outputAudioBuffer = await ctx.startRendering();
return outputAudioBuffer;
}

Send buffer for audiocontext via socket.io (node.js)

I am trying to send audio file (or chunk) via socket.io.
I tried:
// server
socket.on('client-get-audio', function (data) {
//data.fname - name of requested file
var buffer = fs.readFileSync(data.fname, null);
// try 2 - read arraybuffer
//var buffer = fs.readFileSync(data.fname, null).buffer;
buffer = new Uint8Array(buffer);
socket.emit('audio-data', {count: 1, buff: buffer});
});
I can not decode data:
// client
// ctx - AudioContext
// source - buffer source
socket.on('audio-data', function(data){
ctx.decodeAudioData(data.buff, decoded => {
source.buffer = decoded;
source.start(0);
});
I will not describe all my attempts, there were a lot of them.
update
working variant
// server
socket.on('client-get-audio', function (data) {
//data.fname - name of requested file
var buffer = fs.readFileSync(data.fname, null);
socket.emit('audio-data', {count: 1, buff: buffer});
});
// client
function toArrayBuffer(buf) {
var ab = new ArrayBuffer(buf.length);
var view = new Uint8Array(ab);
for (var i = 0; i < buf.length; ++i) {
view[i] = buf[i];
}
return ab;
}
// ctx - AudioContext
// source - buffer source
socket.on('audio-data', function(data){
ctx.decodeAudioData(toArrayBuffer(data.buff.data), decoded => {
source.buffer = decoded;
source.start(0);
});
Maybe someone can suggest a more correct variant?

Try to build an API using the socket.
It's more convenient.
I think.

Categories