How would I correctly receive/send raw audio data?
I currently receive raw audio data over WebSockets (with Node.js) and write them to a file. Data is sent as byte[] array of 1024 byte in size from a Java client. Audio format is PCM, Mono, 16bit, 44100Hz.
I'm trying to pass on the raw audio data directly to browser, but it doesn't seem to work. What am I missing? Sorry for being new at this. The browsers I tested all support (Webkit) AudioContext. For each package received in browser, I get the following message:
Uncaught SyntaxError: An invalid or illegal string was specified. audio.php:19 1024
Node.js (server):
var WebSocketServer = require('ws').Server
, wss = new WebSocketServer({port: 8081});
var Clients = [];
function findClient(url) {
return (Clients.indexOf(url));
}
wss.on('connection', function(ws) {
Clients.push(ws);
console.log('Connected: %s', ws.upgradeReq.url);
console.log('Clients: %s', Clients.length);
ws.on('message', function(message, flags) {
if(flags.binary) {
ws.send(message, {binary:true});
}
console.log('data arrived: %s', message);
});
ws.on('close', function(user) {
console.log('Disconnected: %s', ws.upgradeReq.url);
Clients.splice(findClient(ws));
console.log('Clients: %s', Clients.length);
});
Client browser:
<script language="javascript">
if (window.webkitAudioContext) {
var ctx = new webkitAudioContext();
} else if (window.AudioContext) {
var ctx = new AudioContext();
}
function testSound(buff) {
var src = ctx.createBufferSource();
src.buffer = ctx.createBuffer(buff, false);
//src.looping = false;
src.connect(ctx.destination);
src.noteOn(0);
}
var ws = new WebSocket('ws://localhost:8081');
ws.binaryType = 'arraybuffer';
ws.onmessage = function(e) {
console.log(e.data.byteLength);
testSound(e.data);
}
</script>
Related
I'm using the gcloud API on a Nodejs web server to upload files. I'd prefer the files not be uploaded on the client side and instead uploaded on the server. Currently, I am producing a blob on the client side, then converting it to text and passing that to the server through a POST request. All of the information gets successfully passed from the client to the server as expected. This data is also uploaded to gcloud, however, Gcloud does not recognize this as a valid file nor does my computer when I download it.
What is the best way to get the contents of the file to gcloud from the server side? I've tried using dataURIs and reading the orignal file by text and both produce similiar issues. I've also explored piping a readFileStream from the blob on the server end but blobs are not natively supported by node so I have not done so yet.
Client Side
function readSingleFile(e, func, func2){
var file = e.target.files[0];
if(!file){
return; // Add error msg_here
}
var reader = new FileReader();
reader.onload = function(e){
let contents = e.target.result;
let img = document.createElement('img')
let cvs = document.createElement('canvas');
img.onload = ()=>{
cvs.width = img.width;
cvs.height= img.height;
let ctx = cvs.getContext('2d');
ctx.drawImage(img,0,0);
cvs.toBlob((res)=>{res.text().then((text)=>{func2(text)})}, "image/jpeg", 0.92);
}
img.src=contents;
func(contents);
}
reader.readAsDataURL(file);
}
Server Side
function publishPrintjob(dataObj){
try{
var newElemKey = database.ref().child('queue').push().key; // Get random Key
// Create a new blob in the bucket and upload the file data.
const gcloudFile = storage.file('images/' + newElemKey + '.jpg');
gcloudFile.save(dataObj.sockImageFile, function(err) {
if (!err) {
Console.log("File Uploaded!")
}
});
var data = {
date: dataObj.Date,
email: dataObj.email,
design: dataObj.Design,
author: dataObj.Author,
address: dataObj.address,
imageKey: newElemKey,
}
admin.database().ref('queue/' + newElemKey).set(data);
} catch(err){
console.log(err)
}
}
Note: func simply shows the image on the client side, func2 just adds the contents to the POST object.
Uploading a file directly from the computer would be easiest using the storage.bucket(bucketName).upload() function from the cloud storage library. However, this uses location of a file locally and thus will not work unless a file is transferred to the server and saved first. This could be achieved using multi-part form data. Using multipart or uploading locally are better methods for uploading to google storage.
Instead, I solve this by first converting the image to a dataURI, sending the data URI to the server via the body of a GET request, and then converting it to a buffer with a readable stream that can be piped to google storage.
Client
let formData = getFormData('myForm');
var xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
// Typical action to be performed when the document is ready:
}
};
xhttp.open("POST", "dashboard", true);
xhttp.setRequestHeader('Content-Type', 'application/json');
xhttp.send(JSON.stringify(formData));
xhttp.onload = ()=> {
console.log(JSON.parse(xhttp.response))
// Handle server response here
};
}
Server
// DataObject is the body of the GET request, the property imageFile is the URI from readFileAsURI
function uploadImageOnServer(dataObj){
try{
var newElemKey = database.ref().child('queue').push().key; // Get random Key to use as filename
// Create a new blob in the bucket and upload the file data.
const gcloudFile = storage.file('images/' + newElemKey + '.jpeg');
var fs = require('fs'); // Should be required at the top of the file
var string = dataObj.ImageFile;
var regex = /^data:.+\/(.+);base64,(.*)$/;
var matches = string.match(regex);
var ext = matches[1];
var data = matches[2];
var buffer = Buffer.from(data, 'base64');
// Create the readstream
const readableInstanceStream = new Readable({
read() {
this.push(buffer);
this.push(null);
}
});
readableInstanceStream.pipe(gcloudFile.createWriteStream()) // link to gcloud storage api
.on('error', function(err) {
console.log('error')
})
.on('finish', function() {
console.log('upload complete')
});
} catch(err){
console.log(err)
}
}
I want to send a captured image from the raspberry pie client (python) to the server (node.js).
We encoded the image as base64 and sent it back to the server using decoding the base64 as an image, but the image was broken because the file was in a different format.
Here is my code:
client.py
import base64
from PIL import Image
import os, sys
ip = ''
port = 3008
s = socket.socket()
s.connect((ip, port))
image_path = '/home/pi/TCPproject/test.jpg'
if image_path != '':
with open(image_path, "rb") as imageFile:
image_data = base64.b64encode(imageFile.read())
else:
image_data = 'cusdom_image'
s.send(image_data)
s.close()
server.js
var fs = require('fs');
var base64ToImage = require('base64-to-image');
var sockets = [];
var server = net_server.createServer(function(client) {
console.log('Client connection: ');
console.log('local = %s:%s', client.localAddress, client.localPort);
console.log('remote = %s:%s', client.remoteAddress, client.remotePort);
client.setTimeout(500);
client.setEncoding('utf8');
sockets.push(client);
var imageData;
client.on('data', function(data) {
imageData+= data;
});
client.on('end', function() {
console.log('end!')
var decoded = Buffer.from(imageData, 'base64');
fs.writeFile("test.jpg", decoded, function (err) {
if (err) throw err;
else console.log('Saved!');
});
});
client.on('error', function(err) {
console.log('Socket Error: ', JSON.stringify(err));
});
client.on('timeout', function() {
console.log('Socket Timed out');
});
});
server.listen(3008, function() {
console.log('Server listening: ' + JSON.stringify(server.address()));
server.on('close', function(){
console.log('Server Terminated');
});
server.on('error', function(err){
console.log('Server Error: ', JSON.stringify(err));
});
});
function writeData(socket, data){
var success = socket.write(data);
if (!success){
console.log("Client Send Fail");
}
}
Please let me know if encoding, decoding is wrong, or if the TCP socket communication process is wrong, or if there is another problem.
There are multiple problems with the code. On the client side:
s.send(image_data)
This might send image_data but it might only send part of image_data since send is not guaranteed to send everything. Use sendall to send everything or check the return value of send and make sure to send the rest later if not everything was sent at once.
On the server side:
var imageData;
client.on('data', function(data) {
imageData+= data;
});
client.on('end', function() {
console.log('end!')
var decoded = Buffer.from(imageData, 'base64');
If you would have a look at imageData before decoding you would see that it starts with the string undefined and only then the base64 data are following. But all of this is treated as input to the base64 decoder, leading to corrupt data. To fix this initialize imageData:
var imageData = '';
I am using node.js and socket.io to transfer an image from a server to a client. These are my codes ,
renderer.js:
const imgFile = fs.readFileSync(screenShotPath);
const imgFile64 = new Buffer(imgFile).toString('base64');
console.log("image data:"+ imgFile64);
socket.emit('img', imgFile64);
receiver.js:
var base64ToImage = require('base64-to-image');
socket.on("img", function(info) {
console.log("inside receiver");
var base64Str = info;
var path = 'path to file';
var optionalObj = {'fileName': 'test', 'type':'png'};
var imageInfo = base64ToImage(base64Str,path,optionalObj);
});
I am getting an error stating 'Invalid base64 string'.
Can someone please help?
You can do that without base64-to-image
socket.on("img", function(info) {
console.log("inside receiver");
var base64Str = info;
var buff = new Buffer(base64Str ,"base64");
fs.writeFileSync("test.png", buff)
});
PS:
On the server side, you don't have to do new Buffer(imgFile) because imgFile is already Buffer
The following code snippet receives websocket messages. It tries to readAsText the blob and also display it as a JPEG.
ws.onmessage = function(message) {
console.log('websocket blob:', message.data);
var text = reader.readAsText(message.data);
img.src = window.URL.createObjectURL(message.data);
console.log("jpeg received");
ctx.drawImage(img, 0, 0);
}
}
reader.onload = function(progressEvent) {
var text = this.result;
if (text.startsWith('{"')) {
console.log("status received", text);
document.querySelectorAll('pre')[0].textContent=text;
}
}
(The code is broken as it is trying to display plain text messages as JPEGs.)
How can I discriminate between two different sorts of incoming websocket packets? I am in control of both ends, so I can change what the NodeJS server sends, if required.
My JSON packets always begin with {" and my JPEG packets always begin with FF D8 FF.
Is there a way of attaching some meta-data to the websocket messages, when sending, that I could query in the page?
Can I do any processing of websocket blobs without callback methods?
My current workaround is now:
ws.binaryType = 'arraybuffer';
...
ws.onmessage = function(message) { // FIXME: utf8-unsafe
var arrayView = new Uint8Array(message.data);
console.log('websocket message:', arrayView);
if (arrayView[0] === 0xFF) { // jpeg
console.log("jpeg received");
var blob = new Blob([arrayView], {type: "image/jpeg"});
img.src = window.URL.createObjectURL(blob);
ctx.drawImage(img, 0, 0);
} else { // text
console.log("text received");
document.querySelectorAll('pre')[0].textContent=String.fromCharCode.apply(null, arrayView);
}
}
I would prefer a UTF8 safe handler, but that can wait until other parts of the project are working.
Server code in java:
#OnMessage
public void onMessage(Session session, ByteBuffer message) {
if (session.isOpen()) {
String msg = new String(message.array());
System.out.println("Message from " + session.getId() + ": " + msg);
try {
session.getBasicRemote().sendBinary(ByteBuffer.wrap("I have got the message".getBytes()));
} catch (IOException ioe) {
System.out.println(ioe.toString());
}
} else {
System.out.println("Session is not open");
}
}
Client code in Javascript:
webSocket = new WebSocket("ws://192.168.10.1:2525/myChat/chat");
webSocket.binaryType = 'arraybuffer';
webSocket.onopen = function(event) {
updateOutput("Connected!");
connectBtn.disabled = true;
sendBtn.disabled = false;
};
webSocket.onmessage = function(event) {
updateOutput(event.data);
};
Note:
Server code works fine when I use it with Web GL client as it is send Binary data.
Javascript client works fine when I read String data in Server end
(from java code):
#OnMessage
public void onMessage(Session session, String message) {}
Thanks in advice for any comments.
I've found the solution to the issue.
I have used ByteBuffer.js library to send/read data of ByteBuffer type in JavaScript:
webSocket.binaryType = "arraybuffer";
In the function onmessage for reading data:
var d = event.data;
console.log(d.toString());
In the function send for sending data:
var bb = dcodeIO.ByteBuffer.wrap(text);
webSocket.send(bb.toArrayBiffer());