I am trying to write a file uploader for Meteor framework.
The principle is to split the fileon the client from an ArrayBuffer in small packets of 4096 bits that are sent to the server through a Meteor.method.
The simplified code below is the part of the client that sends a chunk to the server, it is repeated until offset reaches data.byteLength :
// data is an ArrayBuffer
var total = data.byteLength;
var offset = 0;
var upload = function() {
var length = 4096; // chunk size
// adjust the last chunk size
if (offset + length > total) {
length = total - offset;
}
// I am using Uint8Array to create the chunk
// because it can be passed to the Meteor.method natively
var chunk = new Uint8Array(data, offset, length);
if (offset < total) {
// Send the chunk to the server and tell it what file to append to
Meteor.call('uploadFileData', fileId, chunk, function (err, length) {
if (!err) {
offset += length;
upload();
}
}
}
};
upload(); // start uploading
The simplified code below is the part on the server that receives the chunk and writes it to the file system :
var fs = Npm.require('fs');
var Future = Npm.require('fibers/future');
Meteor.methods({
uploadFileData: function(fileId, chunk) {
var fut = new Future();
var path = '/uploads/' + fileId;
// I tried that with no success
chunk = String.fromCharCode.apply(null, chunk);
// how to write the chunk that is an Uint8Array to the disk ?
fs.appendFile(path, chunk, 'binary', function (err) {
if (err) {
fut.throw(err);
} else {
fut.return(chunk.length);
}
});
return fut.wait();
}
});
I failed to write a valid file to the disk, actually the file is saved but I cannot open it, when I see the content in a text editor, it is similar to the original file (a jpg for example) but some characters are different, I think that could be an encoding problem as the file size is not the same, but I don't know how to fix that...
Saving the file was as easy as creating a new Buffer with the Uint8Array object :
// chunk is the Uint8Array object
fs.appendFile(path, Buffer.from(chunk), function (err) {
if (err) {
fut.throw(err);
} else {
fut.return(chunk.length);
}
});
Building on Karl.S answer, this worked for me, outside of any framework:
fs.appendFileSync(outfile, Buffer.from(arrayBuffer));
Just wanted to add that in newer Meteor you could avoid some callback hell with async/await. Await will also throw and push the error up to client
Meteor.methods({
uploadFileData: async function(file_id, chunk) {
var path = 'somepath/' + file_id; // be careful with this, make sure to sanitize file_id
await fs.appendFile(path, new Buffer(chunk));
return chunk.length;
}
});
Related
I am trying to write a buffer to a file in localStorage using BrowserFS.
This is a simplified version of the code I am executing:
// Simple function to create a buffer that is filled with value
function createFilledBuffer(size, value) {
const b = Buffer.alloc(size)
b.fill(value)
return b
}
// Configure localStorage FS instance
BrowserFS.configure({
fs: "LocalStorage"
}, function (e) {
if (e) {
console.error('BrowserFS error:', e);
return;
}
// Get BrowserFS's fs module
const fs = BrowserFS.BFSRequire('fs');
const fileName = '/demo.txt';
fs.exists(self.fileName, async (exists) => {
let file;
// Open the file
if (!exists) {
file = fs.openSync(self.fileName, 'w+')
} else {
file = fs.openSync(self.fileName, 'r+')
}
// Write to file -> here is the problem?
// Fill first two "sectors" of the file with 0s
const sector = 4096;
// Write first sector
const bw1 = fs.writeSync(file, createFilledBuffer(sector, 0), 0, sector, 0)
// Write second sector
const bw2 = fs.writeSync(file, createFilledBuffer(sector, 0), 0, sector, sector)
console.log('Bytes written to file: ', bw1, bw2)
});
});
The output in the console is: Bytes written to file: 4096 4096, suggesting that it had successfully written the data but when I try to get the file size of '/demo.txt' using fs.stat, it is still 0 bytes large.
Later in my code I also try to write a buffer with real data (not just 0) into the file at specific locations, like:
const buffer = ...;
const bytesWritten = fs.writeSync(file, buffer, 0, buffer.length, 7 * sector)
which will also tell me it has written buffer.length bytes (=481), but looking at fs.stat, the file still contains 0 bytes.
What do I have to do to successfully write the bytes to the file?
I m trying to create a resumable file downloader based only on the client side. The server is beyond my control and on an ajax request i get the file which is a very very big binary data file (100mgb).
After long research i have understood that i cannot use the xhr element to stream the response and i cannot read chunks of the file before it is completely cached... So I looked some more and found the fetch api which is quite new but i cannot find any proper documentation or tutorials. I would very much appreciate if someone could illustrate a simple example of fetching some url and reading the stream chunk by chunk
Here's an example from this blog post:
fetch('/big-data.csv').then(function(response) {
var reader = response.body.getReader();
var partialCell = '';
var returnNextCell = false;
var returnCellAfter = "Jake";
var decoder = new TextDecoder();
function search() {
return reader.read().then(function(result) {
partialCell += decoder.decode(result.value || new Uint8Array, {
stream: !result.done
});
// Split what we have into CSV 'cells'
var cellBoundry = /(?:,|\r\n)/;
var completeCells = partialCell.split(cellBoundry);
if (!result.done) {
// Last cell is likely incomplete
// Keep hold of it for next time
partialCell = completeCells[completeCells.length - 1];
// Remove it from our complete cells
completeCells = completeCells.slice(0, -1);
}
for (var cell of completeCells) {
cell = cell.trim();
if (returnNextCell) {
reader.cancel("No more reading needed.");
return cell;
}
if (cell === returnCellAfter) {
returnNextCell = true;
}
}
if (result.done) {
throw Error("Could not find value after " + returnCellAfter);
}
return search();
})
}
return search();
}).then(function(result) {
console.log("Got the result! It's '" + result + "'");
}).catch(function(err) {
console.log(err.message);
});
Note that streaming responses aren't supported yet in all browsers, check the compatibility table on MDN.
I want to create a stream with the wav file, and pass it to my NodeJS addon:
var readableStream = fs.createReadStream('random_file.wav');
readableStream.on('data', function(chunk) {
var chunck_to_binary = chunk.toString('what??'); //binary??
var obj1 = addon.store(chunck_to_binary);
console.log('chunck');
console.log(obj1.caract_count);
//console.log(typeof data);
});
Then, when all the file has been passed. I want to to devolve the bytes to Nodejs, just to be shure that all process is correct. And create a copy:
readableStream.on('end', function() {
console.log("loaded");
var data_copy = addon.return_bytes();
fs.writeFile('copy.wav', data_copy, function (err) {
if (err) return console.log(err);
console.log('done!!');
});
});
In addon I implement some like this:
void store_values(const FunctionCallbackInfo<v8::Value>& args) {
Isolate* isolate = Isolate::GetCurrent();
HandleScope scope(isolate);
if (args.Length() < 1) {
isolate->ThrowException(Exception::TypeError(
String::NewFromUtf8(isolate, "Wrong number of arguments")));
return;
}
v8::String::Utf8Value param1(args[0]->ToString());
std::string aux= std::string(*param1); //JS---->C++
//file_in_memory is global
file_in_memory = file_in_memory + aux;
//(..) return byte size of file_in_memory
}
So far I have not succeed. What is possible wrong? I think the problem is how I am passing the information to addon. Any suggestion?
chunk is a Buffer, which is good because it's basically just a chunk of bytes. Just pass to your add-on as-is:
var obj1 = addon.store(chunk);
In your add-on, you can convert it to std::string like so:
std::string aux(node::Buffer::Data(args[0]), node::Buffer::Length(args[0]));
I'm trying to insert an image into a pdf I'm creating server-side with PDFkit. I'm using cfs:dropbox to store my files. Before when I was using cvs:filesystem, it was easy to add the images to my pdf's cause they were right there. Now that they're stored remotely, I'm not sure how to add them, since PDFkit does not support adding images with just the url. It will, however, accept a buffer. How can I get a buffer from my CollectionFS files?
So far I have something like this:
var portrait = Portraits.findOne('vS2yFy4gxXdjTtz5d');
readStream = portrait.createReadStream('portraits');
I tried getting the buffer two ways so far:
First using dataMan, but the last command never comes back:
var dataMan = new DataMan.ReadStream(readStream, portrait.type());
var buffer = Meteor.wrapAsync(Function.prototype.bind(dataMan.getBuffer, dataMan))();
Second buffering the stream manually:
var buffer = new Buffer(0);
readStream.on('readable', function() {
buffer = Buffer.concat([buffer, readStream.read()]);
});
readStream.on('end', function() {
console.log(buffer.toString('base64'));
});
That never seems to come back either. I double-checked my doc to make sure it was there and it has a valid url and the image appears when I put the url in my browser. Am I missing something?
I had to do something similar and since there's no answer to this question, here is how I do it:
// take a cfs file and return a base64 string
var getBase64Data = function(file, callback) {
// callback has the form function (err, res) {}
var readStream = file.createReadStream();
var buffer = [];
readStream.on('data', function(chunk) {
buffer.push(chunk);
});
readStream.on('error', function(err) {
callback(err, null);
});
readStream.on('end', function() {
callback(null, buffer.concat()[0].toString('base64'));
});
};
// wrap it to make it sync
var getBase64DataSync = Meteor.wrapAsync(getBase64Data);
// get a cfs file
var file = Files.findOne();
// get the base64 string
var base64str = getBase64DataSync(file);
// get the buffer from the string
var buffer = new Buffer(base64str, 'base64')
Hope it'll help!
I'm trying to open a binary file './test' and send it's contents, one byte at a time, to an external device through the UART. The external device echos back each byte.
The regular file './test' and the buffer 'dta' in this case are both 19860 bytes in length however the code will continue to send bytes from beyond the end of buffer 'dta' well after 'a' becomes greater than 'dta.length' and I can't figure out why. Any ideas?
var fs = require('fs');
var SerialPort = require("serialport").SerialPort
var serialPort = new SerialPort("/dev/ttyAMA0", {baudrate: 115200}, false);
stats = fs.statSync(__dirname+"/test");
dta = new Buffer (stats.size);
dta = fs.readFileSync(__dirname+"/test");
a=0;
serialPort.open(function (error) {
if ( error ) {
console.log('failed to open: '+error);
} else {
serialPort.write(dta[a]);
}
});
serialPort.on('data', function(data) {
a++;
if (a < dta.length) serialPort.write(dta[a]);
});