I'm attempting to make a nodejs function which reads back data from a file with the following code:
app.post('/DownloadData', function(req, res)
{
req.on('data', function(data) {
if (fs.existsSync('demoDataFile.dat')) {
var rstream = fs.createReadStream('demoDataFile.dat');
var bufs = [];
rstream.on('data', function(chunk) {
bufs.push(chunk);
console.log("data");
});
rstream.on('end', function() {
downbuf = Buffer.concat(bufs);
console.log(downbuf.length);
});
}
});
req.on('end', function() {
console.log("end length: " + downbuf.length);
res.end(downbuf);
});
req.on('error', function(err)
{
console.error(err.stack);
});
});
The problem is, the buffer comes back as empty as the req.on('end' ... is called before any of the rstream.on events ("data" and the length aren't printed in the console until after "end length: " has been printed). Am I handling the events wrong or is there some other issue? Any guidance would be appreciated.
Not sure why you're reading from req, because you're not using the body data at all. Also, because the data event can trigger multiple times, the code you're using to read the file may also get called multiple times, which probably isn't what you want.
Here's what I think you want:
app.post("/DownloadData", function(req, res) {
let stream = fs.createReadStream("demoDataFile.dat");
// Handle error regarding to creating/opening the file stream.
stream.on('error', function(err) {
console.error(err.stack);
res.sendStatus(500);
});
// Read the file data into memory.
let bufs = [];
stream.on("data", function(chunk) {
bufs.push(chunk);
console.log("data");
}).on("end", function() {
let downbuf = Buffer.concat(bufs);
console.log(downbuf.length);
...process the buffer...
res.end(downbuf);
});
});
You have to be aware that this will read the file into memory entirely. If it's a big file, it may require a lot of memory.
Since you don't specify which operations you have to perform on the file data, I can't recommend an alternative, but there are various modules available that can help you process file data in a streaming fashion (i.e. without having to read the file into memory entirely).
Related
I'm working on a website that stores images in a MongoDB GridFS database and then can display one of the chosen images to the user.
The upload part is already working, but I'm having trouble with the download part of the image because whenever I try to convert the readStream from the database to a string, I get undefined in my console.
I think the problem lies around the readStream part, more specifically, the conversion from readStream to String. When I replaced the readStream with writeStream, I was able to get a perfect copy of my image downloaded.
However, I don't need that. What I need is for the image to be sent to the front-end.
I've tried a lot of different ideas like moving the createServer around and the MongoClient.connect. I've also tried different packages that supposedly convert a stream straight into a String.
MongoClient.connect(dbURL, function(err, client){
if(err) throw err;
// setting db name
var db = client.db("formImgs");
// bucket to interact with GridFS
bucket = new mongodb.GridFSBucket(db);
// loop to upload each image
for(var i = 0; i < fileNames.length; i++){
// uploads to the database using file locations
fs.createReadStream("photos/" + fileNames[i]).pipe(bucket.openUploadStream(fileNames[i]))
.on("error", function(e){
// error handling
console.log(e);
}).on("finish", function(){
// success
console.log("Done uploading!");
});
}
const server = http.createServer((req, res) => {
var readStream = fs.createReadStream("photos/LaptopRegistry.png");
bucket.openDownloadStreamByName("LaptopRegistry.png").pipe(readStream);
var bufs = [];
// THIS IS WHERE I THINK MY CODE MESSES UP
readStream.on("data", function(chunk){
bufs.push(chunk);
}).on("end", function(){
var fbuf = Buffer.concat(bufs);
base64 = fbuf.toString("base64");
});
res.statusCode = 200;
res.setHeader("Content-Type", "text/plain");
fs.readFile("index.html", function(err, data) {
res.writeHead(200, {"Content-Type": "text/html"});
res.write(data);
//insert use of base64 string here
});
});
});
});
Here is my code :
function aCallbackInLoop(dataArray) {
dataArray.forEach(function (item, index) {
fs.appendFile(fileName, JSON.stringify(item) + "\r\n", function (err) {
if (err) {
console.log('Error writing data ' + err);
} else {
console.log('Data written');
}
});
});
}
I get random errors :
Data written
Data written
.
.
Error writing data Error: UNKNOWN, open 'output/mydata.json'
Error writing data Error: UNKNOWN, open 'output/mydata.json'
.
.
Data written
Error writing data Error: UNKNOWN, open 'output/mydata.json'
The function (aCallbackInLoop) is a callback for a web-service request, which returns chunks of data in dataArray. Multiple web-service requests are being made in a loop, so this callback is perhaps being called in parallel. I doubt it's some file lock issue, but I am not sure how to resolve.
PS: I have made sure it's not a data issue (I am logging all items in dataArray)
Edit : Code after trying write stream :
function writeDataToFile(fileName, data) {
try {
var wStream = fs.createWriteStream(fileName);
wStream.write(JSON.stringify(data) + "\r\n");
wStream.end();
} catch (err) {
console.log(err.message);
}
}
function aCallbackInLoop(dataArray){
dataArray.forEach(function(item, index){
writeDataToFile(filename, item); //filename is global var
});
}
As you have observed, multiple appendFile calls are not able to proceed because of the previous appendFile calls. In this particular case, it would be better to create a write stream.
var wstream = fs.createWriteStream(fileName);
dataArray.forEach(function (item) {
wstream.write(JSON.stringify(item + "\r\n");
});
wstream.end();
If you want to know when all the data is written, then you can register a function with the finish event, like this
var wstream = fs.createWriteStream(fileName);
wstream.on("finish", function() {
// Writing to the file is actually complete.
});
dataArray.forEach(function (item) {
wstream.write(JSON.stringify(item + "\r\n");
});
wstream.end();
Try using the synchronous version of appendFile - https://nodejs.org/api/fs.html#fs_fs_appendfilesync_filename_data_options
I am trying to write the binary body of a request to a file and failing. The file is created on server but I am unable to open it. I am getting 'Fatal error: Not a png' on Ubuntu. Here is how I am making the request:
curl --request POST --data-binary "#abc.png" 192.168.1.38:8080
And here is how I am trying to save it with the file. The first snippet is a middleware for appending all the data together and second one is the request handler:
Middleware:
app.use(function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf-8');
req.on('data', function(chunk) {
req.rawBody += chunk;
});
req.on('end', function() {
next();
});
});
Handler:
exports.save_image = function (req, res) {
fs.writeFile("./1.png", req.rawBody, function(err) {
if(err) {
console.log(err);
} else {
console.log("The file was saved!");
}
});
res.writeHead(200);
res.end('OK\n');
};
Here's some info which might help. In the middleware, if I log the length of rawBody, it looks to be correct. I am really stuck at how to correctly save the file. All I need is a nudge in the right direction.
Here is a complete express app that works. I hit it with curl --data-binary #photo.jpg localhost:9200 and it works fine.
var app = require("express")();
var fs = require("fs");
app.post("/", function (req, res) {
var outStream = fs.createWriteStream("/tmp/upload.jpg");
req.pipe(outStream);
res.send();
});
app.listen(9200);
I would just pipe the request straight to the filesystem. As to your actual problem, my first guess is req.setEncoding('utf-8'); as utf-8 is for text data not binary data.
For fix your code: I'm with #Peter Lyons, that the error is probably the req.setEncoding('utf-8'); line.
I know the following don't ask your question directly, but proposes an alternative to it by using req.files functionality provided by Express.js, which you are using.
if (req.files.photo && req.files.photo.name) {
// Get the temporary location of the file.
var tmp_path = req.files.photo.path;
// set where the file should actually exists - in this case it is in the "images" directory.
var target_path = './public/profile/' + req.files.photo.name;
// Move the file from the temporary location to the intended location.
fs.rename(tmp_path, target_path, function (error) {
if (!error) {
/*
* Remove old photo from fs.
* You can remove the following if you want to.
*/
fs.unlink('./public/profile/' + old_photo, function () {
if (error) {
callback_parallel(error);
}
else {
callback_parallel(null);
}
});
}
else {
callback_parallel(error);
}
});
}
My problem seems to be the opposite of every node.js question :-) I have a simple forEach loop to read a list of files and insert them into a Couchbase database. This works great, but it never exits after reading all the lines. So I added a counter to shutdown the couchbase connection after all inserts are complete. This works.
This process is intended to load hundreds of thousands of files, so I brought the async module into the mix to batch the inserts into groups of 100. The async.eachLimit is used to iterate over the array and insert documents in batches. Now the orig problem is back. Whatever magic async.eachLimit uses to recognize the process is complete is not happening.
I've been going through javascript scoping, callbacks, async, etc. Google searches are hitting keywords but not this issue. I've reduced the code down to the following testcase. To test, create three files and add their names to testlist.txt.
The async.eachLimit in place works up until it hits the limit, then hangs. Comment this out and uncomment array.forEach line and it works. Thanks in advance!
var fs = require('fs');
var couchbase = require('couchbase');
var async = require('async');
var filelist = 'testlist.txt';
var key_count = 0;
var cb_config = { host: 'localhost:8091', bucket: 'default'};
var db = new couchbase.Connection(cb_config, function(err) {
if (err) {
console.log('ERRR connect to couchbase at config['+cb_config+']');
throw err;
}
});
var insertFile=function(line) {
console.log('LOAD ['+line+']');
fs.readFile(line, function(file_err, f_doc) {
if(file_err) throw file_err;
db.set(line, f_doc, function(db_err, db_res){
if (db_err) {
console.log('FAIL ['+line+'] err['+db_err+']');
} else {
console.log('PASS ['+line+']');
}
key_count--;
if (key_count == 0) {
console.log('DONE Shutting down client, no more keys');
db.shutdown();
}
});
});
}
// read list of files into data array from file filelist
fs.readFile(filelist, function(filelist_err, lines) {
if(filelist_err) throw filelist_err;
// HACK split adds empty line to array, use replace to fix
var array = lines.toString().replace(/\n$/, '').split('\n');
key_count = array.length;
console.log('INIT lines['+key_count+']');
async.eachLimit(array, 2, insertFile, function(err) { console.log('FAIL async err['+err+']');} );
//array.forEach(function(data){insertFile(data);return;});
});
Testcase output using array.forEach:
INIT lines[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
LOAD [files.txt]
PASS [files.little.txt]
PASS [files.big.txt]
PASS [files.txt]
DONE Shutting down client, no more keys
Testcase output using async.eachLimit:
INIT lines[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
PASS [files.little.txt]
PASS [files.big.txt]
... hang, never gets to 3...
After review with a coworker, they spotted my mistake. I missed the async callback in my insertFile function. Adding that in works and allows me to remove the key counter! Solution code below:
var fs = require('fs');
var couchbase = require('couchbase');
var async = require('async');
var filelist = 'testlist.txt';
var key_count = 0;
var cb_config = { host: 'localhost:8091', bucket: 'default'};
var db = new couchbase.Connection(cb_config, function(err) {
if (err) {
console.log('ERRR connect to couchbase at config['+cb_config+']');
throw err;
}
});
var insertFile=function(line, callback) {
console.log('LOAD ['+line+']');
fs.readFile(line, function(file_err, f_doc) {
if(file_err) throw file_err;
db.set(line, f_doc, function(db_err, db_res){
if (db_err) {
console.log('FAIL ['+line+'] err['+db_err+']');
callback(db_err);
} else {
console.log('PASS ['+line+']');
callback();
}
});
});
}
// read list of files into data array from file filelist
fs.readFile(filelist, function(filelist_err, data) {
if(filelist_err) throw filelist_err;
// HACK stoopid bug split adds empty line to array, use replace to fix
var array = data.toString().replace(/\n$/, '').split('\n');
key_count = array.length;
console.log('READ files['+key_count+']');
async.eachLimit(array, 2, insertFile, function(err) {
if (err) console.log('LAST with async err['+err+']');
console.log('DONE Shutting down client, no more keys');
db.shutdown();
});
});
And successful output:
$ node testcase.js
READ files[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
PASS [files.little.txt]
LOAD [files.txt]
PASS [files.big.txt]
PASS [files.txt]
DONE Shutting down client, no more keys
I'm new to NodeJS and I wonder how to properly send the content of several files using FS and Socket.IO.
My problem is more about best practices on Node/javascript than the actual 'raw' logic of my scripts.
So, the purpose of my app is to watch a log File (File1.log) and a Result File (File2.log).
Until the File2.log contains a string (such as "Done", or "Error"), I need to continue sending the result of the File1.log to the client.
When the key ("Error","Done") has been read, I send the result to the client and have to launch the same process for another couple of log/result files - after having closed the FileWatcher on the first one.
In the end, I need to close the connection and stop all of the sleeping FileWatcher processes.
The 'streaming' of my files is working pretty well, but I am confused about the best way to switch between the differents FileWatch Processes and how to notify the client.
Server.JS
/*
* [SomeCode]...
*/
io.sockets.on('connection', function (client) {
//Starting the process for the first couple of files
client.on('logA', function (res) {
var PATH_to_A = "path/to/A/directory/";
readFile(client,PATH_to_A);
});
//Starting the process for the seconde couple of files
client.on('logB', function (res) {
//I need to stop the first readFile Watcher process
var PATH_to_B = "path/to/B/directory/";
readFile(client,PATH_to_B);
});
});
function readFile(client,PATH){
var File1 = path.join(PATH,'File1.log');
var File2 = path.join(PATH,'File2.log');
//Get the file stats
fs.stat(File1,function(err,stats){
if (err) throw err;
//Send the data;
});
//Watch the first file
var w1 = fs.watch(File1,function(status, file){
if(status == "change"){
fs.stat(File1,function(err,stats){
if (err) throw err;
//Send the data;
});
}
});
//Watch the second file
var w2 = fs.watch(File2,function(status, file){
if(status == "change"){
fs.readFile(File2, "utf8", function (err, body) {
if (err) throw err;
//Some Code....
client.emit('done',body);
});
}
});
//Closing FileWatcher
client.on('ack',function(){
w1.close();
w2.close();
});
}
Client.JS
var socket = io.connect('http://127.0.0.1:8000');
//On connect, waiting for the first couple of files
socket.on('connect', function(server) {
socket.emit('init',data);
socket.emit('logA');
});
//If the first process is done, i ask for the second couple of files
socket.on('done',function(message){
socket.emit('ack');
socket.emit('logB');
});
Thanks for your help!