We Keep Coding

Is there any way of calling a function to erease all files before uploading new ones with multer?

I'm trying to pass a function ereaseFiles() before the upload.array() method is called but I can't figure out how to make it.
The main goal is with a put request to delete all files on disk related to that object before of uploading new ones.
I've tried to set the function in the diskStorage section as well as in the callback of the put route. I even tried handling it in the function itself before the upload.array() method was called. I've tried working with promises but that is way too hard for me.
//function to be called (this works)
function ereaseFiles(req) {
glob("uploads/" + req.body.numeroScheda + "*", function (err, files) {
for (const file of files) {
fs.unlink(file, err => {
if (err) throw err;
console.log('successfully deleted files');
});
}
});
}
//My multer setup:
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, './uploads/');
},
filename: function (req, file, cb) {
cb(null, req.body.numeroScheda + "-" + file.originalname);
}
});
const upload = multer({
storage: storage, limits: {
fileSize: 1024 * 1024 * 2,
},
});
//MY EDIT PUT ROUTE
app.put("/immobili/:_id", upload.array('immaginePrincipale', 30), function (req, res) {
const requestedId = req.params._id;
const proprietaImmagini = req.files;
const immagini = proprietaImmagini.map(function (immagine) {
//console.log(immagine.path);
return immagine.path;
});
Immobile.findOneAndUpdate(requestedId, {
numeroScheda: req.body.numeroScheda,
categoria: req.body.categoria,
titolo: req.body.titolo,
sottotitolo: req.body.sottotitolo,
descrizione: req.body.descrizione,
localita: req.body.localita,
locali: req.body.locali,
superficie: req.body.superficie,
camere: req.body.camere,
bagni: req.body.bagni,
immagini: immagini,
}, function (err, updatedImmobile) {
if (err) return console.error(err);
res.redirect("/immobili/" + requestedId);
});
});
What should happen is that all files on disk associated with the object (numeroScheda) get deleted before the new ones are uploaded to keep the storage of files automated and clean.
EDIT 1:
I've created a delete route that works:
app.delete("/immobili/:_id", (req, res) => {
const requestedId = req.params._id;
Immobile.findOne({ _id: requestedId }, function (err, immobile) {
if (err) return console.error(err);
ereaseFiles(immobile);
});
Immobile.findOneAndRemove(requestedId, err => {
if (err) console.error(err);
else res.redirect('/immobili');
});
});
the function ereaseFiles looks now like this:
ereaseFiles = immobile => {
glob("uploads/" + immobile.numeroScheda + "*", function (err, files) {
for (const file of files) {
fs.unlink(file, err => {
if (err) throw err;
});
}
});
cancellato = true;
}
I've tried to implement this in the edit route with the middleware as kindly suggested with this:
app.use("/immobili/:_id", function (req, res, next) {
const requestedId = req.params._id;
let timer = setInterval(() => {
Immobile.findOne({ _id: requestedId }, (err, immobile) => {
if (err) return console.error(err);
ereaseFiles(immobile);
console.log(this.cancellato);
if (this.cancellato) {
clearInterval(timer);
next();
}
});
}, 1000);
});
This works if the uploaded pictures are more or same than before but if less it outputs with strange behaviors (sometimes it uploads some pictures, sometimes none).

You can use a middleware for that. Just make sure that the middleware is positioned before your put request handler.
app.use("/immobili/:_id", function(req,res,next){
eraseFiles(req);
let timer = setInterval(() => {
if(erased){
clearInterval(timer);
next();
}
},100);
})
app.put("/immobili/:_id", upload.array('immaginePrincipale', 30), function (req, res) { ...
EDIT 1:
Please change your eraseFiles function to:
function ereaseFiles(req) {
glob("uploads/" + req.body.numeroScheda + "*", function (err, files) {
for (let i=0;i<files.length-1;i++) {
fs.unlink(files[i], err => {
if (err) throw err;
console.log('successfully deleted files');
if(i===files.length-1)
erased = true;
});
}
});
}
Edit 2: Changed a lot of things. Now the approach is your route will go to the middleware first. Your eraseFiles function will be called. While the erased variable is not true, your put route will not be hit. When the erasedFiles function is complete it will set erased to true. For this to work, you'll have to set erased=false in the file before all this.

I won! The solution was to put a little timer on the next() function as it was firing too soon and the uploads and it was mixing the two. Thanks for your help everyone!
I've also added an ereased variable that turned off and on as the function completes. Thanks to Mr. Web for that.
Here's the answer if someone runs across this, there's some Italian in the code, hopefully is readable enough anyways!
cancellaFoto = immobile => {
cancellato = false;
glob("uploads/" + immobile.numeroScheda + "*", function (err, files) {
for (const file of files) {
fs.unlink(file, err => {
if (err) throw err;
});
}
});
cancellato = true;
}
app.use("/immobili/:_id", function (req, res, next) {
const requestedId = req.params._id;
Immobile.findOne({ _id: requestedId }, (err, immobile) => {
if (err) return console.error(err);
immobile.immagini = [];
cancellaFoto(immobile);
console.log(immobile.immagini);
if (this.cancellato) {
console.log(this.cancellato);
return setTimeout(next, 500);
} else {
return console.log("Aborted");
}
});
});
//EDIT PUT ROUTE
app.put("/immobili/:_id", upload.array('immaginePrincipale', 30), function (req, res) {
const requestedId = req.params._id;
const proprietaImmagini = req.files;
const immagini = proprietaImmagini.map(function (immagine) {
//console.log(immagine.path);
return immagine.path;
});
console.log(immagini);
Immobile.findOneAndUpdate(requestedId, {
numeroScheda: req.body.numeroScheda,
categoria: req.body.categoria,
titolo: req.body.titolo,
sottotitolo: req.body.sottotitolo,
descrizione: req.body.descrizione,
localita: req.body.localita,
locali: req.body.locali,
superficie: req.body.superficie,
camere: req.body.camere,
bagni: req.body.bagni,
immagini: immagini,
}, function (err, updatedImmobile) {
if (err) return console.error(err);
res.redirect("/immobili/" + requestedId);
});
});

Callback is not a function -manage asynchronous call Node js

I am reading files from ftp using the code below.
var JSFtp = require("jsftp");
var config = require('./config.json');
var FtpService = function () {};
// Connect to FTP
var Ftp = new JSFtp({
host: config.ftp.host,
port: config.ftp.port,
user: config.ftp.user,
pass: config.ftp.pass
});
FtpService.prototype.getFTPDirectoryFiles = function (callback) {
Ftp.list(config.ftp.FilePath, function(err, res) {
if(err){
console.log('File Listing Failed', err);
callback(null,err);
return;
}
else{
console.log(res);
callback(null,res);
}
});
};
FtpService.prototype.closeFtp = function () {
console.log('Disconnect to FTP');
};
module.exports = new FtpService();
Now i include this ftp service js file in my index.js as
var ftp = require('./ftpservice.js');
ftpfiles = ftp.getFTPDirectoryFiles();
console.log(ftpfiles);
getFTPDirectoryFiles returns the list of file. But if i call it via index.js i get undefined ftpfiles. This is because of the asynchronous nature of node js.
so i thought of adding callback but
I am getting the error Callback is not defined in function FtpService.prototype.getFTPDirectoryFiles

In this line:
ftpfiles = ftp.getFTPDirectoryFiles()
you are not passing the callback that that function requires and are trying to use a return value that the function does not return.
You need to do something like this:
var ftp = require('./ftpservice.js');
ftp.getFTPDirectoryFiles(function(err, ftpfiles) {
if (err) {
console.log(err);
} else {
console.log(ftpfiles);
}
});

You need to pass a callbackfunction in your function getFTPDirectoryFiles();
var ftp = require('./ftpservice.js');
var ftpFiles;
function setFtpFiles(err, res) {
if (err) throw err;
ftpFiles = res; // to use "ftpFiles" variable later
console.log(res);
}
ftp.getFTPDirectoryFiles(setFtpFiles);

1 Don't change args order to call callback. (replace callback(null,err); and callback(null,res); by callback(err,res);)
2 You need define a specifc function (your callaback) an give it to ftp.getFTPDirectoryFiles().
var JSFtp = require("jsftp");
var config = require('./config.json');
var FtpService = function () {};
// Connect to FTP
var Ftp = new JSFtp({
host: config.ftp.host,
port: config.ftp.port,
user: config.ftp.user,
pass: config.ftp.pass
});
FtpService.prototype.getFTPDirectoryFiles = function (callback) {
Ftp.list(config.ftp.FilePath, function(err, res) {
if(err){
console.log('File Listing Failed', err);
callback(err, res);
return;
}
else{
console.log(res);
callback(err, res);
}
});
};
FtpService.prototype.getFTPDirectoryFilesSimplify = function (callback) {
// no console.log, but very more simple !
Ftp.list(config.ftp.FilePath, callback);
};
FtpService.prototype.closeFtp = function () {
console.log('Disconnect to FTP');
};
and then :
var ftp = require('./ftpservice.js');
ftpfiles = ftp.getFTPDirectoryFiles(function(err,res){
// do your specifc job here using err and res
});
console.log(ftpfiles);

Getting error can not get header after they send when read files from directory?

I am trying to get the name and created date of the files. In the code below it throws error when I call the api. It is reading the directory and printing all the file names but it's not sending back to callback. Any idea what is implemented wrong?
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var currentDate = new Date();
var objToReturn = [];
var logsDirectory = './logs'
function readDirectory(env, callback) {
fs.readdir(logsDirectory + '/' + env, function(err, files) {
// loop through each file
async.eachSeries(files, function(file, done) {
var dirPath = logsDirectory + '/' + env;
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't exist");
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
objToReturn.push(fileInfo);
done();
}
});
});
},
function(err) {
if (err) {
console.info('error', err);
return;
}
// when you're done reading all the files, do something...
console.log('before Callback', objToReturn);
callback(objToReturn);
});
}
exports.readDirectory = readDirectory;
app.js
var stDirectory = require('./app/serverfiles/stDir');
app.get('/getAllFiles',function(req,res){
var env = req.query.env
console.log('printing',env);
stDirectory.readDirectory(env,function(files){
res.json(files);
console.log('Api files',files);
});
});

There are a few issues:
instead of passing the "final" handler to async.eachSeries(), you're passing it to fs.readdir(), so callback will never get called;
you're declaring objToReturn outside of the function, which isn't a good idea because multiple requests could be handled in parallel;
you're not handling any errors properly;
you should really use the Node.js callback idiom of calling callbacks with two arguments, the first being errors (if there are any) and the second being the result of the asynchronous operation.
The code below should fix these issues:
function readDirectory(env, callback) {
let objToReturn = [];
fs.readdir(
logsDirectory + "/" + env,
function(err, files) {
if (err) return callback(err);
// loop through each file
async.eachSeries(files, function(file, done) {
var dirPath = logsDirectory + "/" + env;
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't exist");
return done(err);
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
objToReturn.push(fileInfo);
done();
}
});
}, function(err) {
if (err) {
console.info("error", err);
return callback(err);
}
// when you're done reading all the files, do something...
console.log("before Callback", objToReturn);
callback(null, objToReturn);
}
);
}
// To call it:
stDirectory.readDirectory(env, function(err, files) {
if (err) {
res.sendStatus(500);
} else {
res.json(files);
console.log('Api files',files);
}
});
You should also consider using async.mapSeries() instead of async.eachSeries() and using a separate array (objToReturn).

How to use async with other asynchronous modules?

Here is a working test program without async:
var fs = require('fs');
function test() {
var finalResponse = '', response = '';
function showFinalResponse() {
console.log(finalResponse);
}
function processTheFile(err, data) {
if (err) {
finalResponse = 'Could not read the file';
} else {
response += data;
response += '</body></html>';
finalResponse = response;
}
showFinalResponse();
}
function readTheFile(exists) {
if (!exists) {
finalResponse = 'File does not exist.';
showFinalResponse();
} else {
response += '<!DOCTYPE html><html lang="en-US"><head></head><body>';
fs.readFile('file.txt', 'utf8', processTheFile);
}
}
fs.exists('file.txt', readTheFile);
};
test();
Here is my attempt at getting the same program to work with async waterfall. I'm having trouble with how to pass the callbacks around in the async and the fs calls.
var fs = require('fs');
var async = require('async');
function testAsync() {var finalResponse, response = '';
async.waterfall( [
function checkIfTheFileExists(done) {
fs.exists('file.txt', done);
},
function readTheFile(err, exists, done) {
response += '<!DOCTYPE html><html lang="en-US"><head></head><body>';
fs.readFile('file.txt', 'utf8', done);
},
function processTheFile(err, data, done) {
response += data;
response += '</body></html>';
finalResponse = response;
done(null);
} ],
function showFinalResponse(err) {
if (err) {
if (err.code === 'ENOENT') { // intended to test for file is missing.
finalResponse = 'File does not exist.';
} else { // any other errors.
finalResponse = 'Could not read the file';
}
console.log(err);
}
console.log(finalResponse);
}
);
}
testAsync()
I can't get the async version to work. I'm getting confused with where the callbacks go.

fs.exists is an oddball in that it doesn't provide an error parameter to its callback function. Instead it only provides a single exists parameter that indicates whether the file was found or not. Presumably, if there was an error, exists would be false. As such you need to wrap its callback in your own function so that you can provide a separate error parameter to the waterfall callback:
async.waterfall( [
function checkIfFileExists(done) {
fs.exists('file.txt', function(exists) { done(null, exists); });
},
function makeSureFileExists(exists, done) {
...
Note the warning in the docs, however, that fs.exists shouldn't be used, typically.

fs.exists('file.txt', done(null));
This calls done immediately. You need to pass the actual done function to fs.exists:
fs.exists('file.txt', done);
Same for the others.

Here is my final working version (in case it helps anyone else). Thanks again for your help!
var fs = require('fs');
var async = require('async');
var addErrParm = function (err, done) {return function(exists) {
done(err, exists);
}}
function testAsync() {var finalResponse, response = '';
function checkIfTheFileExists(done) {
fs.exists('file.txt', addErrParm(null, done));
}
function readTheFile(exists, done) {
if (!exists) {
done('notFound');
} else {
response += '<!DOCTYPE html><html lang="en-US"><head></head><body>';
fs.readFile('file.txt', 'utf8', done);
}
}
function processTheFile(data, done) {
response += (data || 'The file is empty') + '</body></html>';
finalResponse = response;
done(null);
}
function showFinalResponse(err) {
if (err) {
finalResponse = (err === 'notFound' ? 'File does not exist.' : 'Could not read the file');
}
console.log(finalResponse);
}
async.waterfall([ checkIfTheFileExists,
readTheFile,
processTheFile
], showFinalResponse);
}
testAsync()
So basically, async requires removing the err parameter (first argument) from all functions except the final callback, and it requires adding a callback ('done') as an extra parameter on all functions except the final callback.
Also, if there is no err parameter like with fs.exists, you have to create a function to simulate an err parameter so async can remove it.

Bulk upload attachments to couchDB using node.js nano module

I'm trying to bulk upload attachments to CouchDB using node.js and nano.
First, the walk module is used to find all files in upload folder and create array from them.
Next, each file from the array is supposed to be inserted into CouchDB via pipe and nano module.
However, the final result is that only one attachment has been uploaded.
var nano = require('nano')('http://localhost:5984')
var alice = nano.use('alice');
var fs = require('fs');
var walk = require('walk');
var files = [];
// Walker options
var walker = walk.walk('./uploads', {
followLinks: false
});
// find all files and add to array
walker.on('file', function (root, stat, next) {
files.push(root + '/' + stat.name);
next();
});
walker.on('end', function () {
// files array ["./uploads/2.jpg","./uploads/3.jpg","./uploads/1.jpg"]
files.forEach(function (file) {
//extract file name
fname = file.split("/")[2]
alice.get('rabbit', {revs_info: true}, function (err, body) {
fs.createReadStream(file).pipe(
alice.attachment.insert('rabbit', fname, null, 'image/jpeg', {
rev: body._rev
}, function (err, body) {
if (!err) console.log(body);
})
)
});
});
});

This is because you are mixing an asynchronous api with assumptions of this being synchronous.
After the first request you will get conflicts, cause the rabbit document has changed.
Can you confirm this using NANO_ENV=testing node yourapp.js?
I recommend using async if this is the problem

var nano = require('nano')('http://localhost:5984')
var alice = nano.use('alice');
var fs = require('fs');
var walk = require('walk');
var files = [];
// Walker options
var walker = walk.walk('./uploads', {
followLinks: false
});
walker.on('file', function (root, stat, next) {
files.push(root + '/' + stat.name);
next();
});
walker.on('end', function () {
series(files.shift());
});
function async(arg, callback) {
setTimeout(function () {callback(arg); }, 100);
}
function final() {console.log('Done');}
function series(item) {
if (item) {
async(item, function (result) {
fname = item.split("/")[2]
alice.get('rabbit', { revs_info: true }, function (err, body) {
if (!err) {
fs.createReadStream(item).pipe(
alice.attachment.insert('rabbit', fname, null, 'image/jpeg', {
rev: body._rev
}, function (err, body) {
if (!err) console.log(body);
})
)
}
});
return series(files.shift());
});
}
else {
return final();
}
}