We Keep Coding

Node.js - Cannot append global variable when using fs

Im trying to read multiple xml files and parse data from them and i managed to do that but now new problem appeared.
allData variable is never changed, no matter what i do. What am i supposed to do here?
I dont know what to do or what to try, this is my first time working with files and im honestly surprised ive managed to come this far.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
fs.readFile(__dirname + '\\parse\\' + filename, 'utf-8', function (err, content) {
if (err) {
console.log(err)
return;
}
parseString(content, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
})
});
console.log("All data: ",allData)
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
SOLVED
adjusted code to use.readFileSync()(removed callback function) and now it works.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
let file = fs.readFileSync(__dirname + '\\parse\\' + filename, 'utf-8')
parseString(file, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
console.log("All data: ",allData)
});
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)

The .readdir() and .readFile() methods are async, so in fact the console.log() is executed before all of the readFile operations.
In order to access the allData variable after these operations are complete, you have to either make them sync using .readFileSync() instead or you need to promisify the .readFile() method and wait for all of the promises to resolve.

Callback is not a function -manage asynchronous call Node js

I am reading files from ftp using the code below.
var JSFtp = require("jsftp");
var config = require('./config.json');
var FtpService = function () {};
// Connect to FTP
var Ftp = new JSFtp({
host: config.ftp.host,
port: config.ftp.port,
user: config.ftp.user,
pass: config.ftp.pass
});
FtpService.prototype.getFTPDirectoryFiles = function (callback) {
Ftp.list(config.ftp.FilePath, function(err, res) {
if(err){
console.log('File Listing Failed', err);
callback(null,err);
return;
}
else{
console.log(res);
callback(null,res);
}
});
};
FtpService.prototype.closeFtp = function () {
console.log('Disconnect to FTP');
};
module.exports = new FtpService();
Now i include this ftp service js file in my index.js as
var ftp = require('./ftpservice.js');
ftpfiles = ftp.getFTPDirectoryFiles();
console.log(ftpfiles);
getFTPDirectoryFiles returns the list of file. But if i call it via index.js i get undefined ftpfiles. This is because of the asynchronous nature of node js.
so i thought of adding callback but
I am getting the error Callback is not defined in function FtpService.prototype.getFTPDirectoryFiles

In this line:
ftpfiles = ftp.getFTPDirectoryFiles()
you are not passing the callback that that function requires and are trying to use a return value that the function does not return.
You need to do something like this:
var ftp = require('./ftpservice.js');
ftp.getFTPDirectoryFiles(function(err, ftpfiles) {
if (err) {
console.log(err);
} else {
console.log(ftpfiles);
}
});

You need to pass a callbackfunction in your function getFTPDirectoryFiles();
var ftp = require('./ftpservice.js');
var ftpFiles;
function setFtpFiles(err, res) {
if (err) throw err;
ftpFiles = res; // to use "ftpFiles" variable later
console.log(res);
}
ftp.getFTPDirectoryFiles(setFtpFiles);

1 Don't change args order to call callback. (replace callback(null,err); and callback(null,res); by callback(err,res);)
2 You need define a specifc function (your callaback) an give it to ftp.getFTPDirectoryFiles().
var JSFtp = require("jsftp");
var config = require('./config.json');
var FtpService = function () {};
// Connect to FTP
var Ftp = new JSFtp({
host: config.ftp.host,
port: config.ftp.port,
user: config.ftp.user,
pass: config.ftp.pass
});
FtpService.prototype.getFTPDirectoryFiles = function (callback) {
Ftp.list(config.ftp.FilePath, function(err, res) {
if(err){
console.log('File Listing Failed', err);
callback(err, res);
return;
}
else{
console.log(res);
callback(err, res);
}
});
};
FtpService.prototype.getFTPDirectoryFilesSimplify = function (callback) {
// no console.log, but very more simple !
Ftp.list(config.ftp.FilePath, callback);
};
FtpService.prototype.closeFtp = function () {
console.log('Disconnect to FTP');
};
and then :
var ftp = require('./ftpservice.js');
ftpfiles = ftp.getFTPDirectoryFiles(function(err,res){
// do your specifc job here using err and res
});
console.log(ftpfiles);

Getting error can not get header after they send when read files from directory?

I am trying to get the name and created date of the files. In the code below it throws error when I call the api. It is reading the directory and printing all the file names but it's not sending back to callback. Any idea what is implemented wrong?
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var currentDate = new Date();
var objToReturn = [];
var logsDirectory = './logs'
function readDirectory(env, callback) {
fs.readdir(logsDirectory + '/' + env, function(err, files) {
// loop through each file
async.eachSeries(files, function(file, done) {
var dirPath = logsDirectory + '/' + env;
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't exist");
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
objToReturn.push(fileInfo);
done();
}
});
});
},
function(err) {
if (err) {
console.info('error', err);
return;
}
// when you're done reading all the files, do something...
console.log('before Callback', objToReturn);
callback(objToReturn);
});
}
exports.readDirectory = readDirectory;
app.js
var stDirectory = require('./app/serverfiles/stDir');
app.get('/getAllFiles',function(req,res){
var env = req.query.env
console.log('printing',env);
stDirectory.readDirectory(env,function(files){
res.json(files);
console.log('Api files',files);
});
});

There are a few issues:
instead of passing the "final" handler to async.eachSeries(), you're passing it to fs.readdir(), so callback will never get called;
you're declaring objToReturn outside of the function, which isn't a good idea because multiple requests could be handled in parallel;
you're not handling any errors properly;
you should really use the Node.js callback idiom of calling callbacks with two arguments, the first being errors (if there are any) and the second being the result of the asynchronous operation.
The code below should fix these issues:
function readDirectory(env, callback) {
let objToReturn = [];
fs.readdir(
logsDirectory + "/" + env,
function(err, files) {
if (err) return callback(err);
// loop through each file
async.eachSeries(files, function(file, done) {
var dirPath = logsDirectory + "/" + env;
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't exist");
return done(err);
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
objToReturn.push(fileInfo);
done();
}
});
}, function(err) {
if (err) {
console.info("error", err);
return callback(err);
}
// when you're done reading all the files, do something...
console.log("before Callback", objToReturn);
callback(null, objToReturn);
}
);
}
// To call it:
stDirectory.readDirectory(env, function(err, files) {
if (err) {
res.sendStatus(500);
} else {
res.json(files);
console.log('Api files',files);
}
});
You should also consider using async.mapSeries() instead of async.eachSeries() and using a separate array (objToReturn).

How to outsource upload function as service notde.js

I'm trying to write a piece of my upload code as a service, because I need that function overall in my software. My project use sails.js - here the doc for a service.
In a controller I got this code, which uploads a file and after success it calls the function saveTheCampaign() and saves the file information in the DB.
req.file('logo').upload({
maxBytes: 10000000,
saveAs: function (uploadFile, cb) {
cb(null, Date.now() + uploadFile.filename);
},
dirname: sails.config.appPath + '/assets/images/campaign/'
}, function (err, uploadedFiles) {
if (err) {
return res.json(500, err);
}
else if (uploadedFiles.length === 0) {
// proceed without files
res.json({ error: "No image found for upload!"})
}
else {
// Success: handle uploaded file
var fileName = uploadedFiles[0].fd.split('\\');
params["logo"] = fileName[fileName.length - 1];
sails.controllers.campaign.saveTheCampaign(params, req, res);
}
});
saveTheCampaign: function (params, req, res) { //...}
Now I wanted to write this snippet as a service. My service is called UploadService and has a function called upload(), services can take two(2) arguments, option and a callback function. So I tried this to call the upload function of my service:
UploadService.upload(options, sails.controllers.campaign.saveTheCampaign(params, req, res));
The problem is, the params of the callback function (params, req, res) are not known at the time of the call, I get them AFTER the upload function is finished. How can I handle this?

One way to make this happen by using Q Promise Library. The snippet below is a working example for the same. You'll need to set value for sails.config.appPath.
Routes.js
'POST /upload' : 'CampaignController.upload'
UploadService.js
let q = require("q"); // https://github.com/kriskowal/q
module.exports = {
upload: function(options) {
let deferred = q.defer();
options['req'].file(options['fileFieldName']).upload({
maxBytes: 10,
saveAs: function(uploadedFile, cb) {
cb(null, Date.now() + uploadedFile.filename);
},
dirname: sails.config.appPath + '/assets/images/'
}, function(err, uploadedFiles) {
if (err) {
deferred.reject(err);
} else if (uploadedFiles.length === 0) {
// proceed without files
deferred.reject("No image found for upload!");
} else {
// Success: handle uploaded file
let params = [];
var fileName = uploadedFiles[0].fd.split('\\');
params["logo"] = fileName[fileName.length - 1];
deferred.resolve(params)
}
});
return deferred.promise;
}
}
CampaignController.js
module.exports = {
upload: function(req, res) {
let options = [];
options['fileFieldName'] = 'logo';
options['req'] = req;
UploadService.upload(options)
.then((params) => {
sails.controllers.campaign.saveTheCampaign(params);
res.send("Campaign Saved Successfully");
})
.catch((err) => res.send(err))
},
saveTheCampaign: function(params) {
console.log(`campaign ${params['logo']} saved`);
}
}

Multiple node.js requests

My controller is using the request package to make server-side HTTP requests to another API. My question is how can I make MULTIPLE of these requests? Here is my current code:
** UPDATED CODE **
module.exports = function (req, res) {
var context = {};
request('http://localhost:3000/api/single_project/' + req.params.id, function (err, resp1, body) {
context.first = JSON.parse(body);
request('http://localhost:3001/api/reports/' + req.params.id, function (err, resp2, body2) {
context.second = JSON.parse(body2); //this line throws 'SyntaxError: Unexpected token u' error
res.render('../views/project', context);
});
});
};
I need to make two more of those calls and send the data returned from it to my template...
Can someone help?
Thanks in advance!

function makePromise (url) {
return Promise(function(resolve, reject) {
request(url, function(err, resp, body) {
if (err) reject(err);
resolve(JSON.parse(body));
});
});
}
module.exprts = function (req, res) {
let urls = ['http://localhost:3000/api/1st',
'http://localhost:3000/api/2st',
'http://localhost:3000/api/3st'].map((url) => makePromise(url));
Promise
.all(urls)
.then(function(result) {
res.render('../views/project', {'first': result[0], 'second': result[1], 'third': result[2]});
})
.catch(function(error){
res.end(error);
});
}
You can use Promise lib in latest nodejs.

Simple solution
Nest request calls. This is how you can handle the dependency between requests. Just make sure your parameters are unique across scopes if needed.
module.exports = function (req, res) {
var context = {};
request('http://localhost:3000/api/1st', function (err, resp1, body) {
var context.first = JSON.parse(body);
request('http://localhost:3000/api/2nd', function (err, resp2, body) {
context.second = JSON.parse(body);
request('http://localhost:3000/api/3rd', function (err, resp3, body) {
context.third = JSON.parse(body);
res.render('../views/project', context);
});
});
});
};

Simplest way if you use bluebird promise library:
var Promise = require('bluebird');
var request = Promise.promisify(require('request'));
module.exports = function (req, res) {
var id = req.params.id;
var urls = [
'http://localhost:3000/api/1st/' + id,
'http://localhost:3000/api/2st/' + id,
'http://localhost:3000/api/3st/' + id
];
var allRequests = urls.map(function(url) { return request(url); });
Promise.settle(allRequests)
.map(JSON.parse)
.spread(function(json1, json2, json3) {
res.render('../views/project', { json1: json1 , json2: json2, json3: json3 });
});
});
it executes all requests even if one (or more) fails