I am using NodeJS to create an express endpoint that will retrieve the metadata from my images stored on my server. I have the following code for the logic of the endpoint:
/*
* Gallery Controller
*/
var fs = require('fs'),
_ = require('underscore'),
im = require('imagemagick');
/**
* List paths to images stored in selected gallery
*/
exports.list = function(req, res) {
var dir = 'public/images/' + req.params.id;
fs.readdir(dir, function(err, files) {
if (err) return res.send({error: 'No gallery found with provided id'}, 404);
if (files.length > 0) {
var collection = [],
myData = {};
files.forEach(function(file) {
if(file === '.DS_Store') return;
im.readMetadata( dir + '/' + file, function(err, metadata) {
if (err) throw err;
myData = metadata;
console.log(myData); // logs as object with expected data
});
console.log(myData); // logs as empty {}
collection.push(myData);
});
console.log(collection); // logs as [ {}, {} ]
res.json(collection, 200);
} else {
res.json({error: 'Selected gallery is empty'}, 404);
}
});
};
I've listed what the logs appear as in the terminal, why am I getting this scoping issue? I can't seem to wrap my head around it. If I try to return the metadata obj and assign it to the var, I get the following error: TypeError: Converting circular structure to JSON
Use the async module, it'll improve your life in many ways.
The problem you are having is a common one I see, and it is that your loop is asynchronous, but you are treating it as something serial.
Instead of doing files.forEach, you want to loop them asynchronously and then do some more stuff when the looping is done. You can use async.each for that.
async.each(files, function (file, next) {
if (file === '.DS_Store') return next();
im.readMetadata(path.join(dir, file), function (e, data) {
collection.push(data);
next(err);
});
}, function (err) {
if (err) throw err;
console.log(collection);
});
As an alternative, an even more appropriate solution might be to use async.map.
async.map(files, function (file, next) {
if (file === '.DS_Store') return next();
im.readMetadata(path.join(dir, file), next);
}, function (err, collection) {
if (err) throw err;
console.log(collection);
});
You need to restructure your code:
files.forEach(function(file, i) {
if (file === '.DS_Store') return; // see text
im.readMetadata( dir + '/' + file, function(err, metadata) {
if (err) throw err;
collection.push(metadata);
if (i === files.length - 1) {
res.json(collection); // see text
}
});
});
The reason is that the metadata is only available when the callback function to readMetadata is called; that's how asynchronous I/O works in Node.
In that callback, you add the metadata to the collection. If the iteration of the forEach has reached the final element (i is the index of the current element, when its value is one less than the size of the array, it's the last element), the response is sent.
Two issues:
if .DS_Store is the last/only file in the directory, this code will fail because it will never send back a response; I'll leave it to you to deal with that case ;)
res.json will, by default, return a 200 status so you don't have to specify it; if you do want to specify a status, it needs to be res.json(200, collection) (arguments swapped)
Related
I'm writing a function for my Node.js server (using express as a framework, MongoDB as the database). I have to call two mongoose queries:
(1) Find data filtered by parameters of one collection
(2) Aggregate data of another collection (needs result from the first query because one attribute gets appended to these results)
I now use the async.waterfall a function that makes it possible to ensure that the second operation only gets triggered after the first one has finished.
The variable resultResponse (at the end of my code snippet) has the right value when I run the function.
The problem I am struggling with:
The server throws the error TypeError: callback is not a function for the first callback.
This type error is also the answer I receive via postman calling /getRoutes.
So res.send() doesn't send the variable resultResponse back (which is correct), but the error.
Does anyone know, how I can solve this problem? Thanks a lot!
let async = require('async');
app.get('/getRoutes', function (req, res) {
var param = req.query;
let resultResponse;
async.waterfall([
(1)
function (param, callback) {
let dataCallback;
// first mongoose operation
Route.find(param).lean().exec(function (err, data) {
if (err)
throw err;
dataCallback = data;
}
.then(callback(null, dataCallback)));
},
TypeError: callback is not a function
(2)
function (aRoutes, callback) {
let oRoutes = [];
for (let i in aRoutes) {
// second mongoose operation
Rating.aggregate([{$match: {route: aRoutes[i]._id}}
, {
$group:
{_id: null, rating: {$avg: '$rating'}}
}
]).then(function (response) {
let oneRoute = aRoutes[i];
let avgRating = response[0].rating;
oneRoute.avg_rating = avgRating;
oRoutes.push(oneRoute);
if (i == (aRoutes.length - 1)) {
callback(null, oRoutes);
}
});
}
}],
function (err, result) {
if (err) throw err;
resultResponse = result; //console.log -> right result here
});
res.send(resultResponse); });
Your first function will be seeking callback function as the first parameter. Check here for more details. param is defined globally so you don't need to pass.
function (callback) {
let dataCallback;
// first mongoose operation
Route.find(param).lean().exec(function (err, data) {
if (err)
throw err;
dataCallback = data;
}).then(function(){
callback(null, dataCallback);
});
}
OR
Simply return data insteadof creating undefined object.
function (callback) {
// first mongoose operation
Route.find(param).lean().exec(function (err, data) {
if (err)
throw err;
return data;
}).then(function(dataCallback){
callback(null, dataCallback);
});
}
I am using this async module for asynchronously requesting
web content with the help of another module request, as this is an asynchronous call.
Using async.each method, for requesting data from each link,
the result is also successfully returned by the scrap() function (which I have wrote to scrap returned html data
and return it as array of fuel prices by state).
Now, the problem is that when I try to return prices back to async.each() using cb(null, prices), it shows console.log(prices) as undefined
but logging inside the _check_fuel_prices(), works fine. It seems the callback works with only one argument
(or error only callback, as show as an example in the async.each link above). What if I want to it return prices (I can change it with error like cb(prices), but I also want to log error).
router.get('/someRoute', (req, res, next) => {
const fuels = ['diesel', 'petrol'];
async.each(fuels, _check_fuel_prices, (err, prices) => {
if (!err) {
res.statusCode = 200;
console.log(prices);
return res.json(prices);
}
res.statusCode = 400;
return res.json(err);
});
function _check_fuel_prices(fuel, cb) {
let prices = '';
const url_string = 'http://some.url/';
request(`${url_string}-${fuel}-price/`, (error, response, html) => {
if (error) {
cb(error, null);
return;
}
if (response.statusCode === 404) {
console.log(response.statusCode);
cb('UNABLE TO FIND PAGE', null);
return;
}
prices = scrap(html, fuel);
console.log(prices);
cb(null, prices);
return;
});
}
});
As #generalhenry points out, I was able to get the prices by using async.map which returns error first callback instead of error only apart from that async.series can be used here by slightly changing the code.
I'm trying to establish communication between two node.js scripts.
The first one does a get request and write the response in a file.
The second watch the file after changes, then he read it, and prompt the result.
The first (get then write)
var request = require('request');
var parseString = require('xml2js').parseString;
var fs = require('fs');
//Some needed variables
streamInterval = setInterval(function() {
request.get(addr, function (error, response, body) {
if (!error && response.statusCode == 200) {
parseString(body,{ explicitArray : false, ignoreAttrs : true }, function (err, result) {
var jsonResult = JSON.stringify(result);
var result = JSON.parse(jsonResult);
fs.writeFile(outputDeparts, JSON.stringify(result, null, 4), function(err) {
if(err) {
console.log(err);
}
});
});
}else{
console.log("An error occured : " + response.statusCode);
}
}).auth(LOGIN,PASS,true);
}, 30000);
The second (watch after changes, read and prompt)
var fs = require('fs');
//Some needed variables
fs.watch(outputDeparts, (eventType, filename) => {
console.log(`event type is: ${eventType}`);
if (filename) {
console.log(`filename provided: ${filename}`);
fs.readFile(outputDeparts, 'utf8', function (err, data) {
if (err){
throw err;
}else{
console.log('start parsing');
console.log(data);
var result = JSON.parse(data);
var departs = result["passages"]["train"];
console.log(`next train [${departs[0]["num"]}] at : ${departs[0]["date"]}`);
}
});
} else {
console.log('filename not provided');
}
});
The first time the file is changed everything is ok ! But after 30 second, at the second change I get the following error :
undefined:1
SyntaxError: Unexpected end of input
at Object.parse (native)
at /Users/adobe/Documents/workspace/playWithNode/watchFile.js:17:23
at tryToString (fs.js:414:3)
at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:401:12)
I thought it was a problem with reading/writting a file asynchronously but didn't manage to find a fix...
Is everyone able to help me or having clues ? Thanks
..it isn't encouraged to access the file system asynchronously because, while accessing, the target file can be modified by something in the App Life Cycle.
I suggest you to use fs.readFileSync and to wrap the JSON.parse in a try-catch
var fs = require('fs');
function onNextTrain(data) {
console.log("onNextTrain", data);
return data;
}
fs.watch(outputDeparts, (eventType, filename) => {
return new Promise((resolve, reject) => {
if(!filename) {
return reject("FILE NAME NOT PROVIDED");
}
let
data = fs.readFileSync(filename, 'utf8'),
result
;
try {
result = JSON.parse(data);
} catch(error) {
result = data;
}
return resolve(result);
})
.then(onNextTrain)
.catch(err => console.log("error", err))
;
});
Try change code in second file to
var result;
try {
result = JSON.parse(data);
} catch (err) {
console.error(err, data);
}
because SyntaxError: Unexpected end of input it's JSON.parse error.
In first file I see potential problem in
fs.writeFile(outputDeparts, data, function(err) { ...
Callback function must provide ordered write, but your code - only logged error.
Node.js docs
Note that it is unsafe to use fs.writeFile multiple times on the same file without waiting for the callback. For this scenario, fs.createWriteStream is strongly recommended.
I want to render page when CSS will be loaded. Function css_parser.getCSSFiles() reads file asynchronously and sends CSS content to variable css.cssFile . How I can force res.render to wait for end of file reading?
router.get('/main', function(req, res) {
var directory = '../app/public/stylesheets/',
file = 'style_720.css';
css_parser.getCSSFiles(directory,file);
app.locals.css = css.cssFile;
res.render('ua', {
css: app.locals.css,
});
});
//js module
getCSSFiles: function(directory, fileName) {
var array = css.cssFile;
fs.readFile(directory + fileName, 'utf8', function(err, data) {
if (err) {
return console.log(err);
}
(array.push(data));
});
}
You will have to change getCSSFiles() to accept a callback as an argument and then have it call that callback when it is done and you can then call res.render() from inside that callback. This is the usual way to know when an async operation is done in node.js.
There is no other way to know when it is done. It has to tell you via a callback or some sort of event notification (one or the other).
If you show us the code for getCSSFiles(), we can likely help you more specifically.
What you want to end up with (after modifying getCSSFiles()) is something like this:
router.get('/main', function(req, res) {
var directory = '../app/public/stylesheets/',
file = 'style_720.css';
css_parser.getCSSFiles(directory,file, function(err, data) {
if (!err) {
res.render('ua', {css: data});
}
});
});
And, change getCSSFiles to this:
getCSSFiles: function(directory, fileName, callback) {
fs.readFile(directory + fileName, 'utf8', function(err, data) {
if (err) {
callback(err);
return console.log(err);
}
callback(null, data);
});
}
Simple upload method using node.js+express.js:
upload: function(req, res, next){
//go over each uploaded file
async.each(req.files.upload, function(file, cb) {
async.auto({
//create new path and new unique filename
metadata: function(cb){
//some code...
cb(null, {uuid:uuid, newFileName:newFileName, newPath:newPath});
},
//read file
readFile: function(cb, r){
fs.readFile(file.path, cb);
},
//write file to new destination
writeFile: ['readFile', 'metadata', function(cb,r){
fs.writeFile(r.metadata.newPath, r.readFile, function(){
console.log('finished');
});
}]
}, function(err, res){
cb(err, res);
});
}, function(err){
res.json({success:true});
});
return;
}
The method iterates each uploaded file, creates a new filename and writes it to a given location set in metadata.
console.log('finished');
is fired when writing is finished, however the client never receives a response. After 2 minutes the request is cancled, however the file was uploaded.
Any ideas why this method does not return any response?
You're using readFile, which is async, too, and works like this:
fs.readFile('/path/to/file',function(e,data)
{
if (e)
{
throw e;
}
console.log(data);//file contents as Buffer
});
I you could pass a reference to a function here, to take care of this, but IMO, it'd be way easier to simply use readFileSync, which returns the Buffer directly, and can be passed to writeFile without issues:
fs.writeFile(r.metadata.newPath, r.readFileSync, function(err)
{
if (err)
{
throw err;
}
console.log('Finished');
});
Check the docs for readFile and writeFile respectively