I'm building a site with node/express/mongoose and it needs to do the following things when viewing a submission.
The problem I'm running into is doing db fetches in a non-serial fashion. For example, I'll do a few calls to fetch some data, but some of the calls might not finish until the execution context goes to the other. Tried to use the npm module, async, but am having trouble trying to figure out how I would integrate it.
Here is my code:
var getViewCount = function(submissionId) {
Submission.getSubmissionViewCount({
submissionId : submissionId
}, function(err, count) {
if (err) {
throw err;
}
if (count) {
return count;
}
});
};
var getVotes = function(submissionId) {
console.log('getvotes');
Submission.getSubmissionVotes({
submissionId : submissionId
}, function(err, votes) {
return votes;
});
};
var getSubmission = function(id) {
Submission.getSubmission({
id : id
}, function(err, submission) {
if (err) {
throw err;
}
if (submission) {
return submission;
}
});
};
var renderSubmission = function(title, submission, views) {
res.render('submission', {
title: submission.title + ' -',
submission: submission,
views: views.length
});
};
How do I use this with async? Or should I be using async.series isntead of async.async?
async.series([
function(callback) {
var submission = getSubmission(id);
callback(null, submission);
},
function(callback) {
// getViewCount(submissionId);
},
function(callback) {
// getVotes(submissionId);
},
function(callback) {
//renderSubmission(title, submission, views);
}
], function(err, results) {
console.log(results);
});
Basically I want to fetch the views and votes first and then render my submission.
TheBrain's description of the overall structural changes that you should make to your code is accurate. The basic methodology in Node is to nest a series of callbacks; very rarely should you require functions that actually return a value. Instead, you define a function that takes a callback as parameter and pass the result into that callback. Please review the code below for clarification (where cb is a callback function):
var getViewCount = function(submissionId, cb) {
Submission.getSubmissionViewCount({
submissionId : submissionId
}, function(err, count) {
if (err) {
throw err;
}
if (cb) {
cb(count);
}
});
};
var getVotes = function(submissionId, cb) {
console.log('getvotes');
Submission.getSubmissionVotes({
submissionId : submissionId
}, function(err, votes) {
if (cb) {
cb(votes);
}
});
};
var getSubmission = function(id, cb) {
Submission.getSubmission({
id : id
}, function(err, submission) {
if (err) {
throw err;
}
if (cb) {
cb(submission);
}
});
};
var renderSubmission = function(submissionId) {
getSubmission(submissionId, function (submission) {
if (!submission) {
// unable to find submission
// add proper error handling here
} else {
getViewCount(submissionId, function (viewCount) {
res.render('submission', {
title: submission.title + ' -',
submission: submission,
views: viewCount
});
});
}
};
};
Related
Let's say i need to constantly collecting some data from a lot of clients and in parallel running some complex loop that solving some stuff with this data. How can i do it? Should i just write this in my piece of code:
app.get('/', function(req, res) {
res.sendFile(__dirname + '/public/views/index0.html');
});
io.sockets.on('connection', function(socket) {
// SOME STUFF WITH THE SOCKET
socket.on('disconnect', function(data) {
//SOME OTHER STUFF
});
});
while(...) {
//THE LOOP STUFF
}
Or i need to use the setTimeout() and setInterval() functions? How can i do the loop on the server that runs in parallel with the callbacks' stuff?
Don’t use while for make it, this block a thread. setTimeout() will run only once. You need to use setInterval() function.
You can use the async module to handle the async operation with the callback or use promise to avoid callback.
Here how I handle a complex async for each operation, that might helpfull to you get idea handeling ayncs forach
var cond = { _schedule: schedule_id }; // find curse by schedule id
Course.find(cond, function (err, courses) {
if (err) {
callback({ "success": false, "message": "Not able to update" });
} else {
async.forEachLimit(courses, 1, function (course, coursesCallback) {
async.waterfall([
function (callback) {
var schedule_date = moment(change_data.date).format('YYYY-MM-DD') + "T00:00:00.000Z"
var Assignmentcond = {
assignment_schedule_order: {
$gte: schedule_date
},
_course: course._id,
_schedule: schedule_id,
_user: userid
};
Assignment.find(Assignmentcond)
.populate({
path: '_course',
})
.lean()
.sort({ assignment_schedule_order: 1 })
.exec(function (err, AssignmentList) {
if (err) {
callback(null, '');
} else {
//console.log("------------------AssignmentList---------------------------");
//console.log(AssignmentList);
async.forEachLimit(AssignmentList, 1, function (ThisAssignmentCell, ThisAssignmentCellCallback) {
async.waterfall([
function (callback) {
var SearchObj = items;
var lebelObject = {};
for (var i = 0, flag = 0, insert = 0; i < SearchObj.length; i++) {
if (SearchObj[i].date == ThisAssignmentCell.assignment_schedule_date) {
flag = 1;
}
if (flag == 1 && SearchObj[i].label != "") {
if (ThisAssignmentCell.day == SearchObj[i].day_index) {
insert = 1;
var lebelObject = SearchObj[i];
break;
}
}
}
callback(null, ThisAssignmentCell, lebelObject, insert);
},
function (ThisAssignmentCell, SearchObj, insert, callback) {
console.log('----------------------');
console.log('ThisAssignmentCell', ThisAssignmentCell);
console.log('SearchObj', SearchObj);
console.log('----------------------');
if (insert > 0) {
var query = { _id: ThisAssignmentCell._id },
fields = {
assignment_date: moment(SearchObj.date).format('MM/DD/YYYY'),
assignment_schedule_date: moment(SearchObj.date).format('YYYY-MM-DD'),
assignment_schedule_order: new Date(SearchObj.date),
day: SearchObj.day_index,
dayNum: SearchObj.weekday_num
},
options = { upsert: false };
Assignment.update(query, fields, options, function (err, affected) {
callback(null, '');
});
} else {
// var cond = { _id: ThisAssignmentCell._id};
// Assignment.remove(cond)
// .exec(function (err, cnt) {
// callback(null, '');
// });
}
}
], function (err, result) {
// result now equals 'done'
console.log('done')
ThisAssignmentCellCallback();
});
}, function (err) {
console.log("Assignment For Loop Completed");
callback(null, AssignmentList);
});
}
});
}
], function (err, result) {
// result now equals 'done'
console.log('done')
coursesCallback();
});
}, function (err) {
console.log("courses For Loop Completed");
});
}
});
I am writing this code as a project for a customer
and when i go to a show route i got this 500 internal server error
http.get('/files/:id', function(req, res) {
var vid;
var pap;
Videos.find({}, function(err, videos) {
if (err) {
console.log(err);
} else {
vid = videos;
}
});
Papers.find({}, function(err, file) {
if (err) {
console.log(err);
} else {
pap = file;
}
});
Material.findById(req.params.id, function(err, found) {
if (err) {
console.log(err);
} else {
res.render('files', {
file: pap,
video: vid,
current: found
});
}
});
});
this is my show route code.
Note : if i reload the page the error is gone and the page open.
The reason is you need to wait for all the database queries to finish before rendering. In your code, it is possible for the page to render before the other two queries have completed and returned their data. The good news is that Mongoose supports Promises for asynchronous functions.
http.get('/files/:id', function(req, res) {
Promise.all([
Videos.find({}).exec(),
Papers.find({}).exec(),
Material.findById(req.params.id).exec()
]).then( ([video, paper, material]) => {
res.render('files', {
file: paper,
video: video,
current: material
});
}).catch( error => console.log(error) );
});
The functions you're using with Mongoose are asynchronous in nature; the variables vid and pap are not initialized when you run res.render. When you attempt to use those variables in your frontend (template like Jade, Handlebars EJS, I don't know what you're using), they are undefined, and subsequently cause the 500 error. You'll need to run the functions such that the results of all Mongoose queries are available to res.render when it runs; either using an async NodeJS library, or calling each function within one another and then calling res.render at the end.
Solution 1: Using async Node module
var async = require('async');
async.parallel([
// Each function in this array will execute in parallel
// The callback function is executed once all functions in the array complete
function (cb) {
Videos.find({}, function(err, videos) {
if (err) {
return cb(err);
} else {
return cb(null, videos);
}
});
},
function (cb) {
Papers.find({}, function(err, papers) {
if (err) {
return cb(err);
} else {
return cb(null, papers);
}
});
},
function (cb) {
Material.findById(req.params.id, function(err, found) {
if (err) {
return cb(err);
} else {
return cb(null, found);
}
});
}
], function (err, results) {
if (err) {
// If any function returns an error
// (first argument), it will be here
console.log(err);
}
else {
// Even though the functions complete asynchronously,
// the order in which they are declared in the array
// will correspond to the position in the array
// if it returns anything as a second argument.
var videos = results[0];
var files = results[1];
var found = results[2];
res.render('files', {
file: files,
video: videos,
current: found
});
}
});
Solution 2: Nested Callbacks
Videos.find({}, function(err, videos) {
var vid = videos;
if (err) {
console.log(err);
} else {
Papers.find({}, function(err, file) {
var pap = file;
if (err) {
console.log(err);
} else {
Material.findById(req.params.id, function(err, found) {
if (err) {
console.log(err);
} else {
res.render('files', {
file: pap,
video: vid,
current: found
});
}
});
}
});
}
});
How can I return a object of data returned by asynchronous function called multiple times from within a asynchronous function.
I'm trying to implement like this :
var figlet = require('figlet');
function art(dataToArt, callback)
{
var arry[];
figlet(dataToArt, function(err, data) {
if (err) {
console.log('Something went wrong...');
console.dir(err);
return callback('');
}
arry[0] = data;
callback(arry);
});
figlet(dataToArt, function(err, data) {
if (err) {
console.log('Something went wrong...');
console.dir(err);
return callback('');
}
arry[1] = data;
callback(arry);
});
}
art('Hello World', function (data){
console.log(data);
});
How can I do it correctly, I searched and searched but couldn't find a solution.
Ps. I'm using Figlet.js
I don't know if you're ok using an external module, but you can use tiptoe.
Install it using npm install tiptoe like any regular module and it basically goes like this:
var tiptoe = require('tiptoe')
function someAsyncFunction(obj, callback) {
// something something
callback(null, processedData);
}
tiptoe(
function() {
var self = this;
var arr = ['there', 'are', 'some', 'items', 'here'];
arr.forEach(function(item) {
someAsyncFunction(item, self.parallel());
});
},
function() {
var data = Array.prototype.slice.call(arguments);
doSomethingWithData(data, this);
},
function(err) {
if (err) throw (err);
console.log('all done.');
}
);
the someAsyncFunction() is the async function you want to call does something and calls the callback parameter as a function with the parameters error and data. The data parameter will get passed as an array item to the following function on the tiptoe flow.
Did it Myself :) Thanks to mostafa-samir's post
var figlet = require('figlet');
function WaterfallOver(list, iterator, callback) {
var nextItemIndex = 1;
function report() {
nextItemIndex++;
if(nextItemIndex === list.length)
callback();
else
iterator([list[0],list[nextItemIndex]], report);
}
iterator([list[0],list[1]], report);
}
var FinalResult = [];
WaterfallOver(["hello","Standard","Ghost"], function(path, report) {
figlet.text(path[0], { font: path[1] }, function(err, data) {
if (err) {
FinalResult.push("Font name error try help");
report();
return;
}
data = '<pre>.\n' + data + '</pre>';
FinalResult.push(data);
report();
});
}, function() {
console.log(FinalResult[0]);
console.log(FinalResult[1]);
});
(First: I'm sorry, I don't speak english very well!)
I wanna return the results of 3 finds in one array.
My code (next) is running well, but I'm in callback hell!
_Schema
.static('retrieveAll', function(cb) {
query = {};
this.find(query, function(err, data) {
if(err) {
cb(err, null);
return;
}
if(data)
all = data;
else
all = [];
_StoresModel.find(query).select('contact address').exec(function(err, data) {
if(err) {
cb(err, null);
return;
}
if(data) {
all = data.reduce(function(coll, item) {
coll.push(item);
return coll;
}, all);
}
_CustomersModel.find(query).select('contact address').exec(function(err, data) {
if(err) {
cb(err, null);
return;
}
if(data) {
all = data.reduce(function(coll, item) {
coll.push(item);
return coll;
}, all);
}
cb(null, all);
});
});
});
});
I've a FIND inside a FIND inside a FIND.
Is there anyway to improve this?
SOLUTION:
_Schema
.static('retrieveAll', function(cb) {
var model = this;
_async.parallel(
{ contacts: function(cb) {
model.find({}).exec(cb);
}
, stores: function(cb) {
_StoresModel.find({}).select('contact address').exec(cb);
}
, costumers: function(cb) {
_CostumersModel.find({}).select('contact address').exec(cb);
}
}
, function(err, data) {
if(err) {
cb(err, null);
return
}
var ret = [];
if(data.contacts.length > 0) {
ret = ret.concat(data.contacts);
}
if(data.stores.length > 0) {
ret = ret.concat(data.stores);
}
if(data.costumers.length > 0) {
ret = ret.concat(data.costumers);
}
cb(null, ret);
});
You can try using Promises.
(untested) example:
var RSVP = require('rsvp');
var all = [];
_Schema.static('retrieveAll', function(cb) {
query = {};
findPromise(this, query)
.then(function (data) {
all = data;
return findPromise(_StoresModel, query, 'contact address');
})
.then(function (stores) {
all = all.concat(stores);
return findPromise(_CustomersModel, query, 'contact address');
})
.then(function (customers) {
all = all.concat(customers);
cb(null, all);
})
.catch(function (err) {
cb(err, null);
});
});
function findPromise(Model, query, select) {
return new RSVP.Promise(function (resolve, reject) {
Model.find(query).select(select || '*').exec(function (err, data) {
return err ? reject(err) : resolve(data);
});
});
}
That example is using RSVP, but there are also other promise implementations such as Q and bluebird.
And a side note, you can use concat to concatenate arrays instead of using reduce.
take a look at npm Async. It a great library of different patterns that can be used on node.js.
You will probably want to use the waterfall if there is a Chronological priority or parallel pattern if they can all execute in parallel.
Some server-side promise libraries like q and bluebird would clean up your code substantially and eliminate the mess of callback hell.
I am trying to execute an async series in nodejs with https://github.com/caolan/async#series :
async.series([
function getLightsId(callback) {
args = {
path: {
"username": "username"
}
};
client.registerMethod("getLightState", "http://bridgeip/api/${username}/lights/", "GET");
client.methods.getLightState(args, function (data, response) {
var ids = [];
for (key in data) {
ids.push(key);
}
callback(null, ids);
});
},
function getLightsState(ids, callback) {
var lightsState = new Object();
async.each(ids, function (id) {
getLightState(id, function (state) {
lightsState[id] = state;
});
});
callback(null, lightsState);
}
], function (err, result) {
console.log(result);
});
but its returning me this error :
callback(null, lightsState); TypeError : undefined is not a function.
I dont understand why..
How do i pass my object lightsState to my other function ?
You won't get ids in your second task, but you'll get a callback as is described in docs.
You could do:
async.series([
function getLightsId(callback) {
args = {
path: {
"username": "username"
}
};
client.registerMethod("getLightState", "http://bridgeip/api/${username}/lights/", "GET");
client.methods.getLightState(args, function (data, response) {
var ids = [];
for (key in data) {
ids.push(key);
}
callback(null, ids);
});
},
], function (err, result) {
var ids = result[0]
var lightsState = new Object();
async.each(ids, function (id) {
getLightState(id, function (state) {
lightsState[id] = state;
});
});
});
But this goes against the conventional use-case of async.series, you only have one async task there.
What you're after for is probably async.waterfall - so here's how that would look like:
async.waterfall([
function getLightsId(callback) {
args = {
path: {
"username": "username"
}
};
client.registerMethod("getLightState", "http://bridgeip/api/${username}/lights/", "GET");
client.methods.getLightState(args, function (data, response) {
var ids = [];
for (key in data) {
ids.push(key);
}
callback(null, ids);
});
},
function getLightsState(ids, callback) {
var lightsState = new Object();
async.each(ids, function (id) {
getLightState(id, function (state) {
lightsState[id] = state;
});
}, function(err) {
callback(err, lightsState)
});
}
], function (err, result) {
console.log(result);
});
Oh, and I thought that async.each needed a callback. I'm not sure, maybe it was you intention not to be so?
its returning me this error: callback(null, lightsState); TypeError : undefined is not a function.
Because callback is not a function. Only the first argument that is passed to your function getLightsState(ids, callback) { is one. Check the docs for series:
tasks - An array or object containing functions to run, each function is passed a
callback(err, result)
series(tasks) runs the functions in the tasks array in series, each one running once the previous function has completed.
In contrast, these are the docs for the waterfall function - which is what you want:
Runs the tasks array of functions in series, each passing their results to the next in the array.
Btw,
var lightsState = new Object();
async.each(ids, function (id) {
getLightState(id, function (state) {
lightsState[id] = state;
});
});
callback(null, lightsState);
is unlikely to work, you probably want to use reduce instead:
async.reduce(ids, {}, function(lightsState, id, rcallback) {
getLightState(id, function(state) {
lightsState[id] = state;
rcallback(null, lightsState);
});
}, callback);