Let's say i need to constantly collecting some data from a lot of clients and in parallel running some complex loop that solving some stuff with this data. How can i do it? Should i just write this in my piece of code:
app.get('/', function(req, res) {
res.sendFile(__dirname + '/public/views/index0.html');
});
io.sockets.on('connection', function(socket) {
// SOME STUFF WITH THE SOCKET
socket.on('disconnect', function(data) {
//SOME OTHER STUFF
});
});
while(...) {
//THE LOOP STUFF
}
Or i need to use the setTimeout() and setInterval() functions? How can i do the loop on the server that runs in parallel with the callbacks' stuff?
Don’t use while for make it, this block a thread. setTimeout() will run only once. You need to use setInterval() function.
You can use the async module to handle the async operation with the callback or use promise to avoid callback.
Here how I handle a complex async for each operation, that might helpfull to you get idea handeling ayncs forach
var cond = { _schedule: schedule_id }; // find curse by schedule id
Course.find(cond, function (err, courses) {
if (err) {
callback({ "success": false, "message": "Not able to update" });
} else {
async.forEachLimit(courses, 1, function (course, coursesCallback) {
async.waterfall([
function (callback) {
var schedule_date = moment(change_data.date).format('YYYY-MM-DD') + "T00:00:00.000Z"
var Assignmentcond = {
assignment_schedule_order: {
$gte: schedule_date
},
_course: course._id,
_schedule: schedule_id,
_user: userid
};
Assignment.find(Assignmentcond)
.populate({
path: '_course',
})
.lean()
.sort({ assignment_schedule_order: 1 })
.exec(function (err, AssignmentList) {
if (err) {
callback(null, '');
} else {
//console.log("------------------AssignmentList---------------------------");
//console.log(AssignmentList);
async.forEachLimit(AssignmentList, 1, function (ThisAssignmentCell, ThisAssignmentCellCallback) {
async.waterfall([
function (callback) {
var SearchObj = items;
var lebelObject = {};
for (var i = 0, flag = 0, insert = 0; i < SearchObj.length; i++) {
if (SearchObj[i].date == ThisAssignmentCell.assignment_schedule_date) {
flag = 1;
}
if (flag == 1 && SearchObj[i].label != "") {
if (ThisAssignmentCell.day == SearchObj[i].day_index) {
insert = 1;
var lebelObject = SearchObj[i];
break;
}
}
}
callback(null, ThisAssignmentCell, lebelObject, insert);
},
function (ThisAssignmentCell, SearchObj, insert, callback) {
console.log('----------------------');
console.log('ThisAssignmentCell', ThisAssignmentCell);
console.log('SearchObj', SearchObj);
console.log('----------------------');
if (insert > 0) {
var query = { _id: ThisAssignmentCell._id },
fields = {
assignment_date: moment(SearchObj.date).format('MM/DD/YYYY'),
assignment_schedule_date: moment(SearchObj.date).format('YYYY-MM-DD'),
assignment_schedule_order: new Date(SearchObj.date),
day: SearchObj.day_index,
dayNum: SearchObj.weekday_num
},
options = { upsert: false };
Assignment.update(query, fields, options, function (err, affected) {
callback(null, '');
});
} else {
// var cond = { _id: ThisAssignmentCell._id};
// Assignment.remove(cond)
// .exec(function (err, cnt) {
// callback(null, '');
// });
}
}
], function (err, result) {
// result now equals 'done'
console.log('done')
ThisAssignmentCellCallback();
});
}, function (err) {
console.log("Assignment For Loop Completed");
callback(null, AssignmentList);
});
}
});
}
], function (err, result) {
// result now equals 'done'
console.log('done')
coursesCallback();
});
}, function (err) {
console.log("courses For Loop Completed");
});
}
});
Related
Been trying everything to get this to work specifically the async each method. for (const element of resultsHistory) didn't work either.
I'm trying to modify the result array from a previous query by running a foreach over it and doing a mysql query.
However this needs to wait for the query to complete.
Is there a way to access these results without the setTimeout(function() { } I put manually in order to wait for the query to finish?
function getUserLikes(params, callback) {
var usersArrayCat = [];
console.log(`length of array ${params.length}`)
// 1 here means 1 request at a time
async.eachLimit(params, 1, function (element, cb) {
element.liked = 0;
var queryLiked = `SELECT * from users_likes WHERE user_id = \"${req.body.userid}\" AND product_id = \"${element.product_id}\"`;
connectionPromise.query(queryLiked, function (err, result) {
if (!result) {
} else if (result.length == 0) {
} else {
element.liked = result[0].userlike;
usersArrayCat.push(element);
// console.log(usersArrayCat);
cb();
}
})
}, function (err) {
if (err) return callback(err);
callback(null, usersArrayCat)
});
};
getUserLikes(resultsHistory, function (e) {
console.log(e);
});
if(!res.headersSent) {
setTimeout(function() {
res.send(JSON.stringify({"status": 200 ,"error": null, "top3":resultsHistory}));
}, 150);
}
Managed to get it working like this, but still not sure if this is the right way.
async function getUserLikes(resultsHistory) {
for (const element of resultsHistory) {
element.liked = 0;
let queryLiked = `SELECT * from users_likes WHERE user_id = \"${req.body.userid}\" AND product_id = \"${element.product_id}\"`;
let liked = await conn2.query(queryLiked);
if (liked[0]) {
element.liked = liked[0].userlike;
}
}
if(!res.headersSent) {
conn2.release();
let newres = resultsHistory.sort(
firstBy(function (v1, v2) { return v2.rating - v1.rating; })
.thenBy(function (v1, v2) { return v2.dranktimes - v1.dranktimes; })
).filter( function(history) {
return history.event === eventName;
}).slice(0, 3);
res.send({"status": 200 ,"error": null, "top3":newres});
}
}
getUserLikes(resultsHistory);
See my answer:
function getUserLikes(params, callback) {
var usersArrayCat = [];
console.log(`length of array ${params.length}`)
// 1 here means 1 request at a time
async.eachLimit(params, 1, function (element, cb) {
element.liked = 0;
var queryLiked = `SELECT * from users_likes WHERE user_id = \"${req.body.userid}\" AND product_id = \"${element.product_id}\"`;
connectionPromise.query(queryLiked, function (err, result) {
if (!result) {
} else if (result.length == 0) {
} else {
element.liked = result[0].userlike;
usersArrayCat.push(element);
// console.log(usersArrayCat);
}
cb();
})
}, function (err) {
if (err) return callback(err);
callback(null, usersArrayCat)
});
};
getUserLikes(resultsHistory, function (e, usersArrayCat) {
if (e) {
console.log(e);
return res.send({status: 400, error: e}); // your error response
}
console.log(usersArrayCat); // your usersArrayCat with liked property
res.send(JSON.stringify({ // why your need return a string intead of json object ???
"status": 200,
"error": null,
"top3": usersArrayCat // I think return `usersArrayCat` is a right way
}));
});
I have an async waterfall Array where the function otherIngrLists() is the 3rd to be executed. Every function before that worked fine.
function otherIngrLists(userslist, callback){
collection = db.get('ingrList');
collection.find({"userid":{$ne:userid}},{},function(err,docs){
if(!err){
var otherLists = docs;
var otherListsCount = docs.count();
console.log(otherListsCount);
callback(null, otherLists, otherListsCount, userslist);
} else {
callback(err, null);
}
});
},
The Problem is that this function is called twice. I assured this with a simple console.log().
How did I manage to call this function again? Did I get the concept of callbacks wrong as I use them to be passed on to the next function?
Also after this function executing twice an error ist thrown. It has nothing to to with this problem though and I will concern my self with that later.
Thank you for your time!
Waterfall Array in router.get:
router.get('/:userid', function(req, res) {
var db = req.db;
var collection;
var userid = req.params.userid;
async.waterfall(
[
function getIngrList(callback, userid) {
var route = 'http://localhost:3000/users/zutatenliste/'+userid;
request(route, function(err, response, body){
if (!err && response.statusCode === 200) {
var userlist = body;
callback(null, userlist);
} else {
callback(err, null);
return;
}
});
},
function otherIngrLists(userlist, callback){
collection = db.get('zutatenListe');
console.log(userid);
collection.find({"userid":{$ne:userid}},{},function(err,docs){
if(!err){
var otherLists = docs;
var otherListsCount = docs.count();
callback(null, otherLists, otherListsCount, userlist);
} else {
callback(err, null);
}
});
},
function pushInArray(otherLists, otherListsCount, userlist, callback){
console.log("test");
...
...}
}
}
Edit 1: --Also either if cases are executed, first the true one then the false--
// Does not happen anymore
Edit 2: Added the whole Thing until the problematic function
Please provide some Additional details as this function seems perfect and No, You haven't misunderstood the concept of callback you are using it correctly.
Structure of Async Waterfall
var create = function (req, res) {
async.waterfall([
_function1(req),
_function2,
_function3
], function (error, success) {
if (error) { alert('Something is wrong!'); }
return alert('Done!');
});
};
function _function1 (req) {
return function (callback) {
var something = req.body;
callback (null, something);
}
}
function _function2 (something, callback) {
return function (callback) {
var somethingelse = function () { // do something here };
callback (err, somethingelse);
}
}
function _function3 (something, callback) {
return function (callback) {
var somethingmore = function () { // do something here };
callback (err, somethingmore);
}
}
so, in waterfall you can pass the values to the next function and your 3rd function is correct.
Edited
async.waterfall(
[
//can not give userId as second parameter
function getIngrList(callback) {
//if you want userId you can pass as I shown above or directly use here if it's accessible
var route = 'http://localhost:3000/users/zutatenliste/'+userid;
request(route, function(err, response, body){
if (!err && response.statusCode === 200) {
var userlist = body;
callback(null, userlist);
} else {
callback(err, null);
// return; no need
}
});
},
function otherIngrLists(userlist, callback){
collection = db.get('zutatenListe');
console.log(userid);
collection.find({"userid":{$ne:userid}},{},function(err,docs){
if(!err){
var otherLists = docs;
var otherListsCount = docs.count();
callback(null, otherLists, otherListsCount, userlist);
} else {
callback(err, null);
}
});
},
function pushInArray(otherLists, otherListsCount, userlist, callback){
console.log("test");
...
...}
As said you can not pass userId as last parameter over there. Let me know if you still get the same error.
First you need to declare you function:
function myFuntion(userId, callback) {
async.waterfall([
function(callback) {
//do some thing here
callback(null, userlist);
}, function(userId, callback) {
//do something here
callback(null, orderList, orderListCount, userlist);
}
], function(err, orderList, orderListCount, userlist) {
if(err)
console.log(err);
else
callback(orderList, orderList, userlist);
})
}
After that you can use function:
myFuntion(userId, function(orderList, orderListCount, userlist) {
console.log(orderList);
console.log(orderListCount);
console.log(userlist);
})
I'm using NodeJS and using the
tls.connect(port, host, options, callback)
To get my socket to write to. I pass it through a async.waterfall and the socket.writable property stays set to true and I can write to the socket throughout. However, if I try to use it with setTimeout, it ends up being closed. Is there a way to keep it open or is my syntax incorrect?
This is one of the calls in async waterfall:
function (sock, err, callback) {
console.log(sock.writable); // this is true
setTimeout(function(sock) {
console.log(sock.writable); // this is false but I'd like it to be true so I can use it for more logic
}, 3000, sock);
}
I've also tried
function (sock, err, callback) {
console.log(sock.writable); // this is true
var sockz = sock;
setTimeout(function() {
console.log(sockz.writable); // this is false but I'd like it to be true so I can use it for more logic
}, 3000);
}
and this
function (sock, err, callback) {
console.log(sock.writable); // this is true
setTimeout(function(sock) {
console.log(sock.writable); // this is false but I'd like it to be true so I can use it for more logic
}.bind(this, sock), 3000);
// callback(null, sock);
}], function (err, sock) {
console.log(sock.writable);
}
But all of them print false. Is my syntax incorrect or does the socket automatically close in this scenario where I'm trying to get it to wait?
Edit
Full async.waterfall:
async.waterfall([
// insert record into db
function (acb) {
dbConn.query()
.insert(tableName, ["HOST", "SERVER_NAME", "DATE_CREATED"], [hostname, "test", currentTime])
.execute(function (err, result) {
if (err) {
componentStatus.database = false;
// pass the error down to run all the functions
acb(null, sockz, componentStatus, err);
} else {
componentStatus.resultId = result.id;
acb(null, sockz, componentStatus, null);
}
});
},
// read record back
function (sockz, componentStatus, err, acb) {
if (componentStatus.resultId < 0) {
acb(null, sockz, componentStatus, err);
} else {
dbConn.query().select("ID").from(tableName).where("ID=?", [componentStatus.resultId]).execute(function (err, rows, cols) {
if (err) {
componentStatus.database = false;
acb(null, sockz, componentStatus, err);
} else {
acb(null, sockz, componentStatus, err);
}
});
}
},
// send message to rabbit
function (sockz, componentStatus, err, acb) {
sendStatusMessage(currentTime, componentStatus.resultId, function (err1, result) {
if (err1) {
componentStatus.messaging = false;
var error = err + ", " + err1;
acb(null, sockz, componentStatus, error);
} else {
console.log("rabbit timeout");
console.log(sockz.writable);
setTimeout(function(sockz) {
console.log("In timeout");
console.log(sockz.writable);
}.bind(this, sockz), 3000); // wait for 3s to check record was consumed
}
});
}
], function (err, sockz, componentStatus, actualError) {
console.log(sockz.writable);
});
}
});
};
exports.sendStatusMessage = function(dateCreated, id, callback) {
var notification = { "dateCreated": dateCreated, "id": id };
_sendMsg("test.queue", notification, {"contentType": "application/json"}, function (err, result) {
if (err) {
callback(err, null);
} else {
callback(null, result);
}
});
}
function(dataValue, cb) {
req.app.db.models.User.find({
_id: { $ne: dataValue._id }
}, function(err, totalUser) {
if (!err) {
var len = totalUser.length;
if (len !== 0) {
req.app.utility.async.map(totalUser, function(each, callback) {
console.log(each);
req.app.utility.async.mapSeries(each.nonregisterContact, function(element, callback1) {
console.log('element', element.number);
console.log('dataValue', dataValue.mobileNumber);
console.log('kolka', Number(element.number) === Number(dataValue.mobileNumber));
if (Number(element.number) === Number(dataValue.mobileNumber)) {
each.registerContact.push(dataValue._id.toString());
each.nonregisterContact.splice(element, 1);
each.save(function(err, finalResult) {
if (!err) {
} else {
console.log(err);
}
})
callback1(null, null);
} else {
callback1(null, null);
}
}, function(err, final) {
if (!err) {
callback(null, null);
} else {
console.log(err);
}
});
}, function(err, result) {
if (!err) {
console.log('2');
return cb(null, dataValue);
} else {
console.log(err);
}
});
} else {
return cb(null, dataValue);
}
} else {
cb(err);
}
})
}
I don't get any response after each.save method call in the mapSeries method final callback.I am trying this solution.How i will do the same thing. How I resolve that and handle this kind of situation?
I tried to simplify code, but I'm not sure that my code realizes your needs. Also I cann't test it :D
dataValue, each, element, finalResult are very common names, so you should use them with caution to keep code is readable/supportable.
// very bad idea is include other libraries to app
var async = require('async');
var db = require('db'); // this module must export connection to db
...
function (dataValue, cb) {
// processUser use data from closure of current function => inside of current
function processUser (user, callback) {
async.mapSeries(user.nonregisterContact, function(contact, callback){
// Check and exit if condition is not satisfied. It's more readable.
if (Number(contact.number) !== Number(dataValue.mobileNumber))
return callback(null); // ignore user
user.registerContact.push(dataValue._id.toString());
user.nonregisterContact.splice(contact, 1);
user.save(function(err, finalResult) { // Is finalResult ignore?
if (err)
console.log(err);
callback(); // ingnore error
})
}, callback);
db.models.User.find({_id: { $ne: dataValue._id }}, function(err, userList) {
if (!err)
return cb(err);
if (userList.length == 0)
return cb(new Error('Users not found'));
// use named function to avoid stairs of {}
async.map(userList, processUser, cb);
})
};
I'm building a site with node/express/mongoose and it needs to do the following things when viewing a submission.
The problem I'm running into is doing db fetches in a non-serial fashion. For example, I'll do a few calls to fetch some data, but some of the calls might not finish until the execution context goes to the other. Tried to use the npm module, async, but am having trouble trying to figure out how I would integrate it.
Here is my code:
var getViewCount = function(submissionId) {
Submission.getSubmissionViewCount({
submissionId : submissionId
}, function(err, count) {
if (err) {
throw err;
}
if (count) {
return count;
}
});
};
var getVotes = function(submissionId) {
console.log('getvotes');
Submission.getSubmissionVotes({
submissionId : submissionId
}, function(err, votes) {
return votes;
});
};
var getSubmission = function(id) {
Submission.getSubmission({
id : id
}, function(err, submission) {
if (err) {
throw err;
}
if (submission) {
return submission;
}
});
};
var renderSubmission = function(title, submission, views) {
res.render('submission', {
title: submission.title + ' -',
submission: submission,
views: views.length
});
};
How do I use this with async? Or should I be using async.series isntead of async.async?
async.series([
function(callback) {
var submission = getSubmission(id);
callback(null, submission);
},
function(callback) {
// getViewCount(submissionId);
},
function(callback) {
// getVotes(submissionId);
},
function(callback) {
//renderSubmission(title, submission, views);
}
], function(err, results) {
console.log(results);
});
Basically I want to fetch the views and votes first and then render my submission.
TheBrain's description of the overall structural changes that you should make to your code is accurate. The basic methodology in Node is to nest a series of callbacks; very rarely should you require functions that actually return a value. Instead, you define a function that takes a callback as parameter and pass the result into that callback. Please review the code below for clarification (where cb is a callback function):
var getViewCount = function(submissionId, cb) {
Submission.getSubmissionViewCount({
submissionId : submissionId
}, function(err, count) {
if (err) {
throw err;
}
if (cb) {
cb(count);
}
});
};
var getVotes = function(submissionId, cb) {
console.log('getvotes');
Submission.getSubmissionVotes({
submissionId : submissionId
}, function(err, votes) {
if (cb) {
cb(votes);
}
});
};
var getSubmission = function(id, cb) {
Submission.getSubmission({
id : id
}, function(err, submission) {
if (err) {
throw err;
}
if (cb) {
cb(submission);
}
});
};
var renderSubmission = function(submissionId) {
getSubmission(submissionId, function (submission) {
if (!submission) {
// unable to find submission
// add proper error handling here
} else {
getViewCount(submissionId, function (viewCount) {
res.render('submission', {
title: submission.title + ' -',
submission: submission,
views: viewCount
});
});
}
};
};