We Keep Coding

Run loop on the server with node.js

Let's say i need to constantly collecting some data from a lot of clients and in parallel running some complex loop that solving some stuff with this data. How can i do it? Should i just write this in my piece of code:
app.get('/', function(req, res) {
res.sendFile(__dirname + '/public/views/index0.html');
});
io.sockets.on('connection', function(socket) {
// SOME STUFF WITH THE SOCKET
socket.on('disconnect', function(data) {
//SOME OTHER STUFF
});
});
while(...) {
//THE LOOP STUFF
}
Or i need to use the setTimeout() and setInterval() functions? How can i do the loop on the server that runs in parallel with the callbacks' stuff?

Don’t use while for make it, this block a thread. setTimeout() will run only once. You need to use setInterval() function.

You can use the async module to handle the async operation with the callback or use promise to avoid callback.
Here how I handle a complex async for each operation, that might helpfull to you get idea handeling ayncs forach
var cond = { _schedule: schedule_id }; // find curse by schedule id
Course.find(cond, function (err, courses) {
if (err) {
callback({ "success": false, "message": "Not able to update" });
} else {
async.forEachLimit(courses, 1, function (course, coursesCallback) {
async.waterfall([
function (callback) {
var schedule_date = moment(change_data.date).format('YYYY-MM-DD') + "T00:00:00.000Z"
var Assignmentcond = {
assignment_schedule_order: {
$gte: schedule_date
},
_course: course._id,
_schedule: schedule_id,
_user: userid
};
Assignment.find(Assignmentcond)
.populate({
path: '_course',
})
.lean()
.sort({ assignment_schedule_order: 1 })
.exec(function (err, AssignmentList) {
if (err) {
callback(null, '');
} else {
//console.log("------------------AssignmentList---------------------------");
//console.log(AssignmentList);
async.forEachLimit(AssignmentList, 1, function (ThisAssignmentCell, ThisAssignmentCellCallback) {
async.waterfall([
function (callback) {
var SearchObj = items;
var lebelObject = {};
for (var i = 0, flag = 0, insert = 0; i < SearchObj.length; i++) {
if (SearchObj[i].date == ThisAssignmentCell.assignment_schedule_date) {
flag = 1;
}
if (flag == 1 && SearchObj[i].label != "") {
if (ThisAssignmentCell.day == SearchObj[i].day_index) {
insert = 1;
var lebelObject = SearchObj[i];
break;
}
}
}
callback(null, ThisAssignmentCell, lebelObject, insert);
},
function (ThisAssignmentCell, SearchObj, insert, callback) {
console.log('----------------------');
console.log('ThisAssignmentCell', ThisAssignmentCell);
console.log('SearchObj', SearchObj);
console.log('----------------------');
if (insert > 0) {
var query = { _id: ThisAssignmentCell._id },
fields = {
assignment_date: moment(SearchObj.date).format('MM/DD/YYYY'),
assignment_schedule_date: moment(SearchObj.date).format('YYYY-MM-DD'),
assignment_schedule_order: new Date(SearchObj.date),
day: SearchObj.day_index,
dayNum: SearchObj.weekday_num
},
options = { upsert: false };
Assignment.update(query, fields, options, function (err, affected) {
callback(null, '');
});
} else {
// var cond = { _id: ThisAssignmentCell._id};
// Assignment.remove(cond)
// .exec(function (err, cnt) {
// callback(null, '');
// });
}
}
], function (err, result) {
// result now equals 'done'
console.log('done')
ThisAssignmentCellCallback();
});
}, function (err) {
console.log("Assignment For Loop Completed");
callback(null, AssignmentList);
});
}
});
}
], function (err, result) {
// result now equals 'done'
console.log('done')
coursesCallback();
});
}, function (err) {
console.log("courses For Loop Completed");
});
}
});

Async.waterfall function gets called twice

I have an async waterfall Array where the function otherIngrLists() is the 3rd to be executed. Every function before that worked fine.
function otherIngrLists(userslist, callback){
collection = db.get('ingrList');
collection.find({"userid":{$ne:userid}},{},function(err,docs){
if(!err){
var otherLists = docs;
var otherListsCount = docs.count();
console.log(otherListsCount);
callback(null, otherLists, otherListsCount, userslist);
} else {
callback(err, null);
}
});
},
The Problem is that this function is called twice. I assured this with a simple console.log().
How did I manage to call this function again? Did I get the concept of callbacks wrong as I use them to be passed on to the next function?
Also after this function executing twice an error ist thrown. It has nothing to to with this problem though and I will concern my self with that later.
Thank you for your time!
Waterfall Array in router.get:
router.get('/:userid', function(req, res) {
var db = req.db;
var collection;
var userid = req.params.userid;
async.waterfall(
[
function getIngrList(callback, userid) {
var route = 'http://localhost:3000/users/zutatenliste/'+userid;
request(route, function(err, response, body){
if (!err && response.statusCode === 200) {
var userlist = body;
callback(null, userlist);
} else {
callback(err, null);
return;
}
});
},
function otherIngrLists(userlist, callback){
collection = db.get('zutatenListe');
console.log(userid);
collection.find({"userid":{$ne:userid}},{},function(err,docs){
if(!err){
var otherLists = docs;
var otherListsCount = docs.count();
callback(null, otherLists, otherListsCount, userlist);
} else {
callback(err, null);
}
});
},
function pushInArray(otherLists, otherListsCount, userlist, callback){
console.log("test");
...
...}
}
}
Edit 1: --Also either if cases are executed, first the true one then the false--
// Does not happen anymore
Edit 2: Added the whole Thing until the problematic function

Please provide some Additional details as this function seems perfect and No, You haven't misunderstood the concept of callback you are using it correctly.
Structure of Async Waterfall
var create = function (req, res) {
async.waterfall([
_function1(req),
_function2,
_function3
], function (error, success) {
if (error) { alert('Something is wrong!'); }
return alert('Done!');
});
};
function _function1 (req) {
return function (callback) {
var something = req.body;
callback (null, something);
}
}
function _function2 (something, callback) {
return function (callback) {
var somethingelse = function () { // do something here };
callback (err, somethingelse);
}
}
function _function3 (something, callback) {
return function (callback) {
var somethingmore = function () { // do something here };
callback (err, somethingmore);
}
}
so, in waterfall you can pass the values to the next function and your 3rd function is correct.
Edited
async.waterfall(
[
//can not give userId as second parameter
function getIngrList(callback) {
//if you want userId you can pass as I shown above or directly use here if it's accessible
var route = 'http://localhost:3000/users/zutatenliste/'+userid;
request(route, function(err, response, body){
if (!err && response.statusCode === 200) {
var userlist = body;
callback(null, userlist);
} else {
callback(err, null);
// return; no need
}
});
},
function otherIngrLists(userlist, callback){
collection = db.get('zutatenListe');
console.log(userid);
collection.find({"userid":{$ne:userid}},{},function(err,docs){
if(!err){
var otherLists = docs;
var otherListsCount = docs.count();
callback(null, otherLists, otherListsCount, userlist);
} else {
callback(err, null);
}
});
},
function pushInArray(otherLists, otherListsCount, userlist, callback){
console.log("test");
...
...}
As said you can not pass userId as last parameter over there. Let me know if you still get the same error.

First you need to declare you function:
function myFuntion(userId, callback) {
async.waterfall([
function(callback) {
//do some thing here
callback(null, userlist);
}, function(userId, callback) {
//do something here
callback(null, orderList, orderListCount, userlist);
}
], function(err, orderList, orderListCount, userlist) {
if(err)
console.log(err);
else
callback(orderList, orderList, userlist);
})
}
After that you can use function:
myFuntion(userId, function(orderList, orderListCount, userlist) {
console.log(orderList);
console.log(orderListCount);
console.log(userlist);
})

How to force a function to execute first in Jasmine test?

I have a route in Node that gets an auth key. I want to use this auth key in all my jasmine tests as a parameter in the URL request. I want SetUp function to run, set a global var, then allow me to use this variable in all the rest of the test cases.
SetUp Function
var global_key = request({
uri : 'http://localhost:3000/grabToken',
method : 'GET'
},
function (err, body, res) {
if (err) { console.log(err);}
else {
return body['auth_key'];
}
});
Test Suite
function testCases() {
describe(TEST_SUITE, function() {
describe("GET /retrieveSecret/VALID_UUID", function() {
it('Requests the secret - Successful response', function(done) {
// ...
}
}
}
}

You could use asynchronous version of beforeAll function:
describe(TEST_SUITE, function() {
let key;
beforeAll(function (done) {
const params = { uri: 'http://localhost:3000/grabToken', method: 'GET' };
request(params, function (err, body, res) {
if (err) {
console.log(err);
done.fail();
} else {
key = body['auth_key'];
done();
}
});
})
describe("GET /retrieveSecret/VALID_UUID", function() {
it('Requests the secret - Successful response', function(done) {
// `key` is available here
}
});
})

How to use async with other asynchronous modules?

Here is a working test program without async:
var fs = require('fs');
function test() {
var finalResponse = '', response = '';
function showFinalResponse() {
console.log(finalResponse);
}
function processTheFile(err, data) {
if (err) {
finalResponse = 'Could not read the file';
} else {
response += data;
response += '</body></html>';
finalResponse = response;
}
showFinalResponse();
}
function readTheFile(exists) {
if (!exists) {
finalResponse = 'File does not exist.';
showFinalResponse();
} else {
response += '<!DOCTYPE html><html lang="en-US"><head></head><body>';
fs.readFile('file.txt', 'utf8', processTheFile);
}
}
fs.exists('file.txt', readTheFile);
};
test();
Here is my attempt at getting the same program to work with async waterfall. I'm having trouble with how to pass the callbacks around in the async and the fs calls.
var fs = require('fs');
var async = require('async');
function testAsync() {var finalResponse, response = '';
async.waterfall( [
function checkIfTheFileExists(done) {
fs.exists('file.txt', done);
},
function readTheFile(err, exists, done) {
response += '<!DOCTYPE html><html lang="en-US"><head></head><body>';
fs.readFile('file.txt', 'utf8', done);
},
function processTheFile(err, data, done) {
response += data;
response += '</body></html>';
finalResponse = response;
done(null);
} ],
function showFinalResponse(err) {
if (err) {
if (err.code === 'ENOENT') { // intended to test for file is missing.
finalResponse = 'File does not exist.';
} else { // any other errors.
finalResponse = 'Could not read the file';
}
console.log(err);
}
console.log(finalResponse);
}
);
}
testAsync()
I can't get the async version to work. I'm getting confused with where the callbacks go.

fs.exists is an oddball in that it doesn't provide an error parameter to its callback function. Instead it only provides a single exists parameter that indicates whether the file was found or not. Presumably, if there was an error, exists would be false. As such you need to wrap its callback in your own function so that you can provide a separate error parameter to the waterfall callback:
async.waterfall( [
function checkIfFileExists(done) {
fs.exists('file.txt', function(exists) { done(null, exists); });
},
function makeSureFileExists(exists, done) {
...
Note the warning in the docs, however, that fs.exists shouldn't be used, typically.

fs.exists('file.txt', done(null));
This calls done immediately. You need to pass the actual done function to fs.exists:
fs.exists('file.txt', done);
Same for the others.

Here is my final working version (in case it helps anyone else). Thanks again for your help!
var fs = require('fs');
var async = require('async');
var addErrParm = function (err, done) {return function(exists) {
done(err, exists);
}}
function testAsync() {var finalResponse, response = '';
function checkIfTheFileExists(done) {
fs.exists('file.txt', addErrParm(null, done));
}
function readTheFile(exists, done) {
if (!exists) {
done('notFound');
} else {
response += '<!DOCTYPE html><html lang="en-US"><head></head><body>';
fs.readFile('file.txt', 'utf8', done);
}
}
function processTheFile(data, done) {
response += (data || 'The file is empty') + '</body></html>';
finalResponse = response;
done(null);
}
function showFinalResponse(err) {
if (err) {
finalResponse = (err === 'notFound' ? 'File does not exist.' : 'Could not read the file');
}
console.log(finalResponse);
}
async.waterfall([ checkIfTheFileExists,
readTheFile,
processTheFile
], showFinalResponse);
}
testAsync()
So basically, async requires removing the err parameter (first argument) from all functions except the final callback, and it requires adding a callback ('done') as an extra parameter on all functions except the final callback.
Also, if there is no err parameter like with fs.exists, you have to create a function to simulate an err parameter so async can remove it.

Async.js waterfall (functions defined outside of the array)

I am using async.js for the first time and would like to seperate out the function array into seperate functions. I have:
working code
async.waterfall([
function(callback)
{
var querySuccess = function (tx, result)
{
callback(null, result.rows.item(0).EventImportTime || "");
};
var queryError = function (tx, e)
{
callback("Query Error")
};
database.open();
database.query("SELECT EventImportTime FROM Contact WHERE Contact.Id = ?", [contactId], querySuccess, queryError);
},
function(lastImportTime, callback)
{
var url = "";
url += 'MobileGetvents.aspx?';
url += '&LastImportTime=';
url += lastImportTime;
url += '&Format=JSON';
callback(null, url)
},
],
function(err, result)
{
if (err)
console.log("Error Happened");
else
{
console.log(result);
getJSON(result, callback,
errorCallback)
}
})
I want
async.waterfall([
getLastImportTime(callback) ,
buildUrl(lastImportTime, callback),
],
//callback
);
However when I run this code it always returns
Uncaught ReferenceError: lastImportTime is not defined

I would assume that you want this instead:
var getLastImportTime = function(callback) { };
var buildUrl = function(lastImportTime, callback) { };
async.waterfall([
getLastImportTime,
buildUrl,
],
//callback
);