We Keep Coding

Run loop on the server with node.js

Let's say i need to constantly collecting some data from a lot of clients and in parallel running some complex loop that solving some stuff with this data. How can i do it? Should i just write this in my piece of code:
app.get('/', function(req, res) {
res.sendFile(__dirname + '/public/views/index0.html');
});
io.sockets.on('connection', function(socket) {
// SOME STUFF WITH THE SOCKET
socket.on('disconnect', function(data) {
//SOME OTHER STUFF
});
});
while(...) {
//THE LOOP STUFF
}
Or i need to use the setTimeout() and setInterval() functions? How can i do the loop on the server that runs in parallel with the callbacks' stuff?

Don’t use while for make it, this block a thread. setTimeout() will run only once. You need to use setInterval() function.

You can use the async module to handle the async operation with the callback or use promise to avoid callback.
Here how I handle a complex async for each operation, that might helpfull to you get idea handeling ayncs forach
var cond = { _schedule: schedule_id }; // find curse by schedule id
Course.find(cond, function (err, courses) {
if (err) {
callback({ "success": false, "message": "Not able to update" });
} else {
async.forEachLimit(courses, 1, function (course, coursesCallback) {
async.waterfall([
function (callback) {
var schedule_date = moment(change_data.date).format('YYYY-MM-DD') + "T00:00:00.000Z"
var Assignmentcond = {
assignment_schedule_order: {
$gte: schedule_date
},
_course: course._id,
_schedule: schedule_id,
_user: userid
};
Assignment.find(Assignmentcond)
.populate({
path: '_course',
})
.lean()
.sort({ assignment_schedule_order: 1 })
.exec(function (err, AssignmentList) {
if (err) {
callback(null, '');
} else {
//console.log("------------------AssignmentList---------------------------");
//console.log(AssignmentList);
async.forEachLimit(AssignmentList, 1, function (ThisAssignmentCell, ThisAssignmentCellCallback) {
async.waterfall([
function (callback) {
var SearchObj = items;
var lebelObject = {};
for (var i = 0, flag = 0, insert = 0; i < SearchObj.length; i++) {
if (SearchObj[i].date == ThisAssignmentCell.assignment_schedule_date) {
flag = 1;
}
if (flag == 1 && SearchObj[i].label != "") {
if (ThisAssignmentCell.day == SearchObj[i].day_index) {
insert = 1;
var lebelObject = SearchObj[i];
break;
}
}
}
callback(null, ThisAssignmentCell, lebelObject, insert);
},
function (ThisAssignmentCell, SearchObj, insert, callback) {
console.log('----------------------');
console.log('ThisAssignmentCell', ThisAssignmentCell);
console.log('SearchObj', SearchObj);
console.log('----------------------');
if (insert > 0) {
var query = { _id: ThisAssignmentCell._id },
fields = {
assignment_date: moment(SearchObj.date).format('MM/DD/YYYY'),
assignment_schedule_date: moment(SearchObj.date).format('YYYY-MM-DD'),
assignment_schedule_order: new Date(SearchObj.date),
day: SearchObj.day_index,
dayNum: SearchObj.weekday_num
},
options = { upsert: false };
Assignment.update(query, fields, options, function (err, affected) {
callback(null, '');
});
} else {
// var cond = { _id: ThisAssignmentCell._id};
// Assignment.remove(cond)
// .exec(function (err, cnt) {
// callback(null, '');
// });
}
}
], function (err, result) {
// result now equals 'done'
console.log('done')
ThisAssignmentCellCallback();
});
}, function (err) {
console.log("Assignment For Loop Completed");
callback(null, AssignmentList);
});
}
});
}
], function (err, result) {
// result now equals 'done'
console.log('done')
coursesCallback();
});
}, function (err) {
console.log("courses For Loop Completed");
});
}
});

Wait for function to end (in async.series) doesn't work as expected

I have a nodejs script that gets some json data containing item names and prices using an API, then checks the price of that item. The problem I have is that the function that checks the item name gets executed before the function that writes the price list ends. So if the price list file doesn't exist, it gives me 'no such file or directory' for the price list file. I searched for some time on the internet, and I found something about async.series. I tried something, but it doesn't seem to work, since the result is the same. If the price list file exists, the message 'Prices updated successfully! Lenght:' appears after printing the item price.
function getPrices() {
console.log("URL requested: ", API_URL)
restling.get(API_URL).then(function(result) {
var JSON_Data = result.data
if (JSON_Data.status == "success") {
console.log("Prices updated successfully! Lenght: "+JSON_Data.prices.length)
} else {
console.log(JSON_Data)
console.log("An error ocurred during updating prices!")
return
}
fs.writeFileSync("prices/pricelist.txt", JSON.stringify(JSON_Data.prices))
})
}
function getItemPrice(item) {
var file = JSON.parse(fs.readFileSync("prices/pricelist.txt"))
for (var i = 0; i < file.length; i++) {
if (file[i].item_name == item) {
return file[i].price
}
}
}
function getItem() {
var item1 = getItemPrice('Sword');
console.log(item1);
}
async.series([
function(callback){
getPrices();
callback();
},
function(callback){
getItem();
callback();
}
]);
EDIT : I've tried something else, but the problem remains the same
async.series([
function(callback){
getPrices();
callback();
},
function(callback){
getItem();
callback();
}
], function(error){
if (error) {
console.log(error);
}
});
async.waterfall([
function(callback){
getPrices();
callback();
},
function(arg1, callback){
getItem();
callback();
},
], function (error) {
if (error) {
console.log(error);
}
});

Mmm, i think not makes so much sense write the response in a file for read later, since you can use directly the response to find the item, but this should works.
function getPrices(callback) {
restling.get(API_URL).then(function(result) {
var JSON_Data = result.data
if (JSON_Data.status == "success") {
console.log("Prices updated successfully! Lenght: " + JSON_Data.prices.length)
} else {
console.log(JSON_Data)
console.log("An error ocurred during updating prices!")
return
}
fs.writeFileSync("prices/pricelist.txt", JSON.stringify(JSON_Data.prices))
callback(null, null);
});
}
function getItemPrice(item) {
var file = JSON.parse(fs.readFileSync("prices/pricelist.txt"))
for (var i = 0; i < file.length; i++) {
if (file[i].item_name == item) {
return file[i].price
}
}
}
function getItem(callback) {
var item1 = getItemPrice('Sword');
callback(null, item1);
}
async.series([
getPrices,
getItem
],
function(err, result) {
if (err) {
console.log(err);
return;
}
console.log("result", result);
});
I think a promises based approach and manipulating the response directly instead of writing / reading form a file , will be easier to understand.
function getPrices(url) {
return new Promise( function(resolve) {
reslint.get(url).then( function(result) {
var data = result.data;
if(data.status === "success") {
console.log(
"Prices updated successfully! Lenght: ",
data.prices.length
);
return resolve(data.prices);
}
else {
throw new Error("An error ocurred during updating prices!");
}
})
.catch(function(ex) {
throw ex;
});
});
}
function getItemPrice(item, items) {
for (var i = 0; i < items.length; i++) {
if (items[i].item_name == item) {
return items[i].price
}
}
}
getPrices("some/url/which/return/prices")
.then(function(prices) {
console.log(getItemPrice("Sword", prices));
})
.catch(function(err) {
console.log("some error -->", err);
});

If I am not mistaken doesn't series expect a final callback.
async.series([
function(callback){
getPrices();
callback();
},
function(callback){
getItem();
callback();
}
],
function(error, [callback array]){
});
Where the callback array will contain the result returned from the functions array. Maybe try async.waterfall?

Make multiple callbacks from node js asynchronous function

How can I return a object of data returned by asynchronous function called multiple times from within a asynchronous function.
I'm trying to implement like this :
var figlet = require('figlet');
function art(dataToArt, callback)
{
var arry[];
figlet(dataToArt, function(err, data) {
if (err) {
console.log('Something went wrong...');
console.dir(err);
return callback('');
}
arry[0] = data;
callback(arry);
});
figlet(dataToArt, function(err, data) {
if (err) {
console.log('Something went wrong...');
console.dir(err);
return callback('');
}
arry[1] = data;
callback(arry);
});
}
art('Hello World', function (data){
console.log(data);
});
How can I do it correctly, I searched and searched but couldn't find a solution.
Ps. I'm using Figlet.js

I don't know if you're ok using an external module, but you can use tiptoe.
Install it using npm install tiptoe like any regular module and it basically goes like this:
var tiptoe = require('tiptoe')
function someAsyncFunction(obj, callback) {
// something something
callback(null, processedData);
}
tiptoe(
function() {
var self = this;
var arr = ['there', 'are', 'some', 'items', 'here'];
arr.forEach(function(item) {
someAsyncFunction(item, self.parallel());
});
},
function() {
var data = Array.prototype.slice.call(arguments);
doSomethingWithData(data, this);
},
function(err) {
if (err) throw (err);
console.log('all done.');
}
);
the someAsyncFunction() is the async function you want to call does something and calls the callback parameter as a function with the parameters error and data. The data parameter will get passed as an array item to the following function on the tiptoe flow.

Did it Myself :) Thanks to mostafa-samir's post
var figlet = require('figlet');
function WaterfallOver(list, iterator, callback) {
var nextItemIndex = 1;
function report() {
nextItemIndex++;
if(nextItemIndex === list.length)
callback();
else
iterator([list[0],list[nextItemIndex]], report);
}
iterator([list[0],list[1]], report);
}
var FinalResult = [];
WaterfallOver(["hello","Standard","Ghost"], function(path, report) {
figlet.text(path[0], { font: path[1] }, function(err, data) {
if (err) {
FinalResult.push("Font name error try help");
report();
return;
}
data = '<pre>.\n' + data + '</pre>';
FinalResult.push(data);
report();
});
}, function() {
console.log(FinalResult[0]);
console.log(FinalResult[1]);
});

Getting data from rethinkdb database, manipulating said data, then updating the database with the manipulated docs

I am looking to do a get, run a function on the results which will do some manipulation by updating a field, and then put that doc back into the database. Really my issue is being able to chain together multiple DB calls. I have been struggling with this the past week or so. Any suggestions appreciated, thanks.
Here is what I have tried so far but I am receiving an error:
function geocode_cleanup(request, response, next) {
r.table('dealer_locations').filter(r.row('geodata').match('No geodata found.'))
.do(function(row) {
var geodata = opencage_geocoder.geocode(row.Address, function(error, response) {
if (error) {
console.log("Error.");
row.geodata = "No geodata found.";
row.active = true;
} else if (response.length == 0) {
console.log("Empty response.");
} else {
console.log("Success.");
console.log(response);
var latitude = response[0].latitude;
var longitude = response[0].longitude;
row.geodata = r.point(longitude, latitude);
row.active = true;
}
});
return r.table('dealer_locations').update({
geodata: geodata
})
}).run(conn, function(error, cursor) {
response.setHeader("Content-Type", "application/json");
if (error) {
handleError(response, error);
} else {
cursor.toArray(function(error, results) {
if (error) {
handleError(response, error);
} else {
response.send(results);
};
});
}
next();
})
};
Also, this gives the desired results returned in the response, but the second db action never happens because I am still inside of the same db connection I think:
function geocode_cleanup(request, response, next) {
var conn = request._rdbConn;
r.table('dealer_locations').filter({geodata: "No geodata found."}).run(conn, function(error, cursor) {
if (error) {
handleError(response, error);
} else {
cursor.toArray(function(error, results) {
if (error) {
handleError(response, error);
} else {
var i = 1;
async.forEach(results, function(item, callback) {
var address = (item.Address + " " + item.City).toString();
opencage_geocoder.geocode(address, function(err, res) {
if (err) {
console.log(i);
console.log("Error.");
item.id = i;
item.geodata = "No geodata found.";
item.active = true;
i++;
callback();
} else if (res.length == 0) {
console.log(i);
console.log("Empty response.");
i++;
callback();
} else {
console.log(i);
console.log("Success.");
console.log(res);
var latitude = res[0].latitude;
console.log(i + " " + latitude);
var longitude = res[0].longitude;
console.log(i + " " + longitude);
item.id = i;
item.geodata = r.point(longitude, latitude);
item.active = true;
i++;
callback();
}
});
}, function() {
r.table('dealer_locations').insert(results, {
conflict: "replace"
}).run(request._rdbConn, function(error, results) {
if (error) {
console.log("Data not inserted!");
} else {
console.log("Data inserted!");
}
});
console.log("Done!");
response.send(results);
});
}
})
}
})
}

Here's a possible solution which uses promises to organize the code a little bit.
// Guarantee support for promises and provide the `promisify` function
var Promise = require('bluebird');
// Promisify the geocode function to make it easier to use
var geocode = Promise.promisify(opencage_geocoder.geocode);
function geocode_cleanup(request, response, next) {
var conn = request._rdbConn;
r
.table('dealer_locations')
.filter(r.row('geodata').match('No geodata found.'))
.coerceTo('array')
.run(conn)
.then(function(rows) {
// This promise will be resolve when all rows have been geocoded and updated
// We map the rows into an array of promises, which is what Promise.all takes
return Promise.all(rows.map(function (row) {
return geocode(row.Address)
.then(function (response) {
console.log("Success.");
var latitude = response[0].latitude;
var longitude = response[0].longitude;
row.geodata = r.point(longitude, latitude);
row.active = true;
// Return the row
return row;
});
});
}));
})
.then(function (rows) {
// Now that all `dealer_locations` have been updated, re-query them
return r
.table('dealer_locations')
.insert(rows, {conflict: "update", return_changes: true})
.run(conn);
})
.then(function (results) {
// Send the response;
response.setHeader("Content-Type", "application/json");
response.send(results);
return;
})
.catch(function (err) {
return handleError(null, error);
})
};
Some problems I noticed with your code:
1. Use of do
r.table('dealer_locations').filter(r.row('geodata').match('No geodata found.'))
.do(function(row) {
var geodata = opencage_geocoder.geocode ...
})
In this code snippet, you use a JS function inside of do. You can't do that. Remember that what happens inside of do happens in the RethinkDB server (not in your Node.js server). Your RethinkDB server has no knowledge of your opencage_geocoder function and so this woudn't work.
Whatever do returns must be a valid ReQL query or ReQL expression. You can't execute arbitrary JavaScript inside of it.
If you want to run JavaScript with your query results, you have to .run the query and then do whatever you want to do inside the callback or .then function. At that point, that code will get executed in JavaScript and not in your RethinkDB server.
2. Use of update
return r.table('dealer_locations').update({
geodata: geodata
})
The update method can only update a single document. You can't pass it an array of documents. In this scenario you what have needed to do r.table().get().update() in order for this to work, because you have to be referencing a single document when you update something.
If you have an array of documents that you want to update, you can use the forEach method.
r.table('hello')
.merge({
'new_property': 'hello!'
})
.forEach(function (row) {
// Insert that property into the document
return r.table('hello').get(row.id).update(row);
})
You can also do this (which you are already doing):
r.table('hello')
.merge({
'new_property': 'hello!'
})
.do(function (rows) {
// Insert that property into the document
return r.table('hello')
.insert(rows, {conflict: "update", return_changes: true});
})

OK, I have a suggestion. This queries for the documents you're interested in, modifies them (on your app server, not in the db) and then reinserts them using the nifty conflict: 'update' option. It also uses promises because I think that's a bit cleaner.
function geocode_cleanup(request, response, next) {
r.table('dealer_locations')
.filter(r.row('geodata').match('No geodata found.'))
.run(conn).then(function(cursor) {
var to_update = [];
return cursor.toArray().then(function getGeocodes(rows) {
return rows.map(function getGeocode(row) {
row.geodata = opencage_geocoder.geocode(row.Address, function(error, response) {
if (error) {
console.log("Error.");
row.geodata = "No geodata found.";
row.active = true;
} else if (response.length == 0) {
console.log("Empty response.");
} else {
console.log("Success.");
console.log(response);
var latitude = response[0].latitude;
var longitude = response[0].longitude;
row.geodata = r.point(longitude, latitude);
row.active = true;
}
});
return row;
});
});
}).then(function doneGeocoding(modified_rows){
return r.table('dealer_locations')
.insert(modified_rows, {conflict: "update", return_changes: true})('changes')
.coerceTo('array')
.run(conn);
}).then(function finishResponse(changes){
response.setHeader("Content-Type", "application/json");
response.send(results);
next();
}).catch(function(err) {
// handle errors here
});
};
Caveat emptor, I haven't run this, so there may be syntax errors and things

How can I use node async to fetch my mongoose calls?

I'm building a site with node/express/mongoose and it needs to do the following things when viewing a submission.
The problem I'm running into is doing db fetches in a non-serial fashion. For example, I'll do a few calls to fetch some data, but some of the calls might not finish until the execution context goes to the other. Tried to use the npm module, async, but am having trouble trying to figure out how I would integrate it.
Here is my code:
var getViewCount = function(submissionId) {
Submission.getSubmissionViewCount({
submissionId : submissionId
}, function(err, count) {
if (err) {
throw err;
}
if (count) {
return count;
}
});
};
var getVotes = function(submissionId) {
console.log('getvotes');
Submission.getSubmissionVotes({
submissionId : submissionId
}, function(err, votes) {
return votes;
});
};
var getSubmission = function(id) {
Submission.getSubmission({
id : id
}, function(err, submission) {
if (err) {
throw err;
}
if (submission) {
return submission;
}
});
};
var renderSubmission = function(title, submission, views) {
res.render('submission', {
title: submission.title + ' -',
submission: submission,
views: views.length
});
};
How do I use this with async? Or should I be using async.series isntead of async.async?
async.series([
function(callback) {
var submission = getSubmission(id);
callback(null, submission);
},
function(callback) {
// getViewCount(submissionId);
},
function(callback) {
// getVotes(submissionId);
},
function(callback) {
//renderSubmission(title, submission, views);
}
], function(err, results) {
console.log(results);
});
Basically I want to fetch the views and votes first and then render my submission.

TheBrain's description of the overall structural changes that you should make to your code is accurate. The basic methodology in Node is to nest a series of callbacks; very rarely should you require functions that actually return a value. Instead, you define a function that takes a callback as parameter and pass the result into that callback. Please review the code below for clarification (where cb is a callback function):
var getViewCount = function(submissionId, cb) {
Submission.getSubmissionViewCount({
submissionId : submissionId
}, function(err, count) {
if (err) {
throw err;
}
if (cb) {
cb(count);
}
});
};
var getVotes = function(submissionId, cb) {
console.log('getvotes');
Submission.getSubmissionVotes({
submissionId : submissionId
}, function(err, votes) {
if (cb) {
cb(votes);
}
});
};
var getSubmission = function(id, cb) {
Submission.getSubmission({
id : id
}, function(err, submission) {
if (err) {
throw err;
}
if (cb) {
cb(submission);
}
});
};
var renderSubmission = function(submissionId) {
getSubmission(submissionId, function (submission) {
if (!submission) {
// unable to find submission
// add proper error handling here
} else {
getViewCount(submissionId, function (viewCount) {
res.render('submission', {
title: submission.title + ' -',
submission: submission,
views: viewCount
});
});
}
};
};