We Keep Coding

MEAN Node JS requests intercourse at many requests

I have a MEAN app that works well with single requests, let's say calling /api/products?pid=500. But I recently discovered that at a "burst" of requests (i'm updating bulk around 50 products = 50 requests /api/products?pid=500 *** 550 with post data), the req.body sometimes gets a value of a new upcoming request.
The front app makes the calls in a foreach of selected products:
ds.forEach((d, key) => {
this.ApiCall.setData('products', { action: 'send-product', data: d })
.subscribe((result) => {
//we have results
});
});
//setData makes a http.post().map
Back app / mean analyses the post, tried to synthesize the code:
router.route('/')
.post(function (req, response) {
if(req.body.data){
var obj = { id: req.body.data.product_id }
if(req.body.data.linked_products){
req.body.data.linked_products.forEach(function(entry) {
obj.linked = entry; //more ifs
});
}
var async = require('async');
async.series({
q2: function(cb){
queryProducts.findOne({id: req.body.data.product_id, null).exec(cb);
},
q3: function(cb){
queryCategories.findOne({id: req.body.data.category_id, null).exec(cb);
}
}, function(err, qResults){
var alreadysent = false;
if (qResults.q3) qResults.q3.logs.forEach(function(entry) {
if(entry.sent){
alreadysent = true;
}
});
//more ifs
qResults.q3.external_codes.forEach(function(entry) {
obj.external_code = entry;//more ifs
});
if(req.body.data.price < 0){
response.json({message: "Negative price didn't sent"});
return;
}
if(qResults.q2.status=="inactive"){
response.json({message: "Inactive didn't sent"});
return;
}
req.body.data.campaigns(function(entry) {
obj.price_offers = entry;//more ifs
});
//more ifs and foreach similar
queryProducts.update({id: req.body.data.id}, {$push: { synced_products: obj }}, function (err, result) {
//HERE I found req.body.data with values of a future request
if(!err)
response.json({message: "Sent"});
return;
});
});
}
});
module.exports = router;
I understand that making requests
/api/products?pid=500
/api/products?pid=501
/api/products?pid=502
/api/products?pid=503
...
have different timings, but how is possible that a request (pid=501), calling the last req.body to have the value of req.body of new req (pid=503)?
Any ideas how to avoid? putting async first right after the post or making a
var reqbody = req.body
Thanks!

I believe this is due to the async module initialization. To quote from the node docs:
Caching
Modules are cached after the first time they are loaded. This means (among other things) that every call to require('foo') will get exactly the same object returned, if it would resolve to the same file.
Multiple calls to require('foo') may not cause the module code to be executed multiple times. This is an important feature. With it, "partially done" objects can be returned, thus allowing transitive dependencies to be loaded even when they would cause cycles.
To have a module execute code multiple times, export a function, and call that function.
When a burst of requests causes overlapping execution, you will have two (or more) uses of the async variable being modified "concurrently". I would suggest using some sort of mutex to control access to the async variable.

Javascript memory consumption with map() over a large set and callbacks

I don't even know how to properly ask this question but I have concerns about the performance (mostly memory consumption) of the following code. I am anticipating that this code will consume a lot of memory because of map on a large set and a lot of 'hanging' functions that wait for external service. Are my concerns justified here? What would be a better approach?
var list = fs.readFileSync('./mailinglist.txt') // say 1.000.000 records
.split("\n")
.map( processEntry );
var processEntry = function _processEntry(i){
i = i.split('\t');
getEmailBody( function(emailBody, name){
var msg = {
"message" : emailBody,
"name" : i[0]
}
request(msg, function reqCb(err, result){
...
});
}); // getEmailBody
}
var getEmailBody = function _getEmailBody(obj, cb){
// read email template from file;
// v() returns the correct form for person's name with web-based service
v(obj.name, function(v){
cb(obj, v)
});
}

If you're worried about submitting a million http requests in a very short time span (which you probably should be), you'll have to set up a buffer of some kind.
one simple way to do it:
var lines = fs.readFileSync('./mailinglist.txt').split("\n");
var entryIdx = 0;
var done = false;
var processNextEntry = function () {
if (entryIdx < lines.length) {
processEntry(lines[entryIdx++]);
} else {
done = true;
}
};
var processEntry = function _processEntry(i){
i = i.split('\t');
getEmailBody( function(emailBody, name){
var msg = {
"message" : emailBody,
"name" : name
}
request(msg, function reqCb(err, result){
// ...
!done && processNextEntry();
});
}); // getEmailBody
}
// getEmailBody didn't change
// you set the ball rolling by calling processNextEntry n times,
// where n is a sensible number of http requests to have pending at once.
for (var i=0; i<10; i++) processNextEntry();
Edit: according to this blog post node has an internal queue system, it will only allow 5 simultaneous requests. But you can still use this method to avoid filling up that internal queue with a million items if you're worried about memory consumption.

Firstly I would advise against using readFileSync, and instead favour the async equivalent. Blocking on IO operations should be avoided as reading from a disk is very expensive, and whilst that's the sole purpose of your code now, I would consider how that might change in the future - and arbitrarily wasting clock cycles is never a good idea.
For large data files I would read them in in defined chunks and process them. If you can come up with some schema, either sentinels to distinguish data blocks within the file, or padding to boundaries, then process the file piece by piece.
This is just rough, untested off the top of my head, but something like:
var fs = require("fs");
function doMyCoolWork(startByteIndex, endByteIndex){
fs.open("path to your text file", 'r', function(status, fd) {
var chunkSize = endByteIndex - startByteIndex;
var buffer = new Buffer(chunkSize);
fs.read(fd, buffer, 0, chunkSize, 0, function(err, byteCount) {
var data = buffer.toString('utf-8', 0, byteCount);
// process your data here
if(stillWorkToDo){
//recurse
doMyCoolWork(endByteIndex, endByteIndex + 100);
}
});
});
}
Or look into one of the stream library functions for similar functionality.
H2H
ps. Javascript and Node works extremely well with async and eventing.. using sync is an antipattern in my opinion, and likely to cause code to be a headache in future

Batch requests in Node.js

My program is communicating with a web service that only accepts ~10 requests per second. From time to time, my program sends 100+ concurrent requests to the web service, causing my program to crash.
How do I limit concurrent requests in Node.js to 5 per second? Im using the request library.
// IF EVENT AND SENDER
if(data.sender[0].events && data.sender[0].events.length > 0) {
// FIND ALL EVENTS
for(var i = 0; i < data.sender[0].events.length; i++) {
// IF TYPE IS "ADDED"
if(data.sender[0].events[i].type == "added") {
switch (data.sender[0].events[i].link.rel) {
case "contact" :
batch("added", data.sender[0].events[i].link.href);
//_initContacts(data.sender[0].events[i].link.href);
break;
}
// IF TYPE IS "UPDATED"
} else if(data.sender[0].events[i].type == "updated") {
switch (data.sender[0].events[i].link.rel){
case "contactPresence" :
batch("updated", data.sender[0].events[i].link.href);
//_getContactPresence(data.sender[0].events[i].link.href);
break;
case "contactNote" :
batch("updated", data.sender[0].events[i].link.href);
// _getContactNote(data.sender[0].events[i].link.href);
break;
case "contactLocation" :
batch("updated", data.sender[0].events[i].link.href);
// _getContactLocation(data.sender[0].events[i].link.href);
break;
case "presenceSubscription" :
batch("updated", data.sender[0].events[i].link.href);
// _extendPresenceSubscription(data.sender[0].events[i].link.href);
break;
}
}
};
And then the homegrown batch method:
var updated = [];
var added = [];
var batch = function(type, url){
console.log("batch called");
if (type === "added"){
console.log("Added batched");
added.push(url);
if (added.length > 5) {
setTimeout(added.forEach(function(req){
_initContacts(req);
}), 2000);
added = [];
}
}
else if (type === "updated"){
console.log("Updated batched");
updated.push(url);
console.log("Updated length is : ", updated.length);
if (updated.length > 5){
console.log("Over 5 updated events");
updated.forEach(function(req){
setTimeout(_getContactLocation(req), 2000);
});
updated = [];
}
}
};
And an example of the actual request:
var _getContactLocation = function(url){
r.get(baseUrl + url,
{ "strictSSL" : false, "headers" : { "Authorization" : "Bearer " + accessToken }},
function(err, res, body){
if(err)
console.log(err);
else {
var data = JSON.parse(body);
self.emit("data.contact", data);
}
}
);
};

Using the async library, the mapLimit function does exactly what you want. I can't provide an example for your specific use case as you did not provide any code.
From the readme:
mapLimit(arr, limit, iterator, callback)
The same as map only no more than "limit" iterators will be simultaneously
running at any time.
Note that the items are not processed in batches, so there is no guarantee that
the first "limit" iterator functions will complete before any others are
started.
Arguments
arr - An array to iterate over.
limit - The maximum number of iterators to run at any time.
iterator(item, callback) - A function to apply to each item in the array.
The iterator is passed a callback(err, transformed) which must be called once
it has completed with an error (which can be null) and a transformed item.
callback(err, results) - A callback which is called after all the iterator
functions have finished, or an error has occurred. Results is an array of the
transformed items from the original array.
Example
async.mapLimit(['file1','file2','file3'], 1, fs.stat, function(err, results){
// results is now an array of stats for each file
});
EDIT: Now that you provided code, I see that your use is a bit different from what I assumed. The async library is more useful when you know all the tasks to run up front. I don't know of a library off hand that will easily solve this for you. The above note is likely still relevant to people searching this topic so I'll leave it in.
Sorry, I don't have time to restructure your code, but this is an (un-tested) example of a function that makes an asynchronous request while self-throttling itself to 5 requests per second. I would highly recommend working off of this to come up with a more general solution that fits your code base.
var throttledRequest = (function () {
var queue = [], running = 0;
function sendPossibleRequests() {
var url;
while (queue.length > 0 && running < 5) {
url = queue.shift();
running++;
r.get(url, { /* YOUR OPTIONS HERE*/ }, function (err, res, body) {
running--;
sendPossibleRequests();
if(err)
console.log(err);
else {
var data = JSON.parse(body);
self.emit("data.contact", data);
}
});
}
}
return function (url) {
queue.push(url);
sendPossibleRequests();
};
})();
Basically, you keep a queue of all the data to be asynchronously processed (such as urls to be requested) and then after each callback (from a request) you try to launch off as many remaining requests as possible.

This is precisely what node's Agent class is designed to address. Have you done something silly like require('http').globalAgent.maxSockets = Number.MAX_VALUE or passed agent: false as a request option?
With Node's default behavior, your program will not send more than 5 concurrent requests at a time. Additionally, the Agent provides optimizations that a simple queue cannot (namely HTTP keepalives).
If you try to make many requests (for example, issue 100 requests from a loop), the first 5 will begin and the Agent will queue the remaining 95. As requests complete, it starts the next.
What you probably want to do is create an Agent for your web service requests, and pass it in to every call to request (rather than mixing requests in with the global agent).
var http=require('http'), svcAgent = http.Agent();
request({ ... , agent: svcAgent });

Best method to string together variety of DB calls in Node js

I basically need to make about 3 calls to get the data for a json object.. It basically JSON array of JSON object which have some attributes, one of which is an array of other values selected using a second query, then that one also has an array inside which is selected with another db call.
I tried using asyn.concatSeries so that I can dig down into the bottom call and put together all the information I collected for one root json object but that's creating a lot of unexpected behaviour..
Example of JSON
[
{
"item" : "firstDbCall"
"children" : [ {
"name" : "itemDiscoveredWithSecondDBCall"
"children" : [ itemsDiscoveredwith3rdDBCall]
},
]
}
]
This is really difficult using node.js. I really need to figure out how to do this properly since I have to do many of these for different purposes.
EDIT
This is the code i have. There's some strange behaviour with async.concatSeries. The results get called multiple times after each one of the functions finish for each array. So i had to put a check in place. I know it's very messy code but i've been just putting band-aids all over it for the past 2 hours to make it work..
console.log("GET USERS HAREDQARE INFO _--__--_-_-_-_-_____");
var query = "select driveGroupId from tasks, driveInformation where agentId = '"
+ req.params.agentId + "' and driveInformation.taskId = tasks.id order by driveInformation.taskId desc;";
connection.query(query, function(err, rows) {
if (rows === undefined) {
res.json([]);
return;
}
if(rows.length<1) { res.send("[]"); return;}
var driveGroupId = rows[0].driveGroupId;
var physicalQuery = "select * from drives where driveGroupId = " + driveGroupId + ";";
connection.query(physicalQuery, function(err, rows) {
console.log("ROWSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS");
console.log(rows);
async.concatSeries(rows, function(row, cb) {
console.log("-------------------------------SINGLE ROW-------------------------------------");
console.log(row);
if(row.hasLogicalDrives != 0) {
console.log("HAS LOGICAL DRIVES");
console.log(row.id);
var query = "select id, name from logicalDrives where driveId = " + row.id;
connection.query(query, function(error, drives) {
console.log("QUERY RETURNED");
console.log(drives);
parseDriveInfo(row.name, row.searchable, drives, cb);
});
}
else
var driveInfo = { "driveName" : row.name, "searchable" : row.searchable};
console.log("NO SUB ITEMS");
cb(null, driveInfo);
}, function(err, results) {
console.log("GEETTTTINGHERE");
console.log(results);
if(results.length == rows.length) {
console.log("RESULTS FOR THE DRIVE SEARCH");
console.log(results);
var response = {"id": req.params.agentId};
response.driveList = results;
console.log("RESPONSE");
console.log(response);
res.json(response);
}
});
});
});
};
parseDriveInfo = function(driveName, searchable, drives, cb) {
async.concatSeries(drives, function(drive,callback) {
console.log("SERIES 2");
console.log(drive);
console.log("END OF DRIVE INFO");
var query = "select name from supportedSearchTypes where logicalDriveId = " + drive.id;
connection.query(query, function(error, searchTypes) {
drive.searchTypes = searchTypes;
var driveInfo = { "driveName" :driveName,
"searchable" : searchable,
"logicalDrives" : drive
};
callback(null, driveInfo);
});
}, function (err, results) {
console.log("THIS IS ISISIS ISISISSISISISISISISISISISIS");
console.log(results);
if(results.length === drives.length) {
console.log("GOTHERE");
cb(null, results);
}
});
}

Getting good enough with async to use exactly the right combination of methods under the right circumstances takes a fair amount of experience. Most likely your case in particular can be handled with async.waterfall if its query1 then query2(dataFoundByQuery1) then query3(dataFoundByQuery2). But depending on the circumstances you need to mix and match async methods appropriately and sometimes have 2 levels - for example a "big picture" async.waterfall where some of the steps in the waterfall do async.parallel or async.series as needed. I've never used async.concat and given your needs I think you have chosen the wrong method. The workhorses are async.each, async.eachSeries, async.waterfall, and async.map, at least for the web app & DB query use cases I mostly encounter, so make sure you really have those understood before exploring the more specific convenience methods.

EDIT: This is a more in depth example based on use of the connection library you seem to be using. Please note, some of this is javascript psuedo code. Things like adding objects to the resultsArray are clearly not complete, the only thing I took time to make sure was correct is the "flow of logic" as it pertains to callbacks. Everything else is for you to implement. In order to support multiple calls to the same callback function and maintain state from call to call, the best way is to wrap the set of callbacks in a closure. This allows the callbacks to share some state with the main event loop. This allows you to pass arguments to the callbacks, without actually having to pass them as arguments, much like class variables in c++, or even globals in javascript, but we haven't poluted the global scope :)
function queryDataBase(query) {
//wrap the whole query in a function so the callbacks can share some
//variables with similar scope. This is called a closure
int rowCounter = 0;
var dataRowsFromStep2;
var resultsArray = {};
connection.query(query, dataBaseQueryStep2);
function dataBaseQueryStep2(err, rows) {
//do something with err and rows
dataRowsFromStep2 = rows;
var query = getQueryFromRow(dataRowsFromStep2[rowCounter++]);//Always zero the first time. Might need to double check rows isn't empty!
connection.query(query, dataBaseQueryStep3);
}
function dataBaseQueryStep3(err, rows) {
//do something with err and rows
if(rowCounter < dataRowsFromStep2.size) {
resultsArray.add(rows);//Probably needs to be more interesting, but you get the idea
//since this is within the same closure, rowCounter maintains it's state
var query = getQueryFromRow(dataRowsFromStep2[rowCounter++]);
//recursive call query using dataBaseQueryStep3 as it's callback repeatedly until
//we run out of rows to call it on.
connection.query(query, dataBaseQueryStep3)
} else {
//when the if statement fails we have no more rows to run queries on so return to main program flow
returnToMainProgramLogic(resultsArray);
}
}
}
function returnToMainProgramLogic(results) {
//continue running your program here
}
I personally like the above logic better than the syntax async produces... I believe the heart of your problem rests in your nested calls to async, and the fact that ASYN itself, runs the series of functions asynchronously, but in order(confusing I know). If you write your program like this, you won't have to worry about it!

I would strongly suggest using sequelize.js It provides a really powerful orm that allows you to chain queries together. It also allows you to directly load your data into js objects, write dynamic sql, and connect to many different databases. Picture ActiveRecord from the Ruby world for Node.

Synchronous query to Web SQL Database

I'm working on a bit of JavaScript that interacts with a client-side SQLite database, via the newish window.openDatabase(...), database.transaction(...) and related APIs. As most of you know when you execute a query in this way it is an asynchronous call, which is typically good. You can make the call and handle the results as appropriate with callbacks.
In my current situation I'm working on an algo for a client that does some hierarchy walking in the locally stored database. The part of the algo I'm having trouble with requires starting at some row, which has a reference to a "parent" (by id) that is another row further up in the table. I have to keep walking up this tree until I reach the root.
The problem is that I'm at a point where I'm not sure how to use an asynchronous style query with a callback to keep feeding the loop parent ids. Ideally I could get the query to block so that I can do it all in the loop. Here's the key parts of my current setup:
for (i in search.searchResults.resultsArray)
{
hierarchyArr = new Array();
pageHierarchyArr = new Array();
id = search.searchResults.resultsArray[i].ID;
while (id != null && id != "")
{
var hierarchySql = "SELECT ID, parentID, type, content FROM content WHERE ID = " + id;
// This is a prettied up call to database.transaction(...)
var rs = db.getRS(hierarchySql);
// Ideally the code below doesn't execute until rs is populated
hierarchyArr.push(rs[0]);
if (rs[0].type == "page")
{
pageHierarchyArr.push(rs[0]);
// Do some additional work
}
id = rs[0].parentID;
}
}
As you might imagine, it doesn't work well. hierarchyArr gets an "undefined" pushed into it, and then the script crashes when it tries to check the type of rs[0].
When I try to set it up with a callback (db.getRSAndCallback(sql, callbackFunc), which I used for the earlier, non-interdependent queries just fine) it's worse: the inner loop takes off like crazy because id isn't getting updated; presumably because the loop is keeping the JavaScript interpreter so busy that it never actually fills rs. In some artificial testing where I forced the inner loop to break after a few iterations all the callbacks started coming through all at the end, after the loop finished.
The "standard" (such as it is right now) at http://dev.w3.org/html5/webdatabase/#synchronous-database-api seems to indicate that there is a synchronous API, but I haven't seen any sign of it on any WebKit based browsers.
Can anyone offer suggestions on how I might either, a. properly formulate these iterative, interdependent queries using callbacks or, b. somehow get the call to actually happen in a synchronous or apparently synchronous manner.
Many thanks in advance for anyone who takes a crack at this seemingly tricky little problem.
Naim
P.S. Here's the client's implementation of db.getRS for reference:
.
.
.
getRS: function(sql)
{
var output = [];
db.database.transaction(function(tx)
{
tx.executeSql(sql, [], function(tx,rs)
{
for(i = 0; i < rs.rows.length; i++)
{
output.push(rs.rows.item(i));
}
},
function(tx, error) { ... }
)});
return output;
},
.
.
.

I used callbacks and a closure to solve a similar problem, consider:
function getFolder(id, callback) {
var data = [];
ldb.transaction(function (tx) {
tx.executeSql('SELECT * FROM folders where id=?',
[id],
function (tx, results) {
if (results.rows && results.rows.length) {
for (i = 0; i < results.rows.length; i++) {
data.push(results.rows.item(i));
}
}
if (typeof(callback) == 'function')
callback(data);
},
function (tx, error) {
console.log(error);
});
});
}
In the continuation of this example, folder has a property parent to define it's relation to other folders. As does a document. The following will get you the path of a document using a closure (success):
function getDocPath(doc, callback) {
var path = [];
var parent = doc.parent;
var success = function(folder) {
var folder = folder[0];
parent = folder.parent;
path.push({'id':folder.id,'name':folder.name});
if (parent != "undefined")
getFolder(parent, success);
else
if ( typeof(callback) == 'function' ) callback(path.reverse());
}
getFolder(parent, success);
}

You could use callbacks with a closure to your stack of remaining queries. Or you could use recursion, passing the stack as parameters.