We Keep Coding

Foreach with Promise not waiting on method results

I am trying to iterate through the JSON files generated by the protractor tests. I pull all the file names into an array and call a method that opens and parses through the each file, post the results to the database and pass back a passed/failed flag.
I have tried all the examples here
Make angular.forEach wait for promise after going to next object and still get the same results.
The method is actually called, but the results are not posted to the db. I have tested the parser.parseResults on an individual file and it successfully posted to the db, so it has to have something to do with the promise not resolving correctly.
Is it not possible to do something like this in the jasmine/protractor framework? Or do I have something wrong in the code?
I have included the code for my latest attempt.
Thank You
Christine
matches.reduce(function (p, val) {
console.log('val', val);
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
console.log('finalResult = ', finalResult);
}, function (err) {
console.log('error in reduce',err);
});
parser.parseResults code
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
try {
if (err != null) {
console.log('error reading file',err);
reject(err);
}
console.log('obj - ',obj);
var results = [];
var Passed = 0;
var Message = '';
var Stack = '';
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
if (specs.length > 0) {
for (var i = 0; i < specs.length; i++) {
var assert = specs[i];
var tcR = new RegExp(/TC[\d]+/);
var tc = assert.description.match(tcR);
if (!assert.failedExpectations.length) {
Passed = 1;
}
else {
assert.failedExpectations.forEach((expectation) => {
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
})
Passed = 0;
}
if (tc != null) {
utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild,
'P', Message, Stack, 0, moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss'), '')
.then(function (resp) {
resolve(Passed);
}, (err) => {
console.log('Posting to Database failed ', err);
reject(err);
});
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
reject(err);
}
}
}
}
}
catch (err) {
console.log('rejecting opening file');
reject(err);
}
});
})
}

If there is not exactly one suite in the obj, with exactly one spec, then your promise is either resolved not at all or multiple times.
Avoid wrapping too many things in the new Promise constructor - always promisify on the smallest possible level, and use promise chaining afterwards.
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function(obj) {
console.log('obj - ',obj);
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length-1];
Passed = 0;
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.catch(err => {
console.log('Posting to Database failed ', err);
throw err;
}));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results);
});
};

Using Async With waterfall and Recursion in nodejs

I've created a script to migrate data from Dynamo to a Mysql DB.
First I was not using Async, but I started getting bottlenecks on the sql side, so I decided to "throttle" the dymano part using the async lib.
The problem: I have a recursion in the middle of the path, as long as dynamo has data I have to continue the process (ultra simple ETL), but I don't know how to perform the recursion inside the waterfall.
My code :
function main() {
async.waterfall([getMaxTimestamp, scanDynamoDB, printout, saveToMySQL], function(err, result) {
if(err) console.log(err)
console.log(result)
});
}
function getMaxTimestamp(callback) {
console.time("max query");
connection.query("SELECT MAX(created_at) as start_date from Tracking;", function(err, data) {
console.timeEnd("max query");
callback(err, data);
})
}
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
callback(err, data);
// if (!err) {
// if (data != undefined && data.Count > 0) {
// printout(data.Items) // Print out the subset of results.
// if (data.LastEvaluatedKey) { // Result is incomplete; there is more to come.
// query.ExclusiveStartKey = data.LastEvaluatedKey;
// scanDynamoDB(query);
// }
// } else {
// console.log('No fresh data found on Dynamo')
// } else console.dir(err);
});
};
function assembleSql() {
insertSql = "insert into Tracking (";
for (var i = 0; i < headers.length; i++) {
insertSql += headers[i];
if (i < headers.length - 1)
insertSql += ",";
}
insertSql += ") values ?;"
previousInsertSql = insertSql;
}
function saveToMySQL(items, callback) {
assembleSql();
//connection.connect();
console.time("insert sql")
connection.query(insertSql, [items], function(err, result) {
console.timeEnd("insert sql")
if (err){
callback(err, null)
return;
}
totalInserts += result.affectedRows;
callback(err, totalInserts)
//connection.end();
})
}
function printout(items, callback) {
var headersMap = {};
var values;
var header;
var value;
var out = [];
if (headers.length == 0) {
if (items.length > 0) {
for (var i = 0; i < items.length; i++) {
for (var key in items[i]) {
headersMap[key] = true;
}
}
}
for (var key in headersMap) {
headers.push(key);
}
}
for (index in items) {
values = [];
for (i = 0; i < headers.length; i++) {
value = "";
header = headers[i];
// Loop through the header rows, adding values if they exist
if (items[index].hasOwnProperty(header)) {
if (items[index][header].N) {
value = items[index][header].N;
} else if (items[index][header].S) {
value = items[index][header].S;
} else if (items[index][header].SS) {
value = items[index][header].SS.toString();
} else if (items[index][header].NS) {
value = items[index][header].NS.toString();
} else if (items[index][header].B) {
value = items[index][header].B.toString('base64');
} else if (items[index][header].M) {
value = JSON.stringify(items[index][header].M);
} else if (items[index][header].L) {
value = JSON.stringify(items[index][header].L);
} else if (items[index][header].BOOL !== undefined) {
value = items[index][header].BOOL.toString();
}
}
values.push(value)
}
out.push(values)
}
callback(null, out);
}
main();
The commented part is where the recursion happens, but I don't know where to place this inside my flow !
Any help would be appreciated !

Just don't call callback function inside scanDynamoDB while fetching data. You can implement additional function and call it recursive while errors is not appears, like below
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
var result = []; // for accumulate data of each query
function readNext(err, data) {
if (err)
return callback(err);
if (!data || !data.Count)
return callback(null, result);
// add data to result
dynamoDB.scan(query, readNext);
}
dynamoDB.scan(query, readNext);
};

Actually I was able to figure it out by myself.
async.whilst(function() { return canInsert}, function (callback){
scanDynamoDB(query, callback)
}, function(err, res) {}
function scanDynamoDB(data, callback) {
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
if (!err) {
if (data != undefined && data.Count > 0) {
canInsert = data.LastEvaluatedKey;
if (data.LastEvaluatedKey) // Result is incomplete; there is more to come.
query.ExclusiveStartKey = data.LastEvaluatedKey;
}
} else console.dir(err);
});
};
I could have done it just with a while(canInsert). Anyway, I avoided recursion and memory usage is way way lower.

Node.js Function Flow

When I get a request, I want it to generate a 4-character code, then check if it already exists in the database. If it does, then generate a new code. If not, add it and move on. This is what I have so far:
var code = "";
var codeFree = false;
while (! codeFree) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code=" + code, function(err, result) {
if (! err) {
if (result.rows.length > 0) {
codeFree = false;
} else {
codeFree = true;
}
} else {
console.log('DB ERR: %s', err);
}
console.log(codeFree);
});
console.log('here');
}
This does not do nearly what I want it to do. How can I handle something like this?

You are doing an async task.
When you have an asyncronous task inside your procedure, you need to have a callback function which is going to be called with the desired value as its argument.
When you found the free code, you call the function and passing the code as its argument, otherwise, you call the getFreeCode function again and passing the same callback to it. Although you might consider cases when an error happens. If your the db call fails, your callback would never get called. It is better to use a throw/catch mechanism or passing another argument for error to your callback.
You can achieve what you need to do by doing it this way:
function getFreeCode(callback) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code="+code, function(err, result) {
if(!err) {
if(result.rows.length > 0) {
getFreeCode(callback);
} else {
callback(code);
}
}else {
console.log('DB ERR: %s', err);
}
console.log(codeFree);
});
console.log('here');
}
// in your main:
getFreeCode(function (code) {
console.log(' this code was free: ' + code)
})

I recommend you look into two alternatives to help deal with asynchronous code.
node generator functions using the 'yield' keyword
promises
Using generators requires running a recent version of node with the --harmony flag. The reason I recommend generators is because you can write code that flows the way you expect.
var x = yield asyncFunction();
console.log('x = ' + x);
The previous code will get the value of x before logging x.
Without yielding the console.log would write out x before the async function was finished getting the value for x.
Your code could look like this with generators:
var client = {
execute: function (query) {
var timesRan = 0;
var result = [];
return function () {
return setTimeout(function () {
result = ++timesRan < 4 ? ['length_will_be_1'] : [];
return result;
},1);
};
}
};
function* checkCode () {
var code;
var codeFree = false;
while(!codeFree) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
try {
var result = yield client.execute("select * from codes where code="+code);
codeFree = result.rows.length > 0 ? false : true;
}catch(e) {
console.log('DB ERR: %s', err);
} finally {
console.log(codeFree);
}
console.log('here');
}
}
checkCode().next();
You would leave off the client object. I only added that to make a working example that fakes an async call.
If you have to use an older version of node or do not like the yield syntax then promises could be a worthy option.
There are many promise libraries. The reason I recommend promises is that you can write code that flows the way you expect:
asyncGetX()
.then(function (x) {
console.log('x: ' + x);
});
The previous code will get the value of x before logging x.
It also lets you chain async functions and runs them in order:
asyncFunction1()
.then(function (result) {
return asyncFunction2(result)
})
.then(function (x) { /* <-- x is the return value from asyncFunction2 which used the result value of asyncFunction1 */
console.log('x: ' + x);
});
Your code could look like this with the 'q' promise library:
var Q = require('q');
var client = {
timesRan: 0,
execute: function (query, callback) {
var self = this;
var result = {};
setTimeout(function () {
console.log('self.timesRan: ' + self.timesRan);
result.rows = ++self.timesRan < 4 ? ['length = 1'] : [];
callback(null, result);
},1);
}
};
function checkCode () {
var deferred = Q.defer();
var codeFree = false;
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log('rand: %s', rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code="+code, function(err, result) {
console.log('err: '+err+', result: ' + JSON.stringify(result));
console.log('result.rows.length: ' + result.rows.length);
if(!err) {
if(result.rows.length > 0) {
codeFree = false;
console.log('result.rows: %s, codeFree: %s', result.rows, codeFree);
checkCode();
} else {
codeFree = true;
console.log('line 36: codeFree: ' + codeFree);
deferred.resolve(code);
}
}else {
console.log('DB ERR: %s', err);
deferred.reject(err);
}
console.log(codeFree);
});
console.log('waiting for promise');
return deferred.promise;
}
checkCode()
.then(function (code) {
console.log('success with code: ' + code);
})
.fail(function(err) {
console.log('failure, err: ' + err);
});
Also omit the client object here. I only added that to make a working example that fakes an async call.
Promises and generators definitely take some time to get used to. It's worth it because they make the code a lot easier to follow in the end than code written with nested callbacks.

How to set object array back to iterated collection from nested find() in MongoDB?

I develop application using AngularJS , NodeJS and MongoDB. I'd like to load Product with classified by ProductCategoryCode sending from
AngualrJS to NodeJS. First, I need to find Products by ProductCategoryCode and then iterate for each product to find Uoms by UomCode and ContainUomCode
which each product should has 2 uoms. How can I set uom object docUom back to product document doc[i] and update to product collection doc?
For following code line
doc[i].Uom = docUom;
The system throw error cannot set property 'Uom' of undefined.
Here is product.js snippet code.
router.get("/LoadProductByProductCategoryCode/:productCategoryCode", function (req, res) {
console.log('user.js -> /users ');
var productCategoryCode = req.params.productCategoryCode;
console.log(productCategoryCode );
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://localhost:27017/NodeDB', function (err, db) {
if (err) throw err;
var query = { ProductCategoryCode : productCategoryCode}
var new_product = [];
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
if (doc) {
console.log("Found Product..."+doc.length);
　 for (var i = 0; i < doc.length; i++) {
console.log(doc[i].ProductCode + " each document " + doc[i].UomCode + " " + doc[i].ContainUomCode);
var qUom = {
$or: [ { UomCode: doc[i].UomCode}, { UomCode: doc[i].ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
doc[i].Uom = docUom;
console.dir(product);
}
});
}
res.json(doc);
} else {
console.log('something happen');
}
}); //End
}); // MongoClient
var findProduct = function (db, query, callback) {
db.collection('Product').find(query).toArray(function (err, doc) {
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
var findUom = function(db, queryUom, callback) {
db.collection('Uom').find(queryUom).toArray(function (err, doc) {
// db.close();
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
});
Any idea? THANKS

Because of the asynchronous nature of the Node.js MongoDB driver, both the findProduct() and findUom() methods start, but don't necessarily complete by the time you reach res.json(doc) meaning doc will still be empty. You are expecting this to work in a linear fashion, but node works differently.
Instead, you should send your response back once all asynchronous calls complete meaning you could try something like:
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
var processedProduct = function (item) {
console.log(item.ProductCode + " each document " + item.UomCode + " " + item.ContainUomCode);
var qUom = {
$or: [ { UomCode: item.UomCode}, { UomCode: item.ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
item.Uom = docUom;
console.dir(product);
return item;
}
});
} 　
if (doc) {
var productsToFind = doc.length;
var products = [];
console.log("Found Products..." + productsToFind);
for (var i = 0; i < doc.length; i++) {
product = doc[i];
product = processedProduct(product);
products.push(product);
productsToFind -= 1;
if(productsToFind === 0){
res.json(products);
}
}
} else {
console.log('something happen');
}
}); //End
I could explain better about asynchronous calls and callbacks as this topic is a bit broad but from the above you can get the idea that I have used a counter productsToFind for all of the inner async calls that once each findUom() call completes this counter decrements and once it reaches 0 it means that all callbacks have fired.

Necessity of closure in word count example in Node.js in Action

In the book Node.js in Action, there is one example where files in directory "./text" is subject to word count. I want to ask if the closure used is actually necessarily or just a matter of style. The code is below:
var fs = require('fs');
var completedTasks = 0;
var tasks = [];
var wordCounts = {};
var filesDir = './text';
function checkIfComplete() {
completedTasks++;
if (completedTasks == tasks.length) {
for (var index in wordCounts) {
console.log(index + ': ' + wordCounts[index]);
}
}
}
function countWordsInText(text) {
var words = text
.toString()
.toLowerCase()
.split(/\W+/)
.sort()
for (var index in words) {
var word = words[index];
if (word) {
wordCounts[word] = (wordCounts[word]) ? wordCounts[word] + 1 : 1;
}
}
}
// My question is on the use of closure below
fs.readdir(filesDir, function(err, files) {
if (err) throw err;
for (var index in files) {
var task = (function(file) {
return function() {
fs.readFile(file, function(err, text) {
if (err) throw err;
countWordsInText(text);
checkIfComplete();
});
}
})(filesDir + '/' + files[index]);
tasks.push(task);
}
for (var task in tasks) {
tasks[task]();
}
});
This code is used in the book to demonstrate the nature of parallel control flow. My question is why does the code goes through this seemingly twisted elaboration (sorry, lol, a newbie here) of constructing closure then calling them? (I'm referring to the each task in tasks.)
Is that any different from what comes more natural to me like so?
fs.readdir(filesDir, function(err, files) {
if (err) throw err;
tasks = files; // Just to make sure the check for completion runs fine.
for (var index in files) {
var file = files[index];
fs.readFile(filesDir + '/' + file, function(err, text) {
if (err) throw err;
countWordsInText(text);
checkIfComplete();
});
}
});
Wouldn't it still be asynchronous and as "parallel" as it was in the previous code?

It is possible to avoid the closure here. The closure would be required to avoid the infamous closure-in-loop gotcha which would happen if the callback passed to fs.readFile accessed the loop variable or something derived from it. Because then, by the time the callback would be called, the value of the loop variable would be the last value assigned to it.
The way you do it is okay. I've not tried to completely clean the code or make it resilient to faults but I'd do:
var fs = require('fs');
var wordCounts = Object.create(null);
var filesDir = './text';
function countWordsInText(text) {
var words = text
.toString()
.toLowerCase()
.split(/\W+/)
.sort();
for (var index in words) {
var word = words[index];
if (word) {
wordCounts[word] = (wordCounts[word]) ? wordCounts[word] + 1 : 1;
}
}
}
fs.readdir(filesDir, function(err, files) {
if (err) throw err;
var total = files.length;
var completedTasks = 0;
for (var index in files) {
fs.readFile(filesDir + '/' + files[index], function(err, text) {
if (err) throw err;
countWordsInText(text);
completedTasks++;
if (completedTasks === total) {
for (var index in wordCounts) {
console.log(index + ': ' + wordCounts[index]);
}
}
});
}
});