We Keep Coding

node.js async/await function called multiple times (before it ends)

My main file is a "file watcher" that's watching a folder using "node-watch" module. For each file placed in the folder, it has to do couple of MSSQL calls and then process the file accordingly.
The file watcher script looks like that:
var watch = require('node-watch');
const timestamp = require('./SHR_modules/timestamp');
const sql = require('./SHR_modules/sql');
const configuration = require('./SHR_modules/config');
const logger = require('./SHR_modules/logger');
watch('./Incoming', { recursive: true }, async function (evt, name) {
if (evt == 'update') {
logger.addLog(configuration.logFile.watcher, 'File found: ' + name);
var startTime = await timestamp.dateTimeDelimited(); //capture time at proicessing start
filePath = await name.replace(name.split('\\')[name.split('\\').length-1], '');
fileName = await name.replace(filePath, '');
var myLoad = await sql.readLoadSpec(fileName, filePath).catch(err => {
logger.addLog(configuration.logFile.error, 'Error while reading Load Specs for file: ' + name);
logger.addLog(configuration.logFile.error, err);
});
if (typeof myLoad !== 'undefined' && myLoad){
console.log(evt);
logger.addLog(configuration.logFile.watcher, 'Finished processing: ' + name);
};
var finishTime = await timestamp.dateTimeDelimited(); //capture time at processing finish
}
else {
logger.addLog(configuration.logFile.watcher, 'File removed: ' + name);
}
console.log(startTime);
console.log(finishTime);
});
and the SQL function:
var readLoadSpec = (fileName, filePath) => {
return new Promise(async (resolve, reject) => {
specIDs = await findSpecs(fileName, filePath)
.catch(err => {
reject(err);
console.log(fileName + ' not found')
});
if (typeof specIDs !== 'undefined' && specIDs) {
console.log('FOUND!!!!');
var addLoadSpec = [];
for (let i = 0; i < specIDs.length; i++) {
console.log(specIDs);
var tempQuery1 = `select * from LoadSpec where LoadID = ${specIDs[i]}`;
var tempQuery2 = `select * from Mappings where LoadID = ${specIDs[i]} ORDER BY OutputColumn`;
console.log(specIDs); <========= code work sup to this line
const results = await queryDB(tempQuery1);
const resultsMapping = await queryDB(tempQuery2);
console.log(fileName);
console.log(results.recordset[0].OutputFileName);
console.log(specIDs);
const inputFrmt = {
name: results.recordset[0].InputFormatName,
format: results.recordset[0].InputFormat,
delimiter: results.recordset[0].InputDelimiter,
headerRows: results.recordset[0].InputHeaderRows
};
.
.
.
console.log(results.recordset[0].OutputColumnCount);
addLoadSpec.push(loadSpec);
};
resolve(addLoadSpec);
};
});
};
So the code runs to the row I've marked (before the await for query results) and the loops back to beginning.
So all is fine if I drop a single file into the folder, but when multiple files arrive at more or less the same time, the syncIDs var takes value of another file, before the code resumes after the query await part is completed. syncIDs is just an array so let's say for file 'a.csv' it's [1,2], for file 'b.csv' it's [3,4] and for file 'c.csv' it's undefined. So before it get's the answer from the DB, this loop:
specIDs = await findSpecs(fileName, filePath)
.catch(err => {
reject(err);
console.log(fileName + ' not found')
});
if (typeof specIDs !== 'undefined' && specIDs) {
console.log('FOUND!!!!');
var addLoadSpec = [];
for (let i = 0; i < specIDs.length; i++) {
Get's error for index of undefined
How do I make it process files one by one?

ReplaceAll causing issues in array.reduce

I am still pretty new to this, so forgive me if I dont' say this correctly. We have an array.reduce that calls a method with a returning promise that iterates through a list of files and post results to the db. Everything was working great, until it ran into a field that had an apostrophe in it and then the db insert fails. This is the field value. 'Expected 100002822' to be 100002822.'
I tried adding a replaceAll on the field and now get an error in the array.reduce.
Here is the .reduce
console.log('Found test results in ' + matches.length + ' files. Parsing and posting to the database now...');
var startTime = moment();
var parser = new Parser();
matches.reduce(function (p, val) {
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
var endTime = moment();
var testDuration = moment.duration(endTime.diff(startTime));
console.log(chalk.blue('*** File parsing time: ' + testDuration.humanize() + ' ***'));
if (finalResult.insertSuccess == matches.length) {
var publishOut = {
totalFiles: matches.length,
totalTests: 0,
totalTestsSuccess: 0,
totalTestsFailed: 0
}
publishOut.totalTests += finalResult.totalTests;
publishOut.totalTestsSuccess += finalResult.testPassedCount;
publishOut.totalTestsFailed += finalResult.testFailedCount;
console.log(`Successfully inserted ${finalResult.insertSuccess} of ${publishOut.totalTests} test results.`);
// for (var i = 0; i < matches.length; i++) {
// var currentFile = `./testing/results/${matches[i]}`;
// fs.unlinkSync(currentFile);
// }
resolve(publishOut);
} else {
reject('Only ' + finalResult.insertSuccess + ' of ' + matches.length + ' successfully posted to the database');
}
}, function (err) {
reject('error in reduce', err);
});
I have tried several different ways of using the replaceAll with the same failure. It hits this code from the array.reduce
}, function (err) {
reject('error in reduce', err);
});
And this is the called method. The added code causing the failure in the .reduce is this Message = expectation.message.replaceAll("'", "");
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
//console.log('In parseresults', fileName);
var currentFile = './testing/results/' + fileName
json.readFile(currentFile, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function (obj) {
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
testResults.totalTests++;
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length - 1];
Passed = 0;
Message = expectation.message.replaceAll("'", "");
Stack = expectation.stack.split('\n')[1].trim();
testResults.testFailedCount++
} else {
testResults.testPassedCount++
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.then(() => {
//fs.unlinkSync(currentFile);
testResults.insertSuccess++;
//console.log('insertSuccess', testResults.insertSuccess);
},
err => { console.log('… failed', err); throw err; }
));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results).then(() => testResults);
});
};

Foreach with Promise not waiting on method results

I am trying to iterate through the JSON files generated by the protractor tests. I pull all the file names into an array and call a method that opens and parses through the each file, post the results to the database and pass back a passed/failed flag.
I have tried all the examples here
Make angular.forEach wait for promise after going to next object and still get the same results.
The method is actually called, but the results are not posted to the db. I have tested the parser.parseResults on an individual file and it successfully posted to the db, so it has to have something to do with the promise not resolving correctly.
Is it not possible to do something like this in the jasmine/protractor framework? Or do I have something wrong in the code?
I have included the code for my latest attempt.
Thank You
Christine
matches.reduce(function (p, val) {
console.log('val', val);
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
console.log('finalResult = ', finalResult);
}, function (err) {
console.log('error in reduce',err);
});
parser.parseResults code
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
try {
if (err != null) {
console.log('error reading file',err);
reject(err);
}
console.log('obj - ',obj);
var results = [];
var Passed = 0;
var Message = '';
var Stack = '';
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
if (specs.length > 0) {
for (var i = 0; i < specs.length; i++) {
var assert = specs[i];
var tcR = new RegExp(/TC[\d]+/);
var tc = assert.description.match(tcR);
if (!assert.failedExpectations.length) {
Passed = 1;
}
else {
assert.failedExpectations.forEach((expectation) => {
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
})
Passed = 0;
}
if (tc != null) {
utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild,
'P', Message, Stack, 0, moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss'), '')
.then(function (resp) {
resolve(Passed);
}, (err) => {
console.log('Posting to Database failed ', err);
reject(err);
});
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
reject(err);
}
}
}
}
}
catch (err) {
console.log('rejecting opening file');
reject(err);
}
});
})
}

If there is not exactly one suite in the obj, with exactly one spec, then your promise is either resolved not at all or multiple times.
Avoid wrapping too many things in the new Promise constructor - always promisify on the smallest possible level, and use promise chaining afterwards.
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function(obj) {
console.log('obj - ',obj);
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length-1];
Passed = 0;
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.catch(err => {
console.log('Posting to Database failed ', err);
throw err;
}));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results);
});
};

Find and save Mongoose for loop

I have an array of items and that need to be found and saved to MongoDB. Find from one model and get data from it and save via another model. assets is the array and count is the length of the array.
Problem - Do I need to check out all records that are saved to DB or not?
I need to get JSON response after complete pushing all records to the database.
router.route('/move_qa')
.post(function (req, res) {
console.log('/move_qa');
console.log("*assets" + req.body.assets);
var assets = req.body.assets;
var count = req.body.count;
for (var i = 0; i < count; i++) {
var aBarcode = assets[i];
console.log("$" + aBarcode);
searchAndSave(aBarcode, function (resulttt) {
console.log("#"+resulttt);
});
}
res.json({"result":true});
});
function searchAndSave(assetBarcode, callb) {
pallet.findOne({assets: assetBarcode}, function (err, count) {
if (err) {
console.log("1 /pallet_details");
console.log(err)
} else {
console.log("2 /pallet_details");
if (count == null) {
console.log("3/pallet_details");
} else {
var pbarcode = count.pBarcode;
var date = new Date();
var status = "NOT_COMPLETE";
var newMoveToQA = new movetoqa({
keg_barcode: assetBarcode,
relevant_pbarcode: pbarcode,
move_date: date,
status: status
});
newMoveToQA.save(function (err) {
console.log("####******");
if (err) {
console.log("4");
} else {
console.log("5");
callb(true);
}
});
}
}
});
}

How to set object array back to iterated collection from nested find() in MongoDB?

I develop application using AngularJS , NodeJS and MongoDB. I'd like to load Product with classified by ProductCategoryCode sending from
AngualrJS to NodeJS. First, I need to find Products by ProductCategoryCode and then iterate for each product to find Uoms by UomCode and ContainUomCode
which each product should has 2 uoms. How can I set uom object docUom back to product document doc[i] and update to product collection doc?
For following code line
doc[i].Uom = docUom;
The system throw error cannot set property 'Uom' of undefined.
Here is product.js snippet code.
router.get("/LoadProductByProductCategoryCode/:productCategoryCode", function (req, res) {
console.log('user.js -> /users ');
var productCategoryCode = req.params.productCategoryCode;
console.log(productCategoryCode );
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://localhost:27017/NodeDB', function (err, db) {
if (err) throw err;
var query = { ProductCategoryCode : productCategoryCode}
var new_product = [];
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
if (doc) {
console.log("Found Product..."+doc.length);
　 for (var i = 0; i < doc.length; i++) {
console.log(doc[i].ProductCode + " each document " + doc[i].UomCode + " " + doc[i].ContainUomCode);
var qUom = {
$or: [ { UomCode: doc[i].UomCode}, { UomCode: doc[i].ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
doc[i].Uom = docUom;
console.dir(product);
}
});
}
res.json(doc);
} else {
console.log('something happen');
}
}); //End
}); // MongoClient
var findProduct = function (db, query, callback) {
db.collection('Product').find(query).toArray(function (err, doc) {
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
var findUom = function(db, queryUom, callback) {
db.collection('Uom').find(queryUom).toArray(function (err, doc) {
// db.close();
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
});
Any idea? THANKS

Because of the asynchronous nature of the Node.js MongoDB driver, both the findProduct() and findUom() methods start, but don't necessarily complete by the time you reach res.json(doc) meaning doc will still be empty. You are expecting this to work in a linear fashion, but node works differently.
Instead, you should send your response back once all asynchronous calls complete meaning you could try something like:
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
var processedProduct = function (item) {
console.log(item.ProductCode + " each document " + item.UomCode + " " + item.ContainUomCode);
var qUom = {
$or: [ { UomCode: item.UomCode}, { UomCode: item.ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
item.Uom = docUom;
console.dir(product);
return item;
}
});
} 　
if (doc) {
var productsToFind = doc.length;
var products = [];
console.log("Found Products..." + productsToFind);
for (var i = 0; i < doc.length; i++) {
product = doc[i];
product = processedProduct(product);
products.push(product);
productsToFind -= 1;
if(productsToFind === 0){
res.json(products);
}
}
} else {
console.log('something happen');
}
}); //End
I could explain better about asynchronous calls and callbacks as this topic is a bit broad but from the above you can get the idea that I have used a counter productsToFind for all of the inner async calls that once each findUom() call completes this counter decrements and once it reaches 0 it means that all callbacks have fired.