We Keep Coding

CosmosDb Stored procedure There Is An error In My Javascript

There is an error in my Javascript but I just can't understand why. In the code below when the function
function replaceDoc(document,callback)
is called the document is null. I have checked that there are documents in the feed from
function tryQuery(callback) {...
and there is some kind of error. The procedure runs fine and response body shows this. Where is my mistake please?
function bulkChangeJobNumber(docs) {
var collection = getContext().getCollection();
var collectionLink = collection.getSelfLink();
var response = getContext().getResponse();
var responseBody = {
docsRetrieved: 0,
msg:'',
msg2:''
}
var docs;
var count = 0;
//get the docs
tryQuery(callback);
function tryQuery(callback) {
var isAccepted = collection.queryDocuments(
collection.getSelfLink(),
"SELECT * FROM c WHERE c.ty = 'job' AND NOT IS_DEFINED(c.jobNumber)", function (err, feed, options) {
if (err) throw err;
// Check the feed and if empty, set the body to 'no docs found',
// else take 1st element from feed
docs = feed;
responseBody.docsRetrieved = docs.length;
responseBody.msg = options;
responseBody.msg2 = docs[0];
if (feed.length == 0) {
response.setBody("No Docs");
}
else{
replaceDoc(docs[count],callback);
}
});
if (!isAccepted) {
response.setBody("The query was not accepted!");
return;
}
}
function replaceDoc(document,callback){
if(document == null){
responseBody.msg = "document is null";
response.setBody(responseBody);
return;
}
document.jobNumber = document.jobId;
var accept = collection.replaceDocument(document._self,document, callback);
if(!accept){
response.setBody(`Processed: ${count} docs`);
}
}
function callback(err,doc,options){
if (err) throw err;
count++;
if(count > docs.length){
response.setBody(`processed all ${count} docs`)
}else{
replaceDoc(docs[count],callback);
}
}
}

How can I return a value from a function after closing the Oracle DB connection inside nodejs program

Here is what I am trying. In last I am returning the account number so that I can use it in another function but it seems like the value should be returned before closing the connection, or by any different approach. Can you please help me to identify what can be changed here so that I can utilize the returned value.
PS: It's a Nodejs program, using Oracle DB connection.
function getShipmentData(shipment){
var defer = q.defer();
var tnt_account_number;
var account_number;
var dhl_account_number;
var place_to_bil;
var freight_terms_shp;
var place_to_repair;
var shipment_id;
oracledb.outFormat = oracledb.OBJECT;
console.log("before getConnection")
oracledb.getConnection(
{
user : settings.ora_db_user,
password : settings.ora_db_password,
connectString : settings.ora_db_host+':'+settings.ora_db_port+'/'+settings.ora_db_service
},
function(err, connection)
{
console.log("After getConnection")
if (err) {
writeDebug({'error':err.message});
defer.reject(err);
}
const shipment_sql = `select r.PLACE_ID_TO_BILL as Place_TO_BILL, s.freight_terms_shp as Freight_Terms,
r.PLACE_ID_SHIP_TO as Place_To_Repair, s.shipment_id as Shipment_ID
from shipment s inner join request r on s.request_id = r.request_id
where s.shipment_id = :SHIPMENT_ID`
var bindVars = {
SHIPMENT_ID: shipment
};
connection.execute(
shipment_sql,
bindVars,
function(err, result)
{
var row;
if (err) {
console.log("Error:", err)
connection.close();
defer.reject(err);
}
var results = result.rows
Object.keys(results).forEach(function(key){
row = results[key];
place_to_bil = row.PLACE_TO_BILL
freight_terms_shp = row.FREIGHT_TERMS
place_to_repair = row.PLACE_TO_REPAIR
shipment_id = row.SHIPMENT_ID
console.log("place_to_bil from Data:",place_to_bil);
console.log("freight_terms from Data:",freight_terms_shp);
console.log("place_to_repair from Data:",place_to_repair);
console.log("shipment_id from Data:",shipment_id);
})
function getAccountNumber(){
if(freight_terms_shp == 'ABHTC') {
if(place_to_repair == 'MYTPIC-401' && place_to_bil == 'GTHBYR-001')
tnt_account_number = '8934734';
else if (place_to_repair == 'CNHBNT-40000' && place_to_bil == 'CBCINF-001') {
tnt_account_number = '3489523';
}
else tnt_account_number = '5438648';
}
return tnt_account_number;
}
account_number = getAccountNumber();
console.log("TNT Account Number before closing connection:", account_number)
connection.close();
console.log("TNT Account Number after closing connection:", account_number)
defer.resolve(result.rows);
});
});
console.log("Account Number getting returned?", account_number);
return account_number;
}

Using Async With waterfall and Recursion in nodejs

I've created a script to migrate data from Dynamo to a Mysql DB.
First I was not using Async, but I started getting bottlenecks on the sql side, so I decided to "throttle" the dymano part using the async lib.
The problem: I have a recursion in the middle of the path, as long as dynamo has data I have to continue the process (ultra simple ETL), but I don't know how to perform the recursion inside the waterfall.
My code :
function main() {
async.waterfall([getMaxTimestamp, scanDynamoDB, printout, saveToMySQL], function(err, result) {
if(err) console.log(err)
console.log(result)
});
}
function getMaxTimestamp(callback) {
console.time("max query");
connection.query("SELECT MAX(created_at) as start_date from Tracking;", function(err, data) {
console.timeEnd("max query");
callback(err, data);
})
}
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
callback(err, data);
// if (!err) {
// if (data != undefined && data.Count > 0) {
// printout(data.Items) // Print out the subset of results.
// if (data.LastEvaluatedKey) { // Result is incomplete; there is more to come.
// query.ExclusiveStartKey = data.LastEvaluatedKey;
// scanDynamoDB(query);
// }
// } else {
// console.log('No fresh data found on Dynamo')
// } else console.dir(err);
});
};
function assembleSql() {
insertSql = "insert into Tracking (";
for (var i = 0; i < headers.length; i++) {
insertSql += headers[i];
if (i < headers.length - 1)
insertSql += ",";
}
insertSql += ") values ?;"
previousInsertSql = insertSql;
}
function saveToMySQL(items, callback) {
assembleSql();
//connection.connect();
console.time("insert sql")
connection.query(insertSql, [items], function(err, result) {
console.timeEnd("insert sql")
if (err){
callback(err, null)
return;
}
totalInserts += result.affectedRows;
callback(err, totalInserts)
//connection.end();
})
}
function printout(items, callback) {
var headersMap = {};
var values;
var header;
var value;
var out = [];
if (headers.length == 0) {
if (items.length > 0) {
for (var i = 0; i < items.length; i++) {
for (var key in items[i]) {
headersMap[key] = true;
}
}
}
for (var key in headersMap) {
headers.push(key);
}
}
for (index in items) {
values = [];
for (i = 0; i < headers.length; i++) {
value = "";
header = headers[i];
// Loop through the header rows, adding values if they exist
if (items[index].hasOwnProperty(header)) {
if (items[index][header].N) {
value = items[index][header].N;
} else if (items[index][header].S) {
value = items[index][header].S;
} else if (items[index][header].SS) {
value = items[index][header].SS.toString();
} else if (items[index][header].NS) {
value = items[index][header].NS.toString();
} else if (items[index][header].B) {
value = items[index][header].B.toString('base64');
} else if (items[index][header].M) {
value = JSON.stringify(items[index][header].M);
} else if (items[index][header].L) {
value = JSON.stringify(items[index][header].L);
} else if (items[index][header].BOOL !== undefined) {
value = items[index][header].BOOL.toString();
}
}
values.push(value)
}
out.push(values)
}
callback(null, out);
}
main();
The commented part is where the recursion happens, but I don't know where to place this inside my flow !
Any help would be appreciated !

Just don't call callback function inside scanDynamoDB while fetching data. You can implement additional function and call it recursive while errors is not appears, like below
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
var result = []; // for accumulate data of each query
function readNext(err, data) {
if (err)
return callback(err);
if (!data || !data.Count)
return callback(null, result);
// add data to result
dynamoDB.scan(query, readNext);
}
dynamoDB.scan(query, readNext);
};

Actually I was able to figure it out by myself.
async.whilst(function() { return canInsert}, function (callback){
scanDynamoDB(query, callback)
}, function(err, res) {}
function scanDynamoDB(data, callback) {
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
if (!err) {
if (data != undefined && data.Count > 0) {
canInsert = data.LastEvaluatedKey;
if (data.LastEvaluatedKey) // Result is incomplete; there is more to come.
query.ExclusiveStartKey = data.LastEvaluatedKey;
}
} else console.dir(err);
});
};
I could have done it just with a while(canInsert). Anyway, I avoided recursion and memory usage is way way lower.

How to set object array back to iterated collection from nested find() in MongoDB?

I develop application using AngularJS , NodeJS and MongoDB. I'd like to load Product with classified by ProductCategoryCode sending from
AngualrJS to NodeJS. First, I need to find Products by ProductCategoryCode and then iterate for each product to find Uoms by UomCode and ContainUomCode
which each product should has 2 uoms. How can I set uom object docUom back to product document doc[i] and update to product collection doc?
For following code line
doc[i].Uom = docUom;
The system throw error cannot set property 'Uom' of undefined.
Here is product.js snippet code.
router.get("/LoadProductByProductCategoryCode/:productCategoryCode", function (req, res) {
console.log('user.js -> /users ');
var productCategoryCode = req.params.productCategoryCode;
console.log(productCategoryCode );
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://localhost:27017/NodeDB', function (err, db) {
if (err) throw err;
var query = { ProductCategoryCode : productCategoryCode}
var new_product = [];
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
if (doc) {
console.log("Found Product..."+doc.length);
　 for (var i = 0; i < doc.length; i++) {
console.log(doc[i].ProductCode + " each document " + doc[i].UomCode + " " + doc[i].ContainUomCode);
var qUom = {
$or: [ { UomCode: doc[i].UomCode}, { UomCode: doc[i].ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
doc[i].Uom = docUom;
console.dir(product);
}
});
}
res.json(doc);
} else {
console.log('something happen');
}
}); //End
}); // MongoClient
var findProduct = function (db, query, callback) {
db.collection('Product').find(query).toArray(function (err, doc) {
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
var findUom = function(db, queryUom, callback) {
db.collection('Uom').find(queryUom).toArray(function (err, doc) {
// db.close();
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
});
Any idea? THANKS

Because of the asynchronous nature of the Node.js MongoDB driver, both the findProduct() and findUom() methods start, but don't necessarily complete by the time you reach res.json(doc) meaning doc will still be empty. You are expecting this to work in a linear fashion, but node works differently.
Instead, you should send your response back once all asynchronous calls complete meaning you could try something like:
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
var processedProduct = function (item) {
console.log(item.ProductCode + " each document " + item.UomCode + " " + item.ContainUomCode);
var qUom = {
$or: [ { UomCode: item.UomCode}, { UomCode: item.ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
item.Uom = docUom;
console.dir(product);
return item;
}
});
} 　
if (doc) {
var productsToFind = doc.length;
var products = [];
console.log("Found Products..." + productsToFind);
for (var i = 0; i < doc.length; i++) {
product = doc[i];
product = processedProduct(product);
products.push(product);
productsToFind -= 1;
if(productsToFind === 0){
res.json(products);
}
}
} else {
console.log('something happen');
}
}); //End
I could explain better about asynchronous calls and callbacks as this topic is a bit broad but from the above you can get the idea that I have used a counter productsToFind for all of the inner async calls that once each findUom() call completes this counter decrements and once it reaches 0 it means that all callbacks have fired.

Extract common code from sails.js / mongodb

I'm current trying to use sails.js with mongodb, I need some custom mapReduce function to group data.
Now I could achieve what I want by using waterline's native function, but have some questions.
These function has only small variation actually, but I found myself keep repeating codes like the following one:
function getSomeData() {
// First-query
Log.native(function(err, logCollection) {
var mapFunction = function() {
function dateFormatter(date) {
return date.getFullYear() + "-" + (date.getMonth() + 1)
}
//! Generate Grouping Key
emit(dateFormatter(this.emb_date), this.bad_qty)
}
var reduceFunction = function (key, values) {
return Array.sum(values);
}
var outputControl = {
out: {inline: 1},
//! Filters
query: {order_type: product}
}
logCollection.mapReduce(mapFunction, reduceFunction, outputControl, function (err, result) {
if (err) {
callback(err);
return;
}
var resultSet = [];
//! post-processing
for (var i = 0; i < result.length; i++) {
//.....
}
callback(err, resultSet);
});
});
}
Second-query:
function getAnotherData() {
Log.native(function(err, logCollection) {
var mapFunction = function() {
//! Generate Grouping Key
emit(dateFormatter(this.product), this.bad_qty)
}
var reduceFunction = function (key, values) {
return Array.sum(values);
}
var outputControl = {
out: {inline: 1},
//! Filters
query: {order_type: product}
}
logCollection.mapReduce(mapFunction, reduceFunction, outputControl, function (err, result) {
if (err) {
callback(err);
return;
}
var resultSet = [];
//! post-processing
for (var i = 0; i < result.length; i++) {
//......
}
callback(err, resultSet);
});
});
}
As you can see, these two snippet shares lots of common code, only has difference in three place (Generate grouping key, filters, post-process).
So I would really like to extract the common part to make my code cleaner, but have no success.
I first try to make dateFromatter is provided by a callback instead of hard-coding like the following:
function dateFormatter(data) {
return data.emb_date.getFullYear() + "-" + (data.emb_date.getMonth() + 1)
}
function getSomeData(groupingKey) {
// First-query
Log.native(function(err, logCollection) {
var mapFunction = function() {
//! Generate Grouping Key
emit(groupingKey(this.emb_date), this.bad_qty)
}
var reduceFunction = function (key, values) {
return Array.sum(values);
}
var outputControl = {
out: {inline: 1},
//! Filters
query: {order_type: product}
}
logCollection.mapReduce(mapFunction, reduceFunction, outputControl, function (err, result) {
if (err) {
callback(err);
return;
}
var resultSet = [];
//! post-processing
for (var i = 0; i < result.length; i++) {
//.....
}
callback(err, resultSet);
});
});
}
But without any luck, I keep getting error like the following one:
MongoError: exception: ReferenceError: groupingKey is not defined near 'emit(groupingKey(this), this.bad_qty' (line 3)
at Object.toError (/home/brianhsu/zh800/dashboard/node_modules/sails-mongo/node_modules/mongodb/lib/mongodb/utils.js:114:11)
What should I do if I would like to reduce those duplicate part of code?

Finally I found that I need pass the option called 'scope' to mongodb, I come up with the following solution which works quite well.
exports.defineOn = function(options) {
var model = options.model
var groupingFunction = options.groupingFunction
var mongoFilters = options.mongoFilters
var customFilter = options.customFilter
var converter = options.converter
var sorting = options.sorting
return function(callback) {
model.native(function(err, collection) {
var mapFunction = function() { emit(groupingFunction(this), this.bad_qty) }
var reduceFunction = function(key, values) { return Array.sum(values); }
var mapReduceOptions = {
out: {inline: 1},
query: mongoFilters,
scope: {
groupingFunction: groupingFunction,
mongoFilters: mongoFilters,
customFilter: customFilter,
converter: converter
}
}
var processCallback = function (err, result) {
if (err) {
callback(err);
return;
}
if (sorting) {
result.sort(sorting);
}
var resultSet = [];
for (var i = 0; i < result.length; i++) {
if (customFilter && customFilter(result[i])) {
resultSet.push(converter(result[i]));
} else if (!customFilter) {
resultSet.push(converter(result[i]));
}
}
callback(err, resultSet);
}
collection.mapReduce(mapFunction, reduceFunction, mapReduceOptions, processCallback);
});
}
}
Usage:
function machineDetail (year, month, date, machine, callback) {
var startDate = new Date(+year, +(month-1), +date);
var endDate = new Date(+year, +(month-1), (+date) + 1);
var mapReducer = MapReducer.defineOn({
model: Log,
groupingFunction: function(data) {
return {date: data.emb_date, error: data.defact_id};
},
mongoFilters: {
mach_id: machine,
emb_date: {$gte: startDate, $lt: endDate}
},
converter: function (data) {
return {
name: data._id,
value: data.value,
};
}
});
mapReducer(callback);
}