We Keep Coding

How can I return a value from a function after closing the Oracle DB connection inside nodejs program

Here is what I am trying. In last I am returning the account number so that I can use it in another function but it seems like the value should be returned before closing the connection, or by any different approach. Can you please help me to identify what can be changed here so that I can utilize the returned value.
PS: It's a Nodejs program, using Oracle DB connection.
function getShipmentData(shipment){
var defer = q.defer();
var tnt_account_number;
var account_number;
var dhl_account_number;
var place_to_bil;
var freight_terms_shp;
var place_to_repair;
var shipment_id;
oracledb.outFormat = oracledb.OBJECT;
console.log("before getConnection")
oracledb.getConnection(
{
user : settings.ora_db_user,
password : settings.ora_db_password,
connectString : settings.ora_db_host+':'+settings.ora_db_port+'/'+settings.ora_db_service
},
function(err, connection)
{
console.log("After getConnection")
if (err) {
writeDebug({'error':err.message});
defer.reject(err);
}
const shipment_sql = `select r.PLACE_ID_TO_BILL as Place_TO_BILL, s.freight_terms_shp as Freight_Terms,
r.PLACE_ID_SHIP_TO as Place_To_Repair, s.shipment_id as Shipment_ID
from shipment s inner join request r on s.request_id = r.request_id
where s.shipment_id = :SHIPMENT_ID`
var bindVars = {
SHIPMENT_ID: shipment
};
connection.execute(
shipment_sql,
bindVars,
function(err, result)
{
var row;
if (err) {
console.log("Error:", err)
connection.close();
defer.reject(err);
}
var results = result.rows
Object.keys(results).forEach(function(key){
row = results[key];
place_to_bil = row.PLACE_TO_BILL
freight_terms_shp = row.FREIGHT_TERMS
place_to_repair = row.PLACE_TO_REPAIR
shipment_id = row.SHIPMENT_ID
console.log("place_to_bil from Data:",place_to_bil);
console.log("freight_terms from Data:",freight_terms_shp);
console.log("place_to_repair from Data:",place_to_repair);
console.log("shipment_id from Data:",shipment_id);
})
function getAccountNumber(){
if(freight_terms_shp == 'ABHTC') {
if(place_to_repair == 'MYTPIC-401' && place_to_bil == 'GTHBYR-001')
tnt_account_number = '8934734';
else if (place_to_repair == 'CNHBNT-40000' && place_to_bil == 'CBCINF-001') {
tnt_account_number = '3489523';
}
else tnt_account_number = '5438648';
}
return tnt_account_number;
}
account_number = getAccountNumber();
console.log("TNT Account Number before closing connection:", account_number)
connection.close();
console.log("TNT Account Number after closing connection:", account_number)
defer.resolve(result.rows);
});
});
console.log("Account Number getting returned?", account_number);
return account_number;
}

aws sqs: queuing in a loop

In code below, I have a main loop to get each 'assunto' in records(jsonAssuntos). After that, I want to send the 'assunto' to your appropriate queue. The problem is about idSequence. The queue receive the wrong value, sometimes the last number generated.
module.exports.consumeMainQueue = async (event, context, callback) => {
var AssuntoQueueAURL = 'http://localhost:9324/queue/Queue';
for (let recordIndex in event.Records){
var record = event.Records[recordIndex];
var jsonAssuntos = JSON.parse(record.body);
var idMessage = record.messageId;
var idSequence = 0; // initialize *********************
for (var assuntoIndex in jsonAssuntos.dat){
idSequence++; // increment ********************
var assunto = jsonAssuntos.dat[assuntoIndex];
if(assunto.typ == "typ1" || assunto.typ == "typ2" || assunto.typ == "typ3"){
var infoAssunto = {
ParentID: idMessage,
Asssunto: assunto
IdSequence : idSequence; // send to queue **************
};
var paramsAssuntoQueue = {
MessageBody: JSON.stringify(infoAssunto),
QueueUrl: assuntoTyp
};
queue.sendMessage(paramsAssuntoQueue, function(err, data) {
if (err) {
console.log("Error", err);
}
else{
console.log("OK");
//End Log
};
});
}
else{
}
}
};

How to callback an error message when searching a table is unsuccessful in DynamoDB

I'm currently using AWS Lambda JavaScript code to try and search a DynamoDB table this is then implemented into an Amazon Alexa application, but that isn't really important for what I'm asking. Here is the code I'm struggling with:
function readDynamoItem(params2, callback) {
var AWS = require('aws-sdk');
AWS.config.update({region: AWSregion});
var dynamodb = new AWS.DynamoDB();
console.log('reading item from DynamoDB table');
dynamodb.scan(params2, function (err, data){
if (err) {
callback("error");
//console.log(err, err.stack); // an error occurred
}
else{
callback(data);
}
});
}
So when an error occurs I want it to callback the message "error" and then use it here:
const params2 = {
TableName: 'Fixtures',
FilterExpression: 'team1 = :value',
ExpressionAttributeValues: {':value': {"S": MyQuestion.toLowerCase()}}
};
readDynamoItem(params2, myResult=>{
say = myResult;
this.response.speak(say).listen('try again');
this.emit(':responseReady');
});
All I'm getting at the moment is this response when I test, I think due to err just ending the program instead of calling the error back to use in the implementation:
Response:
{
"errorMessage": "RequestId: 0f586880-2ddb-11e8-bdf7-07b4c224b25d Process exited before completing request"
}
Any help would be greatly appreciated.
Here's the full code for my project for further reference:
const AWSregion = 'eu-west-1';
const Alexa = require('alexa-sdk');
const AWS = require('aws-sdk');
AWS.config.update({
region: AWSregion
});
exports.handler = function(event, context, callback) {
var alexa = Alexa.handler(event, context);
// alexa.appId = 'amzn1.echo-sdk-ams.app.1234';
// alexa.dynamoDBTableName = 'YourTableName'; // creates new table for session.attributes
alexa.registerHandlers(handlers);
alexa.execute();
};
const handlers = {
'LaunchRequest': function () {
this.response.speak('welcome to magic answers. ask me a yes or no question.').listen('try again');
this.emit(':responseReady');
},
'MyIntent': function () {
var MyQuestion = this.event.request.intent.slots.MyQuestion.value;
console.log('MyQuestion : ' + MyQuestion);
const params2 = {
TableName: 'Fixtures',
FilterExpression: 'team1 = :value',
ExpressionAttributeValues: {':value': {"S": MyQuestion.toLowerCase()}}
};
const params3 = {
TableName: 'Fixtures',
FilterExpression: 'team2 = :value',
ExpressionAttributeValues: {':value': {"S": MyQuestion.toLowerCase()}}
};
readDynamoItem(params2, myResult=>{
var say = MyQuestion;
//if nothing is found when scanning for team1, scan team2
if (myResult == "error"){
readDynamoItem(params3, myResult2=>{
say = myResult2;
say = 'The top scorer for ' + MyQuestion + ' is ' + myResult2;
this.response.speak(say).listen('try again');
this.emit(':responseReady');
});
}
else{
say = myResult;
say = 'The top scorer for ' + MyQuestion + ' is ' + myResult;
this.response.speak(say).listen('try again');
this.emit(':responseReady');
}
});
},
'AMAZON.HelpIntent': function () {
this.response.speak('ask me a yes or no question.').listen('try again');
this.emit(':responseReady');
},
'AMAZON.CancelIntent': function () {
this.response.speak('Goodbye!');
this.emit(':responseReady');
},
'AMAZON.StopIntent': function () {
this.response.speak('Goodbye!');
this.emit(':responseReady');
}
};
// END of Intent Handlers {} ========================================================================================
// Helper Function =================================================================================================
//reading the Fixtures table
function readDynamoItem(params2, callback) {
var AWS = require('aws-sdk');
AWS.config.update({region: AWSregion});
var dynamodb = new AWS.DynamoDB();
var team1;
var team2;
console.log('reading item from DynamoDB table');
dynamodb.scan(params2, function (err, data){
if (err) {
callback("error");
//callback("error");
//console.log(err, err.stack); // an error occurred
}
else{
console.log(data); // successful response
team1 = jsonToString(data.Items[0].team1);
team2 = jsonToString(data.Items[0].team2);
var t1goals = jsonToString(data.Items[0].t1goals);
var t2goals = jsonToString(data.Items[0].t2goals);
t1goals = parseInt(t1goals);
t2goals = parseInt(t2goals);
var search;
var chosenValue = Math.random() < 0.5 ? team1 : team2;
// if goals are equal in a match then it is random which team will score next
if(t1goals == t2goals){
search = chosenValue;
}
//if a team has 1 goal more than the other then it is a 3rd more likely they will score next
else if(t1goals > t2goals && t1goals == 1){
if(randomInt(1, 3) == 1){
search = team2;
}
else{
search = team1;
}
}
else if(t2goals > t1goals && t2goals == 1){
if(randomInt(1, 3) == 1){
search = team1;
}
else{
search = team2;
}
}
//if a team has more than 1 goal more than the other then it is a 5th more likely they will score next
else if(t1goals > t2goals && t1goals > 1){
if(randomInt(1, 5) == 1){
search = team2;
}
else{
search = team1;
}
}
else if(t2goals > t1goals && t2goals > 1){
if(randomInt(1, 5) == 1){
search = team1;
}
else{
search = team2;
}
}
var params = {
TableName: 'yesno',
FilterExpression: 'team = :value',
ExpressionAttributeValues: {':value': {"S": search}}
};
readDynamoFixtures(params, myResult=>{
callback(myResult);
});
}
});
}
//read player details from the the yesno table
function readDynamoFixtures(params, callback) {
var goals = new Array();
var playing = new Array();
var messages = new Array();
var most = 0;
var mostMessage;
var dynamodb = new AWS.DynamoDB();
dynamodb.scan(params, function (err, data) {
if (err) console.log(err, err.stack); // an error occurred
else{
for(var i = 0; i <= (data.Count - 1); i++){
console.log(data); // successful response
var temp = jsonToString(data.Items[i].playername);
messages[i] = temp;
temp = jsonToString(data.Items[i].goals);
temp = parseInt(temp);
goals[i] = temp;
temp = jsonToString(data.Items[i].playing);
playing[i] = temp;
//compare each players goals
if (goals[i] > most && playing[i] == "true"){
most = goals[i];
mostMessage = messages[i];
}
}
}
callback(mostMessage);
});
}
//convert database items from json format to string
function jsonToString(str){
str = JSON.stringify(str);
str = str.replace('{\"S\":\"', '');
str = str.replace('\"}', '');
return str;
}
//get a random int between min and max
function randomInt(min,max)
{
return Math.floor(Math.random()*(max-min+1)+min);
}
Edit:
I have tried testing this code with .query instead of .scan and the error callback works perfectly which is strange but obviously for this implementation I need to use .scan

When you get the "Process exited" response from the Lambda it is helpful to log heavily to see where the Lambda is getting stuck and then check the Cloudwatch Logs to get to the detail.
Then you can pinpoint the exception and focus on it. At least for me, the root cause was many times unexpected as Lambdas force a different way of thinking.

Find and save Mongoose for loop

I have an array of items and that need to be found and saved to MongoDB. Find from one model and get data from it and save via another model. assets is the array and count is the length of the array.
Problem - Do I need to check out all records that are saved to DB or not?
I need to get JSON response after complete pushing all records to the database.
router.route('/move_qa')
.post(function (req, res) {
console.log('/move_qa');
console.log("*assets" + req.body.assets);
var assets = req.body.assets;
var count = req.body.count;
for (var i = 0; i < count; i++) {
var aBarcode = assets[i];
console.log("$" + aBarcode);
searchAndSave(aBarcode, function (resulttt) {
console.log("#"+resulttt);
});
}
res.json({"result":true});
});
function searchAndSave(assetBarcode, callb) {
pallet.findOne({assets: assetBarcode}, function (err, count) {
if (err) {
console.log("1 /pallet_details");
console.log(err)
} else {
console.log("2 /pallet_details");
if (count == null) {
console.log("3/pallet_details");
} else {
var pbarcode = count.pBarcode;
var date = new Date();
var status = "NOT_COMPLETE";
var newMoveToQA = new movetoqa({
keg_barcode: assetBarcode,
relevant_pbarcode: pbarcode,
move_date: date,
status: status
});
newMoveToQA.save(function (err) {
console.log("####******");
if (err) {
console.log("4");
} else {
console.log("5");
callb(true);
}
});
}
}
});
}

Using Async With waterfall and Recursion in nodejs

I've created a script to migrate data from Dynamo to a Mysql DB.
First I was not using Async, but I started getting bottlenecks on the sql side, so I decided to "throttle" the dymano part using the async lib.
The problem: I have a recursion in the middle of the path, as long as dynamo has data I have to continue the process (ultra simple ETL), but I don't know how to perform the recursion inside the waterfall.
My code :
function main() {
async.waterfall([getMaxTimestamp, scanDynamoDB, printout, saveToMySQL], function(err, result) {
if(err) console.log(err)
console.log(result)
});
}
function getMaxTimestamp(callback) {
console.time("max query");
connection.query("SELECT MAX(created_at) as start_date from Tracking;", function(err, data) {
console.timeEnd("max query");
callback(err, data);
})
}
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
callback(err, data);
// if (!err) {
// if (data != undefined && data.Count > 0) {
// printout(data.Items) // Print out the subset of results.
// if (data.LastEvaluatedKey) { // Result is incomplete; there is more to come.
// query.ExclusiveStartKey = data.LastEvaluatedKey;
// scanDynamoDB(query);
// }
// } else {
// console.log('No fresh data found on Dynamo')
// } else console.dir(err);
});
};
function assembleSql() {
insertSql = "insert into Tracking (";
for (var i = 0; i < headers.length; i++) {
insertSql += headers[i];
if (i < headers.length - 1)
insertSql += ",";
}
insertSql += ") values ?;"
previousInsertSql = insertSql;
}
function saveToMySQL(items, callback) {
assembleSql();
//connection.connect();
console.time("insert sql")
connection.query(insertSql, [items], function(err, result) {
console.timeEnd("insert sql")
if (err){
callback(err, null)
return;
}
totalInserts += result.affectedRows;
callback(err, totalInserts)
//connection.end();
})
}
function printout(items, callback) {
var headersMap = {};
var values;
var header;
var value;
var out = [];
if (headers.length == 0) {
if (items.length > 0) {
for (var i = 0; i < items.length; i++) {
for (var key in items[i]) {
headersMap[key] = true;
}
}
}
for (var key in headersMap) {
headers.push(key);
}
}
for (index in items) {
values = [];
for (i = 0; i < headers.length; i++) {
value = "";
header = headers[i];
// Loop through the header rows, adding values if they exist
if (items[index].hasOwnProperty(header)) {
if (items[index][header].N) {
value = items[index][header].N;
} else if (items[index][header].S) {
value = items[index][header].S;
} else if (items[index][header].SS) {
value = items[index][header].SS.toString();
} else if (items[index][header].NS) {
value = items[index][header].NS.toString();
} else if (items[index][header].B) {
value = items[index][header].B.toString('base64');
} else if (items[index][header].M) {
value = JSON.stringify(items[index][header].M);
} else if (items[index][header].L) {
value = JSON.stringify(items[index][header].L);
} else if (items[index][header].BOOL !== undefined) {
value = items[index][header].BOOL.toString();
}
}
values.push(value)
}
out.push(values)
}
callback(null, out);
}
main();
The commented part is where the recursion happens, but I don't know where to place this inside my flow !
Any help would be appreciated !

Just don't call callback function inside scanDynamoDB while fetching data. You can implement additional function and call it recursive while errors is not appears, like below
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
var result = []; // for accumulate data of each query
function readNext(err, data) {
if (err)
return callback(err);
if (!data || !data.Count)
return callback(null, result);
// add data to result
dynamoDB.scan(query, readNext);
}
dynamoDB.scan(query, readNext);
};

Actually I was able to figure it out by myself.
async.whilst(function() { return canInsert}, function (callback){
scanDynamoDB(query, callback)
}, function(err, res) {}
function scanDynamoDB(data, callback) {
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
if (!err) {
if (data != undefined && data.Count > 0) {
canInsert = data.LastEvaluatedKey;
if (data.LastEvaluatedKey) // Result is incomplete; there is more to come.
query.ExclusiveStartKey = data.LastEvaluatedKey;
}
} else console.dir(err);
});
};
I could have done it just with a while(canInsert). Anyway, I avoided recursion and memory usage is way way lower.