We Keep Coding

find specific returned loop data node.js

I have this function:
var arrayOfNumbers = []
var getTexts = new cronJob('45 * * * * *', function() {
var viewConformationEmails = "select * from clients";
ibmdb.open(ibmdbconn, function(err, conn) {
if (err) return console.log(err);
conn.query(viewConformationEmails, function(err, rows) {
if (err) {
console.log(err);
} else if (!err) {
console.log("Success")
}
for (var i = 0; i < rows.length; i++) {
// arrayOfNumbers.push(rows[i].NAME)
arrayOfNumbers.push(rows[i]["PHONE_NUMBER"])
var stringg = rows[i]["MINUTE"] + " " + rows[i]["HOUR"] + " * " + "* " + "*"
var textJob = new cronJob(stringg, function() {
client.messages.create({
to: arrayOfNumbers[i],
from: '12055578708',
body: 'Your training session just finished.'
}, function(err, data) {});
}, null, true);
}
conn.close(function() {
// console.log("closed the function /login");
});
});
});
}, null, true)
what it does is loop through my DB table clients, and then what I want it to do is in the loop grab the phone number, as well as the minute and hour in which the user needs a text message sent to them. However, what I need it to do is ONLY SEND THE MESSAGE TO THE PHONE_NUMBER that has the time slot at x time.
So if the db returns 5 people, but only 1 of those people have a time slot at 9am, then how do i only save that one person, or check for it.
This is a relativley general question to fixing returned loop data so I am just trying to figure out how to fix this. Any ideas?

NodeJS send response to client only after saving all object to DB

I have the code below, from a REST API, that inserts data in Mysql. I use Node and Express (this is, in fact, my 1st Node project, so please bear in mind I don't understand much about Node).
What I need is that response to client (browser, web application, Postman for testing or whatever access to the API) is returned only when the forEach loop and data insertion into DB terminates, so I get a JSON object with the list error messages, if any.
I've been hitting my head on the wall for half a day, and this is what I got so far.
var wait=require('wait.for');
var async = require('async');
var Promise = require('promise');
var Q = require('q');
var errmsg = [];
router.route('/subscriber').post((req, res, callback) => {
const data = req.body;
var subscriberCollection = data;
this.errmsg = [];
let asyncCall =
(async () => {
let rr = await new Promise (resolve => subscriberCollection.forEach(function (value, key){
var phoneNumber = value.phoneNumber;
var msg = "";
if (phoneNumber == ""){
msg = "ERROR","missing phoneNumber for subscriber index #" + key + ";Skipping";
console.log(msg);
errmsg[key] = msg
return;
}
var sql = "call insertSubscriber(?)";
console.log("INFO",`Inserting subscriber ${phoneNumber} index ${key}`);
connection.query(sql,[ phoneNumber ] ,function (err, data) {
if (err){
var msg = err.errno + " - " + err.sqlMessage;
console.log("ERROR" , msg);
errmsg[key] = msg;
}
});
}) //end forEach
); //end Promise
})();
asyncCall.then(console.log("ENDING!!") ); // THIS IS NOT WORKING
});
On the console, I get this:
INFO Inserting 916311145 for index 0
INFO Inserting 916311146 for index 1
ENDING!!
ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'
ERROR 1062 - Duplicate entry '916311146' for key 'phoneNumber_UNIQUE'
but what I need it to be is:
INFO Inserting 916311145 for index 0
INFO Inserting 916311146 for index 1
ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'
ERROR 1062 - Duplicate entry '916311146' for key 'phoneNumber_UNIQUE'
ENDING!!
Also, when all subscriber objects are saved on DB, I need to return a response to client, something like:
[{"key 0" : "ok"},{"key 1" : "ok"}, {"key 3": "ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'"}...]
and again, the response should only appear when all processing has finished.
How can I get this work?

Hmm try this:
var wait = require('wait.for');
var async = require('async');
var Promise = require('promise');
var Q = require('q');
router.route('/subscriber').post(async (req, res, callback) => {
const data = req.body;
var subscriberCollection = data;
const response = await Promise.all(
subscriberCollection.map(function (value, key) {
var phoneNumber = value.phoneNumber;
var msg = '';
const obj = {};
if (phoneNumber == '') {
msg = 'ERROR missing phoneNumber for subscriber index #' + key + ';Skipping';
console.log(msg);
obj[key] = msg;
Promise.resolve(obj);
return;
}
var sql = 'call insertSubscriber(?)';
console.log('INFO', `Inserting subscriber ${phoneNumber} index ${key}`);
return new Promise((resolve) => {
connection.query(sql, [phoneNumber], function (err, data) {
if (err) {
var msg = 'ERROR' + err.errno + ' - ' + err.sqlMessage;
console.log(msg);
obj[key] = msg;
resolve(obj);
return;
}
obj[key] = 'ok';
resolve(obj);
});
});
}) //end forEach
); //end Promise
console.log('ENDING!!');
res.send(response);
});

How do I read an CSV file in an EC2 instance folder as oppose to an S3 bucket in AWS?

I wrote a script with fast-csv that can read an excel file in amazon s3 and then take the data and store it in mySQL. I now have an ec2 instance set and created a folder titled "upload" and housed the CSV file in there. My question is how do I read a file in the ec2 instance as oppose to the s3 bucket? Below is current script using
const s3Stream = s3.getObject(params).createReadStream()
stream = require('fast-csv').parseStream(s3Stream, {
headers: true, skip_blanks: true
})
.on("data", data => {
dataArr.push(data);
})
stream = require('fast-csv').parseStream(s3Stream)
.on("data", data => {
dataArr2.push(data);
})
.on("end", () => {
let csvStream = csv
.parse({ ignoreEmpty: true })
.on('data', function (dataArr2) {
myData.push(dataArr2);
})
.on('end', function () {
dataArr2.shift();
console.log('dataArr2 ' + myData)
if (dataArr.length > 0) {
let columnsIn = dataArr[0];
for (let key in columnsIn) {
headerDatas.push(key)
}
for (let key in columnsIn) {
orginalHeaderDatas.push(key)
}
for (i = 0; i < headerDatas.length; i++) {
newData = headerDatas[i].split(' ').join('_');
correctHeaderFormat.push(newData)
}
// Assigns approriate Sql property to headers
let databaseId = headerDatas[0].split(' ').join('_');
let leaseDiscription = headerDatas[1].split(' ').join('_');
//Removes Headers that are not DEC propertys
headerDatas.shift();
headerDatas.shift();
let newdatabaseId = databaseId + ' int(25) NOT NULL'
let newleaseDiscription = leaseDiscription + ' varchar(255) NULL'
//adds property to the end of the remaining headers in array
for (i = 0; i < headerDatas.length; i++) {
newData = headerDatas[i].split(' ').join('_') + ' dec(25,2) NULL';
updatedData.push(newData)
}
//Adds headers that were removed from array and primary key to updated array
let key = 'PRIMARY KEY (Database_ID)'
headersWithProperties.push(updatedData)
headersWithProperties.unshift(newleaseDiscription)
headersWithProperties.unshift(newdatabaseId)
headersWithProperties.push(key)
} else {
console.log('No columns');
}
// open the connection
connection.connect((error) => {
if (error) {
console.error(error);
} else {
let createTable = 'CREATE TABLE `CD 1`' + '(' + headersWithProperties + ')'
let insertData = 'INSERT INTO `CD 1` ' + '(' + correctHeaderFormat + ') ' + 'VALUES ?'
//create table
connection.query(createTable, (error, response) => {
console.log("bottom" + connection.query)
console.log(error || response);
});
//insert data
connection.query(insertData, [dataArr2], (error, response) => {
console.log("bottom" + connection.query)
console.log(error || response);
});
}
});
});
stream.pipe(csvStream);
});

If I understand your question correctly you are trying to read the csv files that are local (in the same place as your node.js and mysql) instead of from a S3 bucket.
Don't use the s3 variable to get the csv file, rather you should read it locally.
fs.createReadStream('/path/to/upload/data.csv')
and then you can parse it into the mysql database using a similar method as before. It would look something like this

Only Update Specific Users Socket IO and Node JS

I am trying to learn node JS and am currently attempting to extend this article.
http://www.gianlucaguarini.com/blog/push-notification-server-streaming-on-a-mysql-database/
I am having major issue because I am getting multiple updates in the SQL query.
I only want to send one socket update.
This issue is in the transactions loop I get multiple socket updates.
I have been struggling with this for over a month and can't seem to figure it out on my own (or with google searches)
Can someone please tell me how I can make this work so I only get one socket update per client.
What I would like to happen is when there is a change in one of the transactions that the two parties (buyer and seller) are the only ones that get the socket update.
How can I make this work? It is so close to what I want it do but I can't get over this last challenge.
Please help.
Thank you in advance.
<html>
<head>
<title>GAT UPDATER</title>
<script src="/socket.io/socket.io.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">
<script src = "http://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js"></script>
<script type="text/javascript" src="http://board.gameassettrading.com/js/jquery.cookie.js"></script>
</head>
<body>
<script>
var nodeuserid;
function getUserId() {
var url = window.location.href;
var user_id = url.replace('http://heartbeat.gameassettrading.com:4000/id/', '');
return user_id;
}
user_id = getUserId();
$.cookie('useridcookie', user_id, { expires: 1 });
var useridcookie = $.cookie("useridcookie");
// Get Base Url for development between local and dev enviroment
function getBaseURL() {
var url = location.href; // entire url including querystring - also: window.location.href;
var baseURL = url.substring(0, url.indexOf('/', 14));
if (baseURL.indexOf('http://localhost') != -1) {
// Base Url for localhost
var url = location.href; // window.location.href;
var pathname = location.pathname; // window.location.pathname;
var index1 = url.indexOf(pathname);
var index2 = url.indexOf("/", index1 + 1);
var baseLocalUrl = url.substr(0, index2);
return baseLocalUrl + "/";
}
else {
// Root Url for domain name
return baseURL + "/";
}
}
// set the base_url variable
base_url = getBaseURL();
document.domain = "transactionsserver.com"
// create a new websocket
var socket = io.connect('http://heartbeat.transactionsserver.com:4000');
socket.on('connect',function() {
var data = {
url: window.location.href,
};
socket.emit('client-data', data);
});
// this is always open you have to filter out the data you want
socket.on('notification', function (data) {
if(data.hasOwnProperty("data")) {
if(data.data[0]['seller_id'] != ''){
$('#StatusUpdate', window.parent.document).text( data.data[0]['seller_id']+ ':' + data.data[0]['seller_status'] +':'+ data.data[0]['buyer_id']+':'+ data.data[0]['buyer_status']).click();
}
}
window.parent.checkData(data,user_id);
if(data.hasOwnProperty("changed_user_id")) {
$('#StatusUpdate', window.parent.document).text( data.changed_user_id+ ':' + data.changed_user_status +':'+ data.changed_user_id).click();
}
});
</script>
</body>
</html>
Server . js
var app = require("express")();
var path = require('path');
var mysql = require("mysql");
var http = require('http').Server(app);
var io = require("socket.io")(http);
var sockets = {};
var mysql = require('mysql'),
connectionsArray = [],
connection = mysql.createConnection({
multipleStatements: true,
host: 'localhost',
user: '*****',
password: '******',
database: 'transactionsdb',
port: 3306
}),
POLLING_INTERVAL = 1000,
pollingTimer;
// Add Redis For Comparing SQL Results againts Cache
var redis = require('redis');
var client = redis.createClient();
var express = require('express');
/* Creating POOL MySQL connection.*/
var pool = mysql.createPool({
connectionLimit: 100,
host: 'localhost',
user: '*****',
password: '*****',
database: 'transactionsdb',
debug: false
});
var count = 0;
var clients = [];
function processAllTransactions(sellsreply) {
pool.query('SELECT t.id,t.status,t.original_status, t.active, t.buyer_id, t.seller_id, t.seller_acked, t.seller_complete, t.buyer_complete, b.user_status as buyer_status,t.chat_ended, s.user_status as seller_status FROM transaction t LEFT JOIN sf_guard_user_profile s ON s.user_id = t.seller_id LEFT JOIN sf_guard_user_profile b ON b.user_id = t.buyer_id WHERE active = 1 LIMIT 1', [sellsreply], function (err, sells) {
if (sells != '') {
// attempt to stop the updates if it's not the the active transaction
client.get('active transaction id:'+sellsreply, function (err, active_transaction_id) {
passed_active_transaction_id = active_transaction_id;
});
// is there a trasnaction with status defined and transation id does not equal the active transaction id
if(sells[0].status !== undefined && sells[0].id !== passed_active_transaction_id )
{
client.get('active transaction:'+sellsreply, function (err, data1) {
if(JSON.stringify(sells) != data1){
client.set('active transaction id:'+sellsreply,sells[0]["id"]);
client.set('active transaction:'+sellsreply,JSON.stringify(sells));
console.log(JSON.stringify(sells));
updateSockets({
data: sells // pass the database result
});
}
});
}
}
});
}
// Method
function getUserInfo(user_id, callback) {
var query = connection.query('SELECT user_status from sf_guard_user_profile WHERE user_id = ' + connection.escape(user_id));
query.on('result', function (row) {
callback(null, row.user_status);
});
}
var updateSockets = function (data) {
// adding the time of the last update
data.time = new Date();
console.log('Pushing new data to the clients connected ( connections amount = %s ) - %s', connectionsArray.length , data.time);
// sending new data to all the sockets connected
connectionsArray.forEach(function (tmpSocket) {
console.log(tmpSocket);
tmpSocket.volatile.emit('notification', data);
});
};
var pollingLoop = function () {
var socket;
for (var id in sockets) {
socket = sockets[id];
client.get("uuid:" + socket.id, function (err, useridreply) {
processAllTransactions(useridreply);
});
}
connection.query('SELECT * FROM sf_guard_user_profile; select * FROM transaction', function (err, result) {
// error check
if (err) {
console.log(err);
updateSockets(err);
throw err;
} else {
// loop through the queries
var element =
// compare the cache results againt the database query for users
result[0].forEach(function (element, index, array) {
client.get('logged_in:' + element.user_id, function (err, reply) {
if (reply === null) {
// console.log( element.user_id + ' is disconnected');
}
else {
// console.log(reply);
}
});
client.get("user:" + element.user_id, function (err, userreply) {
if (element.user_status != userreply) {
client.set('user:' + element.user_id, +element.user_status);
changed_users.push(element);
console.log(element.user_id + " is now set to: " + element.user_status);
updateSockets({
changed_user_id: element.user_id,
changed_user_status: element.user_status
});
}
});
});
}
// loop on itself only if there are sockets still connected
if (connectionsArray.length) {
pollingTimer = setTimeout(pollingLoop, POLLING_INTERVAL);
// reset changed users and changed transactions arrays
changed_users = [];
changed_transactions = [];
} else {
console.log('The server timer was stopped because there are no more socket connections on the app');
}
});
};
// count the connections array
Array.prototype.contains = function (k, callback) {
var self = this;
return (function check(i) {
if (i >= self.length) {
return callback(false);
}
if (self[i] === k) {
return callback(true);
}
return process.nextTick(check.bind(null, i + 1));
}(0));
};
io.sockets.on('connection', function (socket) {
// runs for every connection
sockets[socket.id] = socket;
socket.on('client-data', function (data) {
// get the user id from the url that is passed onload
var user_id = data.url.replace('http://servernameremoved.com:4000/id/', '');
console.log('user id ' + user_id + ' is connected with session id ' + socket.id);
client.set('uuid:' + socket.id, +user_id);
});
console.log('Number of connections:' + (connectionsArray.length));
// starting the loop only if at least there is one user connected
if (!connectionsArray.length) {
pollingLoop();
}
socket.on('disconnect', function (socketIndex) {
delete sockets[socket.id];
client.get("uuid:" + socket.id, function (err, userreply) {
console.log('user id ' + userreply + ' got redis disconnected');
});
socketIndex = connectionsArray.indexOf(socket);
console.log('socketID = %s got disconnected', socketIndex);
if (~socketIndex) {
connectionsArray.splice(socketIndex, 1);
}
});
connectionsArray.push(socket);
});
// express js route
app.get('/id/:id', function (req, res) {
clients.contains(req.params.id, function (found) {
if (found) {
console.log("Found");
} else {
client.set('logged_in:' + req.params.id, +req.params.id + 'is logged in');
}
});
res.sendFile(__dirname + '/client.html');
});
// build the server
http.listen(4000, function () {
console.log("Server Started");
});
Here is the console log results
Pushing new data to the clients connected ( connections amount = 2 ) - Sat May 30 2015 21:16:23 GMT-0700 (PDT)
30 2015 21:15:15 GMT-0700 (PDT)
user id 3 is connected with session id CRTtkRIl7ihQ2yaEAAAA
user id 2 is connected with session id wNG7XDcEDjhYKBEIAAAB
***********************************************
[{"id":1,"status":20,"original_status":15,"active":1,"buyer_id":2,"seller_id":1,"seller_acked":1,"seller_complete":0,"buyer_complete":1,"buyer_status":4,"chat_ended":"2015-05-31T03:58:40.000Z","seller_status":4}]
***********************************************
Pushing new data to the clients connected ( connections amount = 2 ) - Sat May 30 2015 21:16:23 GMT-0700 (PDT)
***********************************************
[{"id":1,"status":20,"original_status":15,"active":1,"buyer_id":2,"seller_id":1,"seller_acked":1,"seller_complete":0,"buyer_complete":1,"buyer_status":4,"chat_ended":"2015-05-31T03:58:40.000Z","seller_status":4}]
***********************************************
Pushing new data to the clients connected ( connections amount = 2 ) - Sat May 30 2015 21:16:23 GMT-0700 (PDT)

Node.js mysql queries and net connection

I have a node.js script which takes in SMDR data from a phone system, then inserts it into a database. The code that I have right now inserts everything except for OperatorID (which it inserts [object Object] into that column). I cannot figure out as to why this is happening. Any ideas?
var net = require('net'),
mysql = require('mysql'),
PORT = 1752,
HOST = '192.168.10.2',
pool = mysql.createPool({
host: 'localhost',
port: 3306,
user: 'root',
password: 'root',
database: 'mydb'
});
function connectionListener(conn) {
console.log('Listening for incoming calls...');
}
function logCall(phonenumber, operator) {
pool.getConnection(function(err, connection) {
if (!err) { // If no error exists
//var operID = '';
var opquery = connection.query('SELECT OperatorID FROM tblOperators WHERE Phone_Login = ' + operator, function(err, rows) {
if (err) {
console.error(err);
}
//operID = rows[0];
//console.log(operID);
console.log(rows[0]); //logs on console okay
var query = connection.query('INSERT INTO incoming_calls(phone_number, OperatorID) VALUES("' + phonenumber + '", "' + rows[0] + '") ON DUPLICATE KEY UPDATE OperatorID=OperatorID, date_created=NOW()', function(err, rows) {//fails to insert rows[0]
if (err) {
console.error(err);
}
connection.release();
});
});
} else {
console.error(err);
}
});
}
function processdata(data) {
var phonedata = data.toString().match(/([0-9]?)([0-9]{10})/),
operdata = data.toString().match(/([*])([0-9]{4})/);
if (phonedata !== null && operdata !== null) {
var phonenumber = phonedata[2],
oper = operdata[2];
oper = oper.replace('*', '');
//phonenumber = phonenumber.slice(0,3)+"-"+phonenumber.slice(3,6)+"-"+phonenumber.slice(6);
logCall(phonenumber, oper);
}
};
logCall('999-999-9999', '1203'); //where 1203 is valid
var conn = net.createConnection(PORT, HOST, connectionListener);
conn.on('data', processdata);
conn.setEncoding('utf8');

You need to specify
rows[0].OperatorID
in your INSERT query as you are trying to insert the whole row right now, which is an object.
var query = connection.query('INSERT INTO incoming_calls(phone_number, OperatorID) VALUES("' + phonenumber + '", "' + rows[0].OperatorID + '") ON DUPLICATE KEY UPDATE OperatorID=OperatorID, date_created=NOW()', function(err, rows) {//...