We Keep Coding

Using Promises with MySQL Queries Results in Timeout

I'm wondering if I implemented Promises correctly with MySQL statements. The problem is that it will timeout waiting for the query results. I find this strange because I implemented the code the same way using callback functions and it never timed out. I think I'm using Promises incorrectly but can't figure out how. I confirmed that the code using Promises does work on retrieving small datasets (hundreds), but times out when retrieving thousands of records.
Using Promises
function innermostRetrieveDependencyByResource(dep_name_version){
var sql = "SELECT * FROM dependent WHERE name_version = '" + dep_name_version + "' LIMIT 1";
return database.query(sql).then(result => {
return result;
})
}
function innerRetrieveDependencyByResource(scan_id){
var depList = [];
var sql = "SELECT DISTINCT dep_name_version FROM branch WHERE scanid = '" + scan_id + "'";
return database.query(sql).then(result => {
promiseList = [];
for (r in result){ // iterate over branch scans
dep_name_version = result[r];
promiseList.push(innermostRetrieveDependencyByResource(dep_name_version))
}
return Promise.all(promiseList).then(result => {
return result;
})
})
}
function retrieveDependencyByResource(resource, cb){
promiseList = []
var sql = "SELECT (scanid) FROM scan WHERE resource = '" + resource + "'";
database.query(sql).then(result => { // note: query returns a Promise
var scanPending = result.length;
for (s in result){
scan_id = result[s].scanid;
promiseList.push(innerRetrieveDependencyByResource(scan_id))
}
Promise.all(promiseList).then(result => {
cb(result);
})
})
}
Using Callbacks:
function retrieveDependency(projname, branch, cb){
depList = []
var sql = "SELECT (scanid) FROM scan WHERE project = '" + projname + "' AND branch = '" + branch + "'";
connection.query(sql, function (err, result) { // note: query returns a Promise
if (err) throw err;
scan_id = result[0].scanid;
var sql = "SELECT DISTINCT dep_name_version FROM branch WHERE scanid = '" + scan_id + "'";
connection.query(sql, function (err, result) {
if (err) throw err;
pending = result.length;
for (r in result){
dep_name_version = result[r].dep_name_version;
var sql = "SELECT * FROM dependent WHERE name_version = '" + dep_name_version + "'";
connection.query(sql, function(err, result){
if (err) throw err;
depList.push(result);
if ( 0 === --pending ){
cb(depList);
}
})
}
});
});
}

For proper query promises, use mysql2/promise or mysql21.
Also, I'd recommend loading results with a single query:
// with mysql21
let query = `SELECT scanid, dep_name_version, dependent*
FROM scan
JOIN branch USING (scanid)
JOIN dependent ON (name_version = dep_name_version)
WHERE resource = ?`;
const retrieveDependencyByResource = async (resource) =>
connection.query(query, [resource]);
// await results
let dependencies = await retrieveDependencyByResource(resource));
// or use then
retrieveDependencyByResource(resource)
.then(dependencies => {
...
});

variable probably outside of scope

Although the code may seem a little heavy at first (it probably is anyways), it is kinda simple. All I want to do is to read some data from a JSON file (Which is in following format:
{"news":[{}]}
)
, and push some info into an array that I get from another JSON file that comes from one of newsapi.org's servers. My problem is probably caused by hoisting, but I do not know how to fix it..! In the console it spits out Uncaught TypeError: Cannot read property 'title' of undefined
.
What follows below is a snippet from my defunct code.
(I am aware that my code looks messy :P)
function appendNews(id) {
var xhr = new XMLHttpRequest();
xhr.open("GET", "https://newsapi.org/v2/top-headlines?sources=" + id +
"&apiKey=8fdd732e76664e06b177a20fb295129c", true);
xhr.send();
xhr.addEventListener("readystatechange", processRequest, false);
xhr.onreadystatechange = processRequest();
function processRequest() {
if (xhr.readyState == 4 && xhr.status == 200) {
var response = JSON.parse(xhr.responseText);
var container = document.getElementById("newsContainer");
for (var i = 0; i < response.articles.length; i++) {
fs.readFile('./data/events/news.json', 'utf8', function(err, data) {
if (err) {
console.log(err)
} else {
var file = JSON.parse(data)
file.news.push({
"title": response.articles[i].title,
"description": response.articles[i].description,
"url": response.articles[i].url,
"urlImg": response.articles[i].urlToImage,
"time": response.articles[i].publishedAt,
"sourceAndAuthor": response.articles[i].author + " (" +
response.articles[i].source.name + ")"
})
var toStringify = JSON.stringify(file);
fs.writeFile('./data/events/news.json', toStringify, 'utf8', function(err) {
if (err) {
console.log(err);
} else {
}
});
}
});
}
}
EDIT: Further down in my code, I use the same variables, and then it works:
var holder = document.createElement("button");
holder.setAttribute("id", "newsHolder");
holder.setAttribute("onclick", "openArticle(" + "'" +
response.articles[i].title + "'" + ")")
var title = document.createElement("div");
title.setAttribute("id", "newsTitle");
title.innerHTML = response.articles[i].title;
var image = document.createElement("img");
image.setAttribute("src",response.articles[i].urlToImage)
image.setAttribute("id", "articleImage");
holder.appendChild(title);
holder.appendChild(image);
container.appendChild(holder);
fadeIn("newsContainer",1,0.01,1);

This is a major issue in your control flow:
for (var i = 0; i < response.articles.length; i++) {
fs.readFile('./data/events/news.json', 'utf8', function(err, data) {
...
var file = JSON.parse(data)
...
var toStringify = JSON.stringify(file);
fs.writeFile('./data/events/news.json', toStringify, 'utf8', function(err) {
});
});
}
Writing like this demonstrates a complete lack of understanding for asynchronous programming, as what really ends up happening is a race condition where the last successful fs.writeFile() will append only one entry from response.articles to your JSON, even if you were to fix the TypeError.
You're expecting the code to operate like this:
i=1 2 3 4 5 ... L
r r r r r
p p p p p
w w w w w
Where r, p, w stands for fs.readFile(), file.news.push() and fs.writeFile() respectively, but what really happens is this:
i=1 2 3 4 5 ... L
r r r r r ...
p p p p p ...
w w w w w ...
Your pushes and writes will occur in a non-guaranteed order, they will not necessarily be sequential.
where L is reseponse.articles.length, and you're attempting to access response.articles[i] when i === response.articles.length, so of course there will be no article there.
The first change would be to use
for (let i = 0; i < response.articles.length; i++) {
but that will not fix the race condition explained above, it will just cause i to have lexical scope, so at least you won't be accessing out of bounds.
In order to avoid unnecessary reading and writing, you can simply fs.readFile() once, loop through your articles and push them to file.news in the callback, and then fs.writeFile() after the end of the for loop. That would look something like this:
// add your event listener BEFORE sending
xhr.addEventListener('load', processRequest, false);
xhr.send();
function processRequest() {
// no need to check readyState and status if you use the `load` event
fs.readFile(..., (err, data) => {
// handle error
if (err) {
console.log(err);
return;
}
const { articles = [] } = JSON.parse(xhr.responseText);
const file = JSON.parse(data);
for (const article of articles) {
const {
title,
description,
url,
urlToImage: urlImg,
publishedAt: time,
author,
source: { name }
} = article;
file.news.push({
title,
description,
url,
urlImg,
time,
sourceAndAuthor: `${author} (${name})`
});
}
const json = JSON.stringify(file);
fs.writeFile(..., (err) => {
if (err) {
console.log(err);
return;
}
});
});
}

NodeJS Json Data not fully inserting into database

Been 2 days trying to find a solution to my problem.
I request data(json) from a website.
They return fine and json is valid but when i try to insert them to database almost 10% do not get inserted.
I dont know what to do, i even tried php with same results.
Any help world be appreciated thank you.
This is json ouptut after selecting the data attribute var result = obj.data; pastebin
var request = require("request");
var fs = require('fs');
var sleep = require('system-sleep');
var mysql = require('mysql');
var connection = mysql.createConnection({
host: 'localhost',
user: 'root',
password: '',
database: 'feeds',
timezone: 'Z'
});
request.post({
url: "xxx",
headers: {
"Content-Type": "application/json"
},
body: '{"xx": "true", "xxx": "true"}'
}, function(error, response, body) {
var obj = JSON.parse(body);
var result = obj.data;
console.log(result);
for (var i = 0; i < result.length; i++) {
var object = result[i];
for (property in object) {
var value = object[property];
var pid, pname;
if (property == "id") {
pid = value;
}
if (property == "name") {
pname = value;
}
if (property == "xxxxx") {}
if (property == "xxxxxxxx") {
connection.query('INSERT INTO tournaments(id, name) VALUES (' + pid + ', "' + pname + '")');
}
}
}
});

Welcome to SO.
There is probably an error that you are missing. You request the insertion but then let it go. connection.query allows a function as its second parameter, and that is a callback. Check out the following line
connection.query( 'SELECT * FROM some_table', ( err, rows ) => {
// do something with the results here
} );
For you it would look something like:
connection.query('INSERT INTO tournaments(id, name) VALUES (' + pid + ', "' + pname + '")', function(err, rows){
console.log('Successful write', rows);
console.error('Error! This did not write because: ', err);
})
You should console.log both err and rows, for each time this is called. Odds are you will now see why some of the results are not getting written into the DB, and they will show up as errors.

adding mongodb data value filed using node js

Hello all I have to need to add two field values {type:Number} of one collection from MongoDB using node js
and store the result in the same collection of MongoDB
1st node js query fetching the data value from MongoDB inside my controller.
2nd trying to add the fetched value.
3rd store the result in the same collection of MongoDB using node js.
1). Node js
var levelScoreQuiz = require('../models/levelscoreSchema.js');
try{
var queryObj = {};
var projection = '-id child.quiz_level.score_pre';
var projection2 = '-id child.quiz_level.score_curr';
var a = levelScoreQuiz.findOne(queryObj,projection);
var b = levelScoreQuiz.findOne(queryObj,projection2);
//console.log(a);
//console.log(b);
var add = a + b;
//console.log(add);
res.send(add);
var userObj = {
level_pre:req.params.add
};
var user = new levelScoreQuiz(userObj);
user.save(function(err, result){
if (err) {
console.log('Error While Saving the reuslt ' +err)}
else{
//console.log("User score saved successfully");
console.log("User Previous score saved successfully");
res.json(result);
}
});
}catch(err){
console.log('Error While Saving the reuslt ' +err);
return next(err);
}
2). MongoDB Schema
var userScore = new Schema({
child: {
quiz_level:{
current_level:{type:Number},
score_pre:{type:Number},
score_curr:{type:Number}
}
}
});
3). Result: it shows me object in my browser
"[object Object][object Object]"

var levelScoreQuiz = require('../models/levelscoreSchema.js');
try{
var queryObj = {};
var projection = {id: 0, 'child.quiz_level.score_pre': 1};
var projection2 = {id: 0, 'child.quiz_level.score_curr': 1};
var a = levelScoreQuiz.findOne(queryObj,projection);
var b = levelScoreQuiz.findOne(queryObj,projection2);
//console.log(a);
//console.log(b);
var add = a.child.quiz_level.score_pre +
b.child.quiz_level.score_curr;
//console.log(add);
res.send(add);
var userObj = {
child: {quiz_level: { score_pre: req.params.add}}
};
var user = new levelScoreQuiz(userObj);
user.save(function(err, result){
if (err) {
console.log('Error While Saving the reuslt ' +err)}
else{
//console.log("User score saved successfully");
console.log("User Previous score saved successfully");
res.json(result);
}
});
}catch(err){
console.log('Error While Saving the reuslt ' +err);
return next(err);
}

Only Update Specific Users Socket IO and Node JS

I am trying to learn node JS and am currently attempting to extend this article.
http://www.gianlucaguarini.com/blog/push-notification-server-streaming-on-a-mysql-database/
I am having major issue because I am getting multiple updates in the SQL query.
I only want to send one socket update.
This issue is in the transactions loop I get multiple socket updates.
I have been struggling with this for over a month and can't seem to figure it out on my own (or with google searches)
Can someone please tell me how I can make this work so I only get one socket update per client.
What I would like to happen is when there is a change in one of the transactions that the two parties (buyer and seller) are the only ones that get the socket update.
How can I make this work? It is so close to what I want it do but I can't get over this last challenge.
Please help.
Thank you in advance.
<html>
<head>
<title>GAT UPDATER</title>
<script src="/socket.io/socket.io.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">
<script src = "http://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js"></script>
<script type="text/javascript" src="http://board.gameassettrading.com/js/jquery.cookie.js"></script>
</head>
<body>
<script>
var nodeuserid;
function getUserId() {
var url = window.location.href;
var user_id = url.replace('http://heartbeat.gameassettrading.com:4000/id/', '');
return user_id;
}
user_id = getUserId();
$.cookie('useridcookie', user_id, { expires: 1 });
var useridcookie = $.cookie("useridcookie");
// Get Base Url for development between local and dev enviroment
function getBaseURL() {
var url = location.href; // entire url including querystring - also: window.location.href;
var baseURL = url.substring(0, url.indexOf('/', 14));
if (baseURL.indexOf('http://localhost') != -1) {
// Base Url for localhost
var url = location.href; // window.location.href;
var pathname = location.pathname; // window.location.pathname;
var index1 = url.indexOf(pathname);
var index2 = url.indexOf("/", index1 + 1);
var baseLocalUrl = url.substr(0, index2);
return baseLocalUrl + "/";
}
else {
// Root Url for domain name
return baseURL + "/";
}
}
// set the base_url variable
base_url = getBaseURL();
document.domain = "transactionsserver.com"
// create a new websocket
var socket = io.connect('http://heartbeat.transactionsserver.com:4000');
socket.on('connect',function() {
var data = {
url: window.location.href,
};
socket.emit('client-data', data);
});
// this is always open you have to filter out the data you want
socket.on('notification', function (data) {
if(data.hasOwnProperty("data")) {
if(data.data[0]['seller_id'] != ''){
$('#StatusUpdate', window.parent.document).text( data.data[0]['seller_id']+ ':' + data.data[0]['seller_status'] +':'+ data.data[0]['buyer_id']+':'+ data.data[0]['buyer_status']).click();
}
}
window.parent.checkData(data,user_id);
if(data.hasOwnProperty("changed_user_id")) {
$('#StatusUpdate', window.parent.document).text( data.changed_user_id+ ':' + data.changed_user_status +':'+ data.changed_user_id).click();
}
});
</script>
</body>
</html>
Server . js
var app = require("express")();
var path = require('path');
var mysql = require("mysql");
var http = require('http').Server(app);
var io = require("socket.io")(http);
var sockets = {};
var mysql = require('mysql'),
connectionsArray = [],
connection = mysql.createConnection({
multipleStatements: true,
host: 'localhost',
user: '*****',
password: '******',
database: 'transactionsdb',
port: 3306
}),
POLLING_INTERVAL = 1000,
pollingTimer;
// Add Redis For Comparing SQL Results againts Cache
var redis = require('redis');
var client = redis.createClient();
var express = require('express');
/* Creating POOL MySQL connection.*/
var pool = mysql.createPool({
connectionLimit: 100,
host: 'localhost',
user: '*****',
password: '*****',
database: 'transactionsdb',
debug: false
});
var count = 0;
var clients = [];
function processAllTransactions(sellsreply) {
pool.query('SELECT t.id,t.status,t.original_status, t.active, t.buyer_id, t.seller_id, t.seller_acked, t.seller_complete, t.buyer_complete, b.user_status as buyer_status,t.chat_ended, s.user_status as seller_status FROM transaction t LEFT JOIN sf_guard_user_profile s ON s.user_id = t.seller_id LEFT JOIN sf_guard_user_profile b ON b.user_id = t.buyer_id WHERE active = 1 LIMIT 1', [sellsreply], function (err, sells) {
if (sells != '') {
// attempt to stop the updates if it's not the the active transaction
client.get('active transaction id:'+sellsreply, function (err, active_transaction_id) {
passed_active_transaction_id = active_transaction_id;
});
// is there a trasnaction with status defined and transation id does not equal the active transaction id
if(sells[0].status !== undefined && sells[0].id !== passed_active_transaction_id )
{
client.get('active transaction:'+sellsreply, function (err, data1) {
if(JSON.stringify(sells) != data1){
client.set('active transaction id:'+sellsreply,sells[0]["id"]);
client.set('active transaction:'+sellsreply,JSON.stringify(sells));
console.log(JSON.stringify(sells));
updateSockets({
data: sells // pass the database result
});
}
});
}
}
});
}
// Method
function getUserInfo(user_id, callback) {
var query = connection.query('SELECT user_status from sf_guard_user_profile WHERE user_id = ' + connection.escape(user_id));
query.on('result', function (row) {
callback(null, row.user_status);
});
}
var updateSockets = function (data) {
// adding the time of the last update
data.time = new Date();
console.log('Pushing new data to the clients connected ( connections amount = %s ) - %s', connectionsArray.length , data.time);
// sending new data to all the sockets connected
connectionsArray.forEach(function (tmpSocket) {
console.log(tmpSocket);
tmpSocket.volatile.emit('notification', data);
});
};
var pollingLoop = function () {
var socket;
for (var id in sockets) {
socket = sockets[id];
client.get("uuid:" + socket.id, function (err, useridreply) {
processAllTransactions(useridreply);
});
}
connection.query('SELECT * FROM sf_guard_user_profile; select * FROM transaction', function (err, result) {
// error check
if (err) {
console.log(err);
updateSockets(err);
throw err;
} else {
// loop through the queries
var element =
// compare the cache results againt the database query for users
result[0].forEach(function (element, index, array) {
client.get('logged_in:' + element.user_id, function (err, reply) {
if (reply === null) {
// console.log( element.user_id + ' is disconnected');
}
else {
// console.log(reply);
}
});
client.get("user:" + element.user_id, function (err, userreply) {
if (element.user_status != userreply) {
client.set('user:' + element.user_id, +element.user_status);
changed_users.push(element);
console.log(element.user_id + " is now set to: " + element.user_status);
updateSockets({
changed_user_id: element.user_id,
changed_user_status: element.user_status
});
}
});
});
}
// loop on itself only if there are sockets still connected
if (connectionsArray.length) {
pollingTimer = setTimeout(pollingLoop, POLLING_INTERVAL);
// reset changed users and changed transactions arrays
changed_users = [];
changed_transactions = [];
} else {
console.log('The server timer was stopped because there are no more socket connections on the app');
}
});
};
// count the connections array
Array.prototype.contains = function (k, callback) {
var self = this;
return (function check(i) {
if (i >= self.length) {
return callback(false);
}
if (self[i] === k) {
return callback(true);
}
return process.nextTick(check.bind(null, i + 1));
}(0));
};
io.sockets.on('connection', function (socket) {
// runs for every connection
sockets[socket.id] = socket;
socket.on('client-data', function (data) {
// get the user id from the url that is passed onload
var user_id = data.url.replace('http://servernameremoved.com:4000/id/', '');
console.log('user id ' + user_id + ' is connected with session id ' + socket.id);
client.set('uuid:' + socket.id, +user_id);
});
console.log('Number of connections:' + (connectionsArray.length));
// starting the loop only if at least there is one user connected
if (!connectionsArray.length) {
pollingLoop();
}
socket.on('disconnect', function (socketIndex) {
delete sockets[socket.id];
client.get("uuid:" + socket.id, function (err, userreply) {
console.log('user id ' + userreply + ' got redis disconnected');
});
socketIndex = connectionsArray.indexOf(socket);
console.log('socketID = %s got disconnected', socketIndex);
if (~socketIndex) {
connectionsArray.splice(socketIndex, 1);
}
});
connectionsArray.push(socket);
});
// express js route
app.get('/id/:id', function (req, res) {
clients.contains(req.params.id, function (found) {
if (found) {
console.log("Found");
} else {
client.set('logged_in:' + req.params.id, +req.params.id + 'is logged in');
}
});
res.sendFile(__dirname + '/client.html');
});
// build the server
http.listen(4000, function () {
console.log("Server Started");
});
Here is the console log results
Pushing new data to the clients connected ( connections amount = 2 ) - Sat May 30 2015 21:16:23 GMT-0700 (PDT)
30 2015 21:15:15 GMT-0700 (PDT)
user id 3 is connected with session id CRTtkRIl7ihQ2yaEAAAA
user id 2 is connected with session id wNG7XDcEDjhYKBEIAAAB
***********************************************
[{"id":1,"status":20,"original_status":15,"active":1,"buyer_id":2,"seller_id":1,"seller_acked":1,"seller_complete":0,"buyer_complete":1,"buyer_status":4,"chat_ended":"2015-05-31T03:58:40.000Z","seller_status":4}]
***********************************************
Pushing new data to the clients connected ( connections amount = 2 ) - Sat May 30 2015 21:16:23 GMT-0700 (PDT)
***********************************************
[{"id":1,"status":20,"original_status":15,"active":1,"buyer_id":2,"seller_id":1,"seller_acked":1,"seller_complete":0,"buyer_complete":1,"buyer_status":4,"chat_ended":"2015-05-31T03:58:40.000Z","seller_status":4}]
***********************************************
Pushing new data to the clients connected ( connections amount = 2 ) - Sat May 30 2015 21:16:23 GMT-0700 (PDT)