We Keep Coding

Sending data to psql from node

I had an excel file that I managed to read in the NodeJS, here is the excel format:
I have no header, only 4 columns. And the code I used is the following:
const { Client } = require("pg");
const xlsx = require("node-xlsx");
var data = xlsx.parse(__dirname + "/disease_data.xlsx");
var diseaseCode = [];
var disease = [];
var symptomCode = [];
var symptom = [];
const client = new Client({
host: "localhost",
user: "postgres",
port: 5432,
password: "dizertatie",
database: "SymptomChecker",
});
client.connect();
for (var i = 0; i < data.length; i++) {
var sheet = data[i];
console.log(sheet["data"][i][1]);
for (var j = 0; j < sheet["data"].length; j++) {
//add the row to the rows array
// diseaseCode.push(sheet["data"][j][0]);
//Here is the problem
client.query(`INSERT INTO diseases (diseaseCode, disease, symptomCode, symptom) VALUES ('${}')`)
}
}
The problem is, I managed to stack each column from the excel into different arrays, but I don't know to push them corresponding line by line. What can I do?

Variable Names with Node SQLite [duplicate]

From the npm docs, only visible prepared statements are for insert. Does these prepared statement work for Select, update, and delete?
I tried for select, there isn't a .each function where the rows are called back. Anyone been able to do this or have links to resources, cause I can sure as hell unable to find any.

According to the node-sqlite3 API documentation, you can use parameters in your SQL queries in several different ways:
// Directly in the function arguments.
db.run("UPDATE tbl SET name = ? WHERE id = ?", "bar", 2);
// As an array.
db.run("UPDATE tbl SET name = ? WHERE id = ?", [ "bar", 2 ]);
// As an object with named parameters.
db.run("UPDATE tbl SET name = $name WHERE id = $id", {
$id: 2,
$name: "bar"
});

Yes, prepared statements are supported.
With node-sqlite3:
var sqlite3 = require('sqlite3').verbose();
var db = new sqlite3.Database('data.db');
db.serialize(function() {
var stmt = db.prepare("INSERT INTO users VALUES (?,?)");
for (var i = 0; i < 10; i++) {
stmt.run("user " + i, "email " + i);
}
stmt.finalize();
stmt = db.prepare("SELECT * FROM users WHERE id=?");
stmt.each(userId, function(err, row) {
console.log(row.name, row.email);
}, function(err, count) {
stmt.finalize();
});
});
With better-sqlite3:
var Database = require('better-sqlite3');
var db = new Database('foobar.db', options);
var stmt = db.prepare("INSERT INTO users VALUES (?,?)");
for (var i = 0; i < 10; i++) {
stmt.run("user " + i, "email " + i);
}
var stmt = db.prepare('SELECT * FROM users WHERE id=?');
var row = stmt.get(userId);
console.log(row.name, row.email);

NodeJs - Error: Can't set headers after they are sent & TypeError: req.next is not a function

I am trying to get some results from the database mysql. I am calling a method to insert into database and another one to read what I inserted before.
I am doing this because node.js not working synchronous. This is my code:
exports.list = function(req, res){
var moduleRows;
req.getConnection(function(err,connection){
var query = connection.query('SELECT * FROM questions WHERE module_id = ?',[X],function(err,rows)
{
for(var i = 0;i < rows.length;i++){
moduleRows = rows[i];
GetResults(moduleRows);
}
});
var GetResults = function(moduleRows) {
var RightAnswer = 0;
var WrongAnswer1 = 0;
var query = connection.query("SELECT * FROM studentsanswers WHERE Question = '"+moduleRows.question+"' ",function(err,rows2)
{
for(var ii = 0;ii < rows2.length;ii++){
if (rows2[ii].Answer == moduleRows.RightAnswer){
RightAnswer++;
}
else if (rows2[ii].Answer == moduleRows.WrongAnswer1){
WrongAnswer1++;
}
}
var Data = {
question: moduleRows.question,
RightAnswer: moduleRows.RightAnswer,
WrongAnswer1: moduleRows.WrongAnswer1,
NumberOfRightAnswer: RightAnswer,
NumberOfWrongAnswer1 : WrongAnswer1
};
insertResults(Data);
return;
});
}
var insertResults = function(Data) {
var query = connection.query("INSERT INTO statistics set ? ",Data, function(err, rows)
{
PrintResults();
return;
});
}
var PrintResults = function() {
var query = connection.query("SELECT * FROM statistics" ,function(err,rows)
{
res.render('StatisticsByModules',{page_title:"Questions - Node.js",data:rows});
return;
});
}
});
};
I am getting the errors:
Error: Can't set headers after they are sent.
TypeError: req.next is not a function

In following code, you call GetResults(moduleRows) few times which calls PrintResults Few times, In your PrintResults Method you are sending res.render few time which you cannot do for a single request. First, you will have to make your data list then call req.render once with queried data.
var query = connection.query('SELECT * FROM questions WHERE module_id = ?', [X], function (err, rows) {
for (var i = 0; i < rows.length; i++) { // <== HERE
moduleRows = rows[i];
GetResults(moduleRows);
}
});

Node / MySQL - code: 'ER_PARSE_ERROR', when trying to insert ~800 records

I am working on small idea to collect errors from pages and to store them in DB and then use graph API to display information visually.
There is 8 sites and on each of them there is 100 entries - so 800 transactions per time.
I loop through each site and then sub-loop through table of errors and collect them.
I got it working if I make insert query on each of those sub-loops for all 800 entries but I am getting some sort of memory leak from so many transactions and after few minutes - Node breaks due to memory exceeding.
So I tried queuing all 800 entries into Array of Arrays and then performing multi-insert at the end of every iteration but I am getting ER_PARSE_ERROR.
var tabletojson = require('tabletojson');
var mysql = require("mysql");
var striptag = require("striptags");
var fs = require("fs");
var path = require('path');
var startCollector;
var iterations = 0;
var insertions = 0;
var duplicated = 0;
var datas = [];
var clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
var appDir = path.dirname(require.main.filename);
var errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
var con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
function CollectErrors() {
startCollector = new Date();
for(var a = 0; a < clients.length; a++) {
(function(a) {
tabletojson.convertUrl("http://example.com" + clients[a] + "/page.php?limit=100", { stripHtmlFromCells: false }, function(response) {
var rs = response[0];
for(var l = rs.length-1; l > -1; l--) {
var newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
var user = getUser(striptag(rs[l]["User"]));
var msg = striptag(rs[l]["Error"]);
var splitError = rs[l]["Error"].split("<a href=\"");
var link = getUrl(splitError[1]);
var id = getId(link);
var type = getType(striptag(splitError[0]));
var temp = [newDate, link, type, user, clients[a], id, msg];
datas.push(temp);
}
});
})(a);
}
con.getConnection(function(err, connection) {
connection.query("INSERT IGNORE INTO entries (time, url, type, author, client, uid, message) VALUES ?", [datas], function(err, rows) {
console.log(err);
});
connection.release();
datas = [];
});
setTimeout(CollectErrors, 10000);
}
function formatDate(date, time) {
var newdate = date.split("/").reverse().join("-");
var newtime = time+":00";
return newdate + " " + newtime;
}
function getUrl(uri) {
return "http://example.com/"+uri.split("\">Details")[0];
}
function getId(url) {
return decodeURIComponent((new RegExp('[?|&]' + "id" + '=' + '([^&;]+?)(&|#|;|$)').exec(url) || [null, ''])[1].replace(/\+/g, '%20')) || null;
}
function getType(error) {
for(var a = 0; a < errorList.length; a++) {
if(error.indexOf(errorList[a]) !== -1) {
return errorList[a];
}
}
return "Other";
}
function getUser(user) {
if(user == "" || user == " " || user == null) {
return "System";
}
return user;
}
CollectErrors();
I've tried mysql.createConnection too but that also gave me same issue.
I've been stuck for past 12 hours and I can't see what's wrong, I've even tried populating Datas table with just strings but got same error.

I've changed your code to use ES6 and correct modules features.
Useful links: correct pooling with mysql, correct insert query, async/await, IIFE, enhanced object
const tabletojson = require('tabletojson'),
mysql = require("mysql"),
striptag = require("striptags"),
fs = require("fs"),
path = require('path');
const startCollector,
iterations = 0,
insertions = 0,
duplicated = 0;
let datas = [];
const clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
const appDir = path.dirname(require.main.filename);
const errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
const con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
// We'll use async/await from ES6
const collectErrors = async() => {
// Up to here I've only changed syntax to ES6
let startCollector = new Date();
// We'll try to iterate through each client. And we use here for..of syntax to allow us using await
for (let client of clients) {
// Please, check that client value return correct data. If not, change for..of to your for..each and client variable to clients[a]
const tbj = await tabletojson.convertUrl("http://example.com" + client + "/page.php?limit=100", {
stripHtmlFromCells: false
});
const result = tgj[0];
for (rs of result) {
// I can't check this part, but I hope your example was with correct values.
let newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
let user = getUser(striptag(rs[l]["User"]));
let link = getUrl(splitError[1]);
let msg = striptag(rs[l]["Error"]);
let id = getId(link);
let splitError = rs[l]["Error"].split("<a href=\"");
let getType = getType(striptag(splitError[0]));
// ES6 enhanced object syntax
datas.push({
newDate,
user,
msg,
id,
splitError,
link,
getType,
temp: [newDate, link, type, user, client, id, msg]
});
}
}
// OK, here we have fulfilled datas array. And we want to save it.
con.getConnection((err, connection) => {
// Please, notice, here I've changed your insert query to prepared statement.
connection.query("INSERT IGNORE INTO entries SET ?", datas, (err, rows) => {
console.log(err);
connection.release();
datas = [];
});
});
// I don't see why do you need timeout here, so I've left it commented.
// setTimeout(CollectErrors, 10000);
};
// Here your other methods go....
// And to call your async function we'll use IIFE
(async() => {
await collectErrors();
})();
Probably there may be errors with mysql insert, but that's not for sure. If occurred, please write in comments and I'll help you with that.

Adding rows to a table, looping an array, doing crazy things, readding first row each time

I feel like this is probably an issue with the loop somewhere, but I am a bit newer to coding and I am having trouble figuring out where exactly the issue is.
Every time I add a new task to the list, it prints the first task again. So if i add a second task, it inserts the first task and second task into the table, if i add a third task it inserts the first task and third task into the table.. its weird.
I looked in my local storage files, and its being stored properly. as in, (task one, task two, task three) no repetition there and its getting stored the way I want it to.
The issue is I am trying to do this:
I want to loop through the array, and have everything that is stored in local storage be posted on the table. (Currently, when I refresh it resets completely. its acting like session storage rather than local storage. it shows up in local storage after I refresh, but as soon as I click to add more after the refresh it disappears)
its a to do list, so I want it to basically show all items by looping through the array, and when I add a new item, have it store that and loop through the array again and post it on the table.
var table = document.getElementById("tableBody");
toDoArray = [];
function buildTable(){
addToStorage();
var retrievedTaskObject = localStorage.getItem("task");
var parsedObject = JSON.parse(retrievedTaskObject);
var addTheTaskName = parsedObject.taskName;
var addTheTaskDate = parsedObject.taskDate;
for(i=0; i < toDoArray.length; i++){
var row = table.insertRow(0);
var cellName = row.insertCell(0);
var cellDate = row.insertCell(1);
var cellId = row.insertCell(2);
var cellCheck = row.insertCell(3);
cellName.innerHTML= parsedObject[i].name;
cellDate.innerHTML= parsedObject[i].date;
var checkStuff = "<input type='checkbox'>";
cellCheck.innerHTML = checkStuff;
}
}
function submitForm(name,date) {
var addTaskName = document.getElementById("taskName").value;
var addTaskDate = document.getElementById("dateTask").value;
var taskSomething = getTaskObj(addTaskName,addTaskDate);
toDoArray.push(taskSomething);
buildTable();
};
function getTaskObj(taskName,taskData){
var taskObject = {
name: taskName,
date: taskData,
};
return taskObject;
}
function addToStorage(){
var storedArray = JSON.stringify(toDoArray);
localStorage.setItem("task",storedArray);}

When you are calling buildTable in submit form, its looping over the whole array again and adding the elements to table. Try this,
var table = document.getElementById("tableBody");
toDoArray = [];
function buildTable(){
addToStorage();
var retrievedTaskObject = localStorage.getItem("task");
var parsedObject = JSON.parse(retrievedTaskObject);
var addTheTaskName = parsedObject.taskName;
var addTheTaskDate = parsedObject.taskDate;
for(i=0; i < toDoArray.length; i++){
addTaskToTable(parsedObject[i]);
}
}
function addTaskToTable(obj){
var row = table.insertRow(0);
var cellName = row.insertCell(0);
var cellDate = row.insertCell(1);
var cellId = row.insertCell(2);
var cellCheck = row.insertCell(3);
cellName.innerHTML= obj.name;
cellDate.innerHTML= obj.date;
var checkStuff = "<input type='checkbox'>";
cellCheck.innerHTML = checkStuff;
}
function submitForm(name,date) {
var addTaskName = document.getElementById("taskName").value;
var addTaskDate = document.getElementById("dateTask").value;
var taskSomething = getTaskObj(addTaskName,addTaskDate);
toDoArray.push(taskSomething);
addTaskToTable(taskSomething);
};
function getTaskObj(taskName,taskData){
var taskObject = {
name: taskName,
date: taskData,
};
return taskObject;
}
function addToStorage(){
var storedArray = JSON.stringify(toDoArray);
localStorage.setItem("task",storedArray);
}

Here's the snippet with my comments.
var tasksStorageName = 'myTasks';
var tasksTable = document.getElementById("tasks_table");
var saveBtn = document.getElementById("save_task_btn");
var tasksList;
var storage = getStorage();
// this is dirty workaround, because localStorage disabled
// due to security reasons. feel free to remove this function
// and use localStorage directly
function getStorage() {
try {
return window.localStorage;
}
catch(e) {
return {
get: function(key) {
return JSON.stringify([
{
id: Date.now(),
name: 'first task',
date: '10/10/2016',
done: false
}
]);
},
set: function(key, object) {
}
};
}
}
function getTasksList() {
var dumpedTasks = storage.get(tasksStorageName);
// in case there are no tasks saved we return empty list
// otherwise parse our tasks.
return dumpedTasks == null ? [] : JSON.parse(dumpedTasks);
}
function dumpTaskList(tasksList) {
var dumpedTasks = JSON.stringify(tasksList);
storage.set(tasksStorageName, dumpedTasks);
}
// try to reuse things, so we have `renderTasks` function
// to render all our tasks to table which uses `renderTask`
// for each task. later we'll use it to add our newly created
// tasks.
function renderTasks(tasks) {
for (var index = 0; index < tasks.length; index++) {
var task = tasks[index];
renderTask(task);
}
}
// helper function to create checkbox
function createCheckboxForTask(task){
var checkbox = document.createElement('input');
checkbox.type = 'checkbox';
checkbox.checked = task.done;
return checkbox;
}
function renderTask(task) {
var taskRow = tasksTable.insertRow();
var doneCell = taskRow.insertCell();
var nameCell = taskRow.insertCell();
var dateCell = taskRow.insertCell();
taskRow.id = task.id;
doneCell.appendChild(createCheckboxForTask(task));
nameCell.innerText = task.name;
dateCell.innerText = task.date;
}
function createTask(name, date) {
return {
// quite naive id generation, don't use it in production
id: Date.now(),
name: name,
date: date,
done: false
}
}
// to restore your previously added tasks
// you need to get them from localStorage
// on load event and render them.
window.addEventListener('load', function(e) {
tasksList = getTasksList();
if (tasksList.length > 0) {
renderTasks(tasksList);
}
});
saveBtn.addEventListener('click', function() {
var name = document.getElementById("task_name").value;
var date = document.getElementById("task_date").value;
var newTask = createTask(name, date);
tasksList.push(newTask);
renderTask(newTask);
dumpTaskList(tasksList);
});
/*
// function does too many things
function buildTable(){
// if you try to call `buildTable` at page load then
// `addToStorage` function will set localStorage task item
// to empty array. thats why you lost your state every time
// you reload page
addToStorage();
var retrievedTaskObject = localStorage.getItem("task");
var parsedObject = JSON.parse(retrievedTaskObject);
// why this two unnecessary variables?
var addTheTaskName = parsedObject.taskName;
var addTheTaskDate = parsedObject.taskDate;
for(i=0; i < toDoArray.length; i++){
addTaskToTable(parsedObject[i]);
}
}
// try to give parametes as well as variables meaningful names
// like `task` this time, not `obj`
function addTaskToTable(obj){
var row = table.insertRow(0);
var cellName = row.insertCell(0);
var cellDate = row.insertCell(1);
var cellId = row.insertCell(2);
var cellCheck = row.insertCell(3);
cellName.innerHTML= obj.name;
cellDate.innerHTML= obj.date;
var checkStuff = "<input type='checkbox'>";
cellCheck.innerHTML = checkStuff;
}
function submitForm(name,date) {
var addTaskName = document.getElementById("taskName").value;
var addTaskDate = document.getElementById("dateTask").value;
var taskSomething = getTaskObj(addTaskName,addTaskDate);
toDoArray.push(taskSomething);
addTaskToTable(taskSomething);
};
// actually you are creating new task object, not getting it
// from somewhere, so it's better to name it like createTask
// or newTask
function getTaskObj(taskName,taskData){
var taskObject = {
name: taskName,
date: taskData,
};
return taskObject;
}
function addToStorage(){
var storedArray = JSON.stringify(toDoArray);
localStorage.setItem("task",storedArray);
}
*/
<div>
<label>Name: </label>
<input id="task_name" type="text"/>
</div>
<div>
<label>Date: </label>
<input id="task_date" type="text"/>
</div>
<button id="save_task_btn" type="button">Save</button>
<table id="tasks_table">
<thead>
<tr>
<th>Done</th>
<th>Name</th>
<th>Date</th>
</tr>
</thead>
<tbody>
</tbody>
<table>