We Keep Coding

Empty array getting logged in, in nodeJs

Flow:
The user submits a queryValue in index.html.
Three API calls are made (using a function called ytAxiosGetFunc) based on the queryValue.
The returned values are put in three arrays: ytQueryAppJs, ytCoverAppJs and ytLiveAppJs.
ytCoverAppJs and ytLiveAppJs contains redundant values. These are removed using a function called compareAndRemove.
Two new arrays are allocated which contain unique values from for each of these respectively. These are ytCoverUniqueAppJs and ytLiveUniqueAppJs.
Hence, a total of five arrays get logged in console, based on the query.
Expected Console Log:
All the arrays are filled.
Current Console Log:
All the arrays are filled, except ytCoverUniqueAppJs and ytLiveUniqueAppJs. These are empty.
Source Code from 'app.js':
// https://stackoverflow.com/a/14930567/14597561
function compareAndRemove(removeFromThis, compareToThis) {
return (removeFromThis = removeFromThis.filter(val => !compareToThis.includes(val)));
}
// Declaring variables for the function 'ytAxiosGetFunc'
let apiKey = "";
let urlOfYtAxiosGetFunc = "";
let ytResponse = "";
let ytExtractedResult = [];
// This function GETs data, parses it, allocates required values in an array.
async function ytAxiosGetFunc(queryOfYtAxiosGetFunc, maxResultsOfYtAxiosGetFunc) {
apiKey = "AI...5U"
urlOfYtAxiosGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&order=relevance&type=video";
try {
ytResponse = await axios({
url: urlOfYtAxiosGetFunc,
method: "get",
params: {
q: queryOfYtAxiosGetFunc,
maxResults: maxResultsOfYtAxiosGetFunc
}
})
let ytResult = ytResponse.data;
for (i = 0; i < (ytResult.items).length; i++) {
ytExtractedResult[i] = ytResult.items[i].id.videoId;
// console.log(ytExtractedResult);
}
return (ytExtractedResult);
ytExtractedResult.length = 0;
ytResponse.length = 0;
} catch (e) {
console.log(e);
}
}
app.post("/", async function(req, res) {
// Accessing the queryValue user submitted in index.html.
query = req.body.queryValue;
// Fetcing top results related to user's query and putting them in the array.
ytQueryAppJs = await ytAxiosGetFunc(query, 4);
console.log("ytQueryAppJs");
console.log(ytQueryAppJs);
// Fetching 'cover' songs related to user's query and putting them in the array.
if (query.includes("cover") == true) {
ytCoverAppJs = await ytAxiosGetFunc(query, 8);
console.log("ytCoverAppJs");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else if (query.includes("live") == true) {
ytCoverAppJs = await ytAxiosGetFunc(query.replace("live", " cover "), 8);
console.log("ytCoverAppJs");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else {
ytCoverAppJs = await ytAxiosGetFunc(query + " cover ", 8);
console.log("ytCoverAppJs");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
}
// Fetching 'live performances' related to user's query and putting them in the array.
if (query.includes("live") == true) {
ytLiveAppJs = await ytAxiosGetFunc(query, 8);
console.log("ytLiveAppJs");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else if (query.includes("cover") == true) {
ytLiveAppJs = await ytAxiosGetFunc(query.replace("cover", " live "), 8);
console.log("ytLiveAppJs");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else {
ytLiveAppJs = await ytAxiosGetFunc(query + " live ", 8);
console.log("ytLiveAppJs");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
}
// Emptying all the arrays.
ytQueryAppJs.length = 0;
ytCoverAppJs.length = 0;
ytCoverUniqueAppJs.length = 0;
ytLiveAppJs.length = 0;
ytLiveUniqueAppJs.length = 0;
});
(I am a beginner. Please guide and suggest a title to categorize this question for coming viewers.)

My friend suggested me to localise ytResponse and ytExtractedResult. So I declared them inside ytAxiosGetFunc.
Notice the commented code. Here's the required change:
// Declaring variables for the function 'ytAxiosGetFunc'
let apiKey = "";
let urlOfYtAxiosGetFunc = "";
// let ytResponse = "";
// let ytExtractedResult = [];
// This function GETs data, parses it, allocates required values in an array.
async function ytAxiosGetFunc(queryOfYtAxiosGetFunc, maxResultsOfYtAxiosGetFunc) {
let ytExtractedResult = [];
apiKey = "A...U"
urlOfYtAxiosGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&order=relevance&type=video";
try {
let ytResponse = await axios({
url: urlOfYtAxiosGetFunc,
method: "get",
params: {
q: queryOfYtAxiosGetFunc,
maxResults: maxResultsOfYtAxiosGetFunc
}
})
let ytResult = ytResponse.data;
for (i = 0; i < (ytResult.items).length; i++) {
ytExtractedResult[i] = ytResult.items[i].id.videoId;
// console.log(ytExtractedResult);
}
return (ytExtractedResult);
// ytExtractedResult.length = 0; // These are unnecessary now.
// ytResponse.length = 0;
} catch (e) {
console.log(e);
}
}
Here's what I think was happening earlier:
ytQueryAppJs was getting 're-filled' for every ytAxiosGetFunc call. This was making ytQueryAppJs to have exact same values (on the same index) as the variable for which the call is actually made. For example, if the function call is made to assign values in ytCoverAppJs, it would set the same values for ytQueryAppJs as well. The same would happen at the time of calling function for ytLiveAppJs. This ultimately caused compareAndRemove function to clear out all the values.
Here's what I think is happening now:
The variables ytExtractedResult and ytResponse are being re-initialised for every call to the function ytAxiosGetFunc. This imply that they don't have the previous values. (And do not need to have their length set to zero.)
Thank you everyone for responding. :)

Nodejs and SQLite how create multiple tables?

I have question about creating multiple tables in one DB.
I try to run for 2 tables, but seems after the first db.run(CREATE TABLE , second db.run(CREATE TABLE does not create a second table in the DB, here is the code i used:
//database.js
var sqlite3 = require('sqlite3').verbose();
var md5 = require('md5');
const users = require("./users");
const usersStatistic = require("./users_statistic");
const DBSOURCE = "db.sqlite";
let db = new sqlite3.Database(DBSOURCE, (err) => {
if (err) {
// Cannot open database
console.error(err.message)
throw err
}else{
console.log('Connected to the SQlite database.');
db.run(`CREATE TABLE user (
id integer,
first_name text,
last_name text,
email text UNIQUE,
gender text,
ip_address text,
CONSTRAINT email_unique UNIQUE (email)
)`,(err) => {
if (err) {
// Table already created
}else{
// Table just created, creating some rows
let props = '';
for (let i = 0; i < Object.keys(users[0]).length; i++) {
if(i === Object.keys(users[0]).length - 1) {
props += `${Object.keys(users[0])[i]}`
}else {
props += `${Object.keys(users[0])[i]}, `
}
}
const insert = `INSERT INTO user (${props}) VALUES (?,?,?,?,?,?)`;
for (let i = 0; i < users.length; i++) {
const user = users[i];
db.run(insert,
Object.values(user)
// user['id'],
// user['first_name'],
// user['last_name'],
// user['email'],
// user['gender'],
// user['ip_address']
)
}
// db.run(insert, [users[0]['first_name'], users[0]['last_name'], "admin#example.com"])
// db.run(insert, [users[1]['first_name'], users[2]['last_name'],"user#example.com"])
}
});
//Start second table
db.run(`CREATE TABLE user_statistic (
user_id integer,
date text,
page_views integer,
clicks integer,
)`,(err) => {
if (err) {
// Table already created
}else{
// Table just created, creating some rows
let props = '';
for (let i = 0; i < Object.keys(usersStatistic[0]).length; i++) {
if(i === Object.keys(usersStatistic[0]).length - 1) {
props += `${Object.keys(usersStatistic[0])[i]}`
}else {
props += `${Object.keys(usersStatistic[0])[i]}, `
}
}
const insert = `INSERT INTO user (${props}) VALUES (?,?,?,?)`;
console.log(insert)
for (let i = 0; i < usersStatistic.length; i++) {
const userStatistic = usersStatistic[i];
db.run(insert,
Object.values(userStatistic)
)
}
}
});
}
});
module.exports = db;

Hey there it has been a while but no one seems to reply.
I think you could try to use
db.serialize(()=>{
db.run([your 1st query])
db.run([your another query])
db.run([and another query])
})
db.close()

Cycle through existing users until one isn't found

I'm just messing about really but ran into a problem. I'm trying to create a username when a user clicks on a div within the page.
I want it to store the usernames in an array. If the username already exists in the array I want it to re-run the function to create a new username until it finds one that doesn't exist then put that into the array.
I just can't seem to workout how to achieve this. Could anyone help please? Thank You
$(document).ready(function() {
jQuery(function() {
var usernames = [];
function createUser() {
var i = Math.floor((Math.random() * 10) + 1);
var newUser = "Username" + i;
return newUser;
}
$(".CreateUser").click(function() {
var newUser = createUser();
if (jQuery.inArray(newUser, usernames) !== -1) {
newUser = createUser();
console.log("User Exists");
} else {
usernames.push(newUser);
$(".Usernames").append("<br>" + newUser);
console.log(usernames);
console.log("User Added");
}
});
});
});

Could be time for a do/while loop:
do {
var newUser = createUser();
} while (jQuery.inArray(newUser, usernames) !== -1)
usernames.push(newUser);
The loop will keep creating new random usernames until it finds one which does not exist in the array. Then it breaks the loop and pushes that last name into the array.

We can use while loop , but I don't recommend it with your function , so better way to make a function with an initial value , so each time you click on the button , the username will be changed .
function createUser(i) {
var newUser = "Username" + i;
return newUser;
}
$(document).ready(function () {
var usernames = [];
//initial value
var i = 1;
$(".CreateUser").click(function () {
var newUser = createUser(i);
//increase the initial value so next username will be increased by 1
i += 1;
usernames.push(newUser);
$(".Usernames").append("<br>" + newUser);
console.log(usernames);
console.log("User Added");
});
});
check this example
I made it as simple as I can .
to check in an array , you can use includes method
var user_in_array = usernames.includes(newUser) // returns true of false
to generate a random string and check if it exists or not
function createUser() {
var random_txt = Math.random().toString(36).substring(2);
var newUser = "Username_" + random_txt;
return newUser;
}
$(document).ready(function () {
var usernames = [];
//initial value
$(".CreateUser").click(function () {
var newUser = createUser();
//increase the initial value so next username will be increased by 1
while (usernames.includes(newUser)) {
newUser = createUser();
}
usernames.push(newUser);
$(".Usernames").append("<br>" + newUser);
console.log(usernames);
console.log("User Added");
});
});
working example here

How to get child Snapshots inside snapshots on Google Cloud functions (Firebase)?

I'm trying to get some for Loops running inside a google cloud functions everytime I delete my /users node.
This is the code I'm using
exports.deleteUserAssets = functions.database.ref('/users/{userId}').onWrite((change, context) => {
const beforeData = change.before.val();
const afterData = change.after.val();
const userBuildings = Object.keys(beforeData.isAdmin); // get the buildings of the user stored in the user/userId/isAdmin node .. so far so good
const userId = beforeData.userIDforCloudFunctions; // I'm getting this from a /users/userid/userIDforCloudFucntions node ...so far so good (i've been logging it to confirm)
// making sure it was a delete operation ... so far so good
if (afterData !== null) {
return 0;
}
else {
// on each building
for (var i = 0; i < userBuildings.length; i++) {
let eachBuilding = [userBuildings[i]]
// HERE IS WERE THE PROBLEM IS: Trying to delete all depts + rooms + doors
admin.database().ref('/buildings/' + eachBuilding)
.child("hasDepts")
.once("value")
.then(function(snapshot) { // This is where it goes south – snapshot is returning null
snapshot.forEach(function(childSnapshot) {
var deptKeyString = childSnapshot.key; // will try to get the keys of the departments stored under this space
var deptsOnNode = admin.database().ref('/depts/' + deptKeyString);
deptsOnNode.remove(); // and use the keys to delete each of the depts on depts
});
});
admin.database().ref('/buildings/' + eachBuilding).set({}); // this is working
admin.database().ref('/buildingsUserUid/' + userId + '/' + eachBuilding).remove(); // this is working
}
}
return 0;
});
The snapshot of admin.database().ref('/buildings/' + eachBuilding).child("hasDepts") is returning null.
How can I get to it? Besides admin.database().ref() I've tried to reach it with firebase.database().ref() which is the command/object i use to get this running on frontend functions. I've also tried functions.database() with no result.

Taking in consideration what Doug Stevenson mentioned in his second comment:
exports.deleteUserAssets = functions.database.ref('/users/{userId}').onDelete((change, context, event) => {
const beforeData = change.before.val(); // data before the write (data of all the doors child nodes)
const afterData = change.after.val(); // data before the write (data of all the doors child nodes)
const userBuildings = Object.keys(beforeData.isAdmin); // get the buildings of the user
const userId = beforeData.userIDforCloudFunctions;
// make sure user was deleted
if (afterData !== null) {
return 0;
}
else {
// on each building
for (var i = 0; i < userBuildings.length; i++) {
let eachBuilding = [userBuildings[i]]
// Need to RETURN the whole chain of promises
return admin.database().ref('/buildings/' + eachBuilding)
.child("hasDepts")
.once("value")
.then(function(snapshot) {
console.log(snapshot.val()) // this now works
snapshot.forEach(function(childSnapshot) {
console.log(childSnapshot.val()) // this works as well
var deptKeyString = childSnapshot.key; // get the keys of the departments stored under this space
var deptsOnNode = admin.database().ref('/depts/' + deptKeyString);
// and you can keep on going deeper if you return promises
return deptsOnNode
.child('hasRooms')
.once('value')
.then(function(grandchildSnapshot){
console.log(grandchildSnapshot.val())
grandchildSnapshot.forEach(function(grandGrandchildSnapshot){
var roomKeyString = grandGrandchildSnapshot.key;
var roomsOnDepts = admin.database().ref('/rooms/' + roomKeyString);
admin.database().ref('/roomOwners/' + userId + '/' + roomKeyString).remove();
// and return again here...
return roomsOnDepts
.child('hasDoors')
.once('value')
.then(function(grandgrandGrandchildSnapshot){
grandgrandGrandchildSnapshot.forEach(function(grandgrandGrandchildSnapshot){
var doorKeyString = grandgrandGrandchildSnapshot.key;
var doorsOnRooms = admin.database().ref('/doors/' + doorKeyString);
doorsOnRooms.remove();
let clipOwners = admin.database().ref('/clipOwners/' + doorKeyString);
clipOwners.remove();
})
roomsOnDepts.remove();
})
})
deptsOnNode.remove(); // use the keys to delete the depts on depts main Node
})
});
admin.database().ref('/buildings/' + eachBuilding).set({});
admin.database().ref('/buildingsUserUid/' + userId + '/' + eachBuilding).remove();
});
}
}
return 0;
});

Node / MySQL - code: 'ER_PARSE_ERROR', when trying to insert ~800 records

I am working on small idea to collect errors from pages and to store them in DB and then use graph API to display information visually.
There is 8 sites and on each of them there is 100 entries - so 800 transactions per time.
I loop through each site and then sub-loop through table of errors and collect them.
I got it working if I make insert query on each of those sub-loops for all 800 entries but I am getting some sort of memory leak from so many transactions and after few minutes - Node breaks due to memory exceeding.
So I tried queuing all 800 entries into Array of Arrays and then performing multi-insert at the end of every iteration but I am getting ER_PARSE_ERROR.
var tabletojson = require('tabletojson');
var mysql = require("mysql");
var striptag = require("striptags");
var fs = require("fs");
var path = require('path');
var startCollector;
var iterations = 0;
var insertions = 0;
var duplicated = 0;
var datas = [];
var clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
var appDir = path.dirname(require.main.filename);
var errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
var con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
function CollectErrors() {
startCollector = new Date();
for(var a = 0; a < clients.length; a++) {
(function(a) {
tabletojson.convertUrl("http://example.com" + clients[a] + "/page.php?limit=100", { stripHtmlFromCells: false }, function(response) {
var rs = response[0];
for(var l = rs.length-1; l > -1; l--) {
var newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
var user = getUser(striptag(rs[l]["User"]));
var msg = striptag(rs[l]["Error"]);
var splitError = rs[l]["Error"].split("<a href=\"");
var link = getUrl(splitError[1]);
var id = getId(link);
var type = getType(striptag(splitError[0]));
var temp = [newDate, link, type, user, clients[a], id, msg];
datas.push(temp);
}
});
})(a);
}
con.getConnection(function(err, connection) {
connection.query("INSERT IGNORE INTO entries (time, url, type, author, client, uid, message) VALUES ?", [datas], function(err, rows) {
console.log(err);
});
connection.release();
datas = [];
});
setTimeout(CollectErrors, 10000);
}
function formatDate(date, time) {
var newdate = date.split("/").reverse().join("-");
var newtime = time+":00";
return newdate + " " + newtime;
}
function getUrl(uri) {
return "http://example.com/"+uri.split("\">Details")[0];
}
function getId(url) {
return decodeURIComponent((new RegExp('[?|&]' + "id" + '=' + '([^&;]+?)(&|#|;|$)').exec(url) || [null, ''])[1].replace(/\+/g, '%20')) || null;
}
function getType(error) {
for(var a = 0; a < errorList.length; a++) {
if(error.indexOf(errorList[a]) !== -1) {
return errorList[a];
}
}
return "Other";
}
function getUser(user) {
if(user == "" || user == " " || user == null) {
return "System";
}
return user;
}
CollectErrors();
I've tried mysql.createConnection too but that also gave me same issue.
I've been stuck for past 12 hours and I can't see what's wrong, I've even tried populating Datas table with just strings but got same error.

I've changed your code to use ES6 and correct modules features.
Useful links: correct pooling with mysql, correct insert query, async/await, IIFE, enhanced object
const tabletojson = require('tabletojson'),
mysql = require("mysql"),
striptag = require("striptags"),
fs = require("fs"),
path = require('path');
const startCollector,
iterations = 0,
insertions = 0,
duplicated = 0;
let datas = [];
const clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
const appDir = path.dirname(require.main.filename);
const errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
const con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
// We'll use async/await from ES6
const collectErrors = async() => {
// Up to here I've only changed syntax to ES6
let startCollector = new Date();
// We'll try to iterate through each client. And we use here for..of syntax to allow us using await
for (let client of clients) {
// Please, check that client value return correct data. If not, change for..of to your for..each and client variable to clients[a]
const tbj = await tabletojson.convertUrl("http://example.com" + client + "/page.php?limit=100", {
stripHtmlFromCells: false
});
const result = tgj[0];
for (rs of result) {
// I can't check this part, but I hope your example was with correct values.
let newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
let user = getUser(striptag(rs[l]["User"]));
let link = getUrl(splitError[1]);
let msg = striptag(rs[l]["Error"]);
let id = getId(link);
let splitError = rs[l]["Error"].split("<a href=\"");
let getType = getType(striptag(splitError[0]));
// ES6 enhanced object syntax
datas.push({
newDate,
user,
msg,
id,
splitError,
link,
getType,
temp: [newDate, link, type, user, client, id, msg]
});
}
}
// OK, here we have fulfilled datas array. And we want to save it.
con.getConnection((err, connection) => {
// Please, notice, here I've changed your insert query to prepared statement.
connection.query("INSERT IGNORE INTO entries SET ?", datas, (err, rows) => {
console.log(err);
connection.release();
datas = [];
});
});
// I don't see why do you need timeout here, so I've left it commented.
// setTimeout(CollectErrors, 10000);
};
// Here your other methods go....
// And to call your async function we'll use IIFE
(async() => {
await collectErrors();
})();
Probably there may be errors with mysql insert, but that's not for sure. If occurred, please write in comments and I'll help you with that.