async function inside a for loop - javascript

Hi i´m trying to convert sqlite database to NeDb, with this code:
const sqliteJSON = require('sqlite-json');
const Datastore = require('nedb')
const exporter = sqliteJSON('etecsa.db');
db = new Datastore('etecsa.nedb');
db.loadDatabase();
tables = ['fix','movil'];
tables.forEach(function(table) {
sql = 'select count(1) from ' + table;
exporter.json(sql, function (err, json) {
toNeDB(table, JSON.parse(json)[0]['count(1)'])
});
}, this);
var toNeDB = function(table, count) {
var inc = 10000;
console.log(table + ' => ' + count)
for (var i = 0; i < count + inc; i += inc) {
var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC LIMIT '+ i + ' , ' + inc;
console.log(i)
exporter.json(sql, function(err, json) {
var data = JSON.parse(json);
db.insert(data, function (err, newDoc) {});
});
}
}
the problem is that the for loop its not working as I desire. I need to use it to change the sql pagination because the sqlite database is very huge and I can´t pass all the data on a single query.
UPDATE using async.map
const sqliteJSON = require('sqlite-json');
const Datastore = require('nedb')
var range = require("range");
var async = require("async");
const exporter = sqliteJSON('etecsa.db');
db = new Datastore('etecsa.nedb');
db.loadDatabase();
tables = ['fix','movil'];
tables.forEach(function(table) {
sql = 'select count(1) from ' + table;
exporter.json(sql, function (err, json) {
toNeDB(table, JSON.parse(json)[0]['count(1)'])
});
}, this);
var toNeDB = function(table, count, cb) {
var inc = 10000;
var pagination = range.range(1,count+inc,inc)
async.map(pagination, function (page, cb){
var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC LIMIT '+ page + ' , ' + inc;
console.log(page, table, inc);
exporter.json(sql, function(err, json) {
var data = JSON.parse(json);
console.log(data[0])
db.insert(data, function (err, newDoc) {});
});
}.bind({ table: table, inc: inc }), function(err,results){
})
}
and the output:
1 'fix' 10000
10001 'fix' 10000
....
1150001 'fix' 10000
1 'movil' 10000
10001 'movil' 10000
...
3730001 'movil' 10000
{ number: '8775031',
name: 'UNION ELECTRICA',
address: 'S ALLENDE #666 OQUENDO SOLEDAD',
province: 7 }
{ number: '8734454',
name: 'EMP ESTB ESP Y SERVICIOS',
address: 'ESAPDA #256 CONCORDIA S LAZARO',
province: 7 }

If you need to know when each action occurred, you should put the console.log inside the callback.
Something like that:
var toNeDB = function(table, count) {
var inc = 10000;
console.log(table + ' => ' + count)
for (var i = 0; i < count + inc; i += inc) {
var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC LIMIT '+ i + ' , ' + inc;
exporter.json(sql, (function(i) {
return function(err, json) {
console.log(i)
var data = JSON.parse(json);
db.insert(data, function (err, newDoc) {});
}
})(i));
}
}

You could use recursion instead of a loop, that way you would be sure the next iteration won't execute until the first is done.
var proc = function (i, count, table) {
var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC
LIMIT ' + i + ' , ' + inc'
console.log(i)
exporter.json(sql, function (err, json) {
var data = JSON.parse(json)
db.insert(data, function (err, newDoc) {
if (i < count) {
i += inc
proc(i, count, table)
}
})
})
}
var toNeDB = function (table, count) {
var inc = 10000
console.log(table + ' => ' + count)
proc(0, count, table)
}
let me know if that works

Related

Storing mysql query rows in variable for later use in another mysql query

I have 2 different tables where the first table is used for retrieving the data to store in the second table, so for example, if there are 3 items in the first table, ill run a for loop 3 times to retrieve the information and storing it in the second table row by row, i also have to use a unique id for the primary key for the second table, so i have to run another mysql query to retrieve all the ID in the second table and add 1 to the id to generate a new ID, but the problem for me is that when i try to put a connection.query in another connection.query, the outer connection.query runs 3 time before running the inner connection.query, hence the items does not get stored and the id does not update, giving me a duplicate primary key error, i tried using functions but it doesnt help, here is
app.get('/submit-check', function (request, response) {
function insert(sql2) {
connection.query(sql2, function (err, result1) {
if (err) throw err;
});
}
function other(value, index) {
let sql = "SELECT * FROM ORDERS"
connection.query(sql, function (err, results) {
var id;
if (results.length == 0) {
id = 0
} else {
id = results[results.length - 1].orderID;
}
// for (let j = 0; j < results.length; j++) {
// list.push(results[j].orderID)
// }
// if (list.length == 0) {
// id = 0
// }
// else {
// var largest = 0;
// for (let i = 0; i <= list.length; i++) {
// if (parseInt(list[i]) > largest) {
// var largest = parseInt(list[i]);
// }
// }
// id = largest + 1
// }
id = id + 1;
var add = request.query.state + " " + request.query.address + " " + request.query.Unumber + " " + request.query.zip
var custID = value[index].custID
var bookID = value[index].bookID
var Email = request.query.email
var CardName = request.query.cardname
var CardNumber = request.query.cardnumber
var ExDate = request.query.expmonth + "/" + request.query.expyear
var CVV = request.query.cvv
var qty = 1
var sql2 = "INSERT INTO orders (orderID,custID, bookID, QTY, CardName, CardNumber, ExDate, CVV, Email, Address, createdAt, updatedAt) VALUES('" + id + "','" + custID + "', '" + bookID + "', '" + qty + "', '" + CardName + "', '" + CardNumber + "', '" + ExDate + "', '" + CVV + "', '" + Email + "', '" + add + "', CURDATE(), CURDATE() )"
insert(sql2)
})
}
function setValue(value) {
for (let index = 0; index < value.length; index++) {
other(value, index)
}
}
let sql3 = "SELECT * FROM carts"
var cart_db;
connection.query(sql3, function (err, result) {
cart_db = result
setValue(cart_db)
})
// console.log(cart_db)
response.render("aftercheck", { attributes: request.query, cardno: "****-****-****" + request.query.cardnumber.slice(-5,) });
})
thank you in advance
There are many, many things wrong with your code. I've peppered this await/async-based rewrite with TODO comments where you'll probably need to fix up things. Naturally I haven't been able to test this since I don't have your database.
async function queryP(connection, sql, parameters = undefined) {
return new Promise((resolve, reject) => {
connection.query(sql, parameters, function (err, results) {
if (err) return reject(err);
resolve(results);
});
});
}
app.get("/submit-check", async function (request, response) {
// TODO: these need to be validated
const {
address,
Unumber,
zip,
expyear,
state,
expmonth,
email: Email,
cardname: CardName,
cardnumber: CardNumber,
cvv: CVV,
} = request.query;
const add = `${state} ${address} ${Unumber} ${zip}`;
const ExDate = `${expmonth}/${expyear}`;
const cartData = await queryP(connection, "SELECT * FROM carts"); // TODO: this is not safe with multiple customers
const [maxOrderIdRow] = await queryP(connection, "SELECT MAX(id) as maxid FROM orders");
const orderId = (maxOrderIdRow.maxid || 0) + 1; // TODO: this is not safe in the face of concurrent requests
// Using a for loop rather than .map and await and so on is simpler here
for (let i = 0; i < cartData.length; i++) {
const cartRow = cartData[i];
const custID = cartRow.custID;
const bookID = cartRow.bookID;
const qty = 1;
// TODO: rethink how credit cards are saved – we can't save them in the database like this or we'll be out of business
await queryP(
connection,
`INSERT INTO orders (orderID, custID, bookID, QTY, CardName, CardNumber, ExDate, CVV, Email, Address, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURDATE(), CURDATE() )`,
[orderId, custID, bookID, qty, CardName, CardNumber, ExDate, CVV, Email, add],
);
}
response.render("aftercheck", {
attributes: request.query,
cardno: "****-****-****" + CardNumber.slice(-5),
});
});

Filter data obtained through GitHub API

I created this function to obtain GitHub issues:
retrieveEnerpriseIssues: function(repoOrg, repoName, callback) {
let data = null;
// token auth
octokit.authenticate({
type: 'basic',
username: config.githubEnterprise.username,
password: config.githubEnterprise.token
});
async function paginate(method) {
let response = await method({
q: "repo:" + repoOrg + "/" + repoName + " is:issue",
per_page: 100
});
data = response.data.items;
var count = 0;
while (octokit.hasNextPage(response)) {
count++;
console.log(`request n°${count}`);
response = await octokit.getNextPage(response);
data = data.concat(response.data.items);
}
return data;
}
paginate(octokit.search.issues)
.then(data => {
callback(data);
})
.catch(error => {
console.log(error);
});
}
It is called in this function which takes the issues, filters out all of the unwanted keys into json format and puts it in my db.
extractToDb: function() {
let gitIssues = null;
for(var i = 0; i < config.githubEnterprise.orgs.length; i++) {
for(var j = 0; j < config.githubEnterprise.orgs[i].repos.length; j++) {
gitHubService.retrieveEnerpriseIssues(
config.githubEnterprise.orgs[i].owner,
config.githubEnterprise.orgs[i].repos[j].repoName,
function(data, err) {
if(err) {
console.log('err: ', err);
} else {
gitIssues = data;
}
gitIssues = JSON.stringify(gitIssues);
gitIssues = JSON.parse(gitIssues);
let issueFormatForDb = null;
for(var i = 0; i < gitIssues.length; i++) {
issueFormatForDb = gitIssues[i];
const body = '{' +
'"github_id": "' + issueFormatForDb.id + '",' +
'"issue_title": "' + issueFormatForDb.title + '",' +
'"issue_number": "' + issueFormatForDb.number + '",' +
'"issue_url": "' + issueFormatForDb.url + '",' +
'"issue_state": "' + issueFormatForDb.state + '"' +
'}';
console.log('Body: ', body);
getGitHubIssues.postToDb(body);
}
});
}
}
}
I'd like to take this a step further by filtering out any issues where the state is closed. How is this done and should it be handled in my retrieveEnerpriseIssues function or my extractToDb?
Possible solution
I tried this in my extractToDb function:
gitIssues = JSON.parse(gitIssues);
gitIssues = _.where(gitIssues, {state: "open"});
let issueFormatForDb = null;
Is it the best solution or is there a better way?
As #givehug stated:
Better use _.filter, or native filter method like
gitIssues = gitIssues.filter(i => i.state === 'open')
I think .where was deprecated in later versions of lodash github.com/lodash/lodash/wiki/Deprecations. Other than that its perfectly fine.
I just realsied I can filter the state in my paginate function with this:
let response = await method({
q: "repo:" + repoOrg + "/" + repoName + " is:issue" + " label:issue_label" + " state:open",
per_page: 100
});

Retrieve data from firebase resulting in undefined

I was having this problem when trying to pull from firebase. My database structure as such:
I am trying to pull the merchantName and its relevant branches details:
function getAllMerchant(){
var query = firebase.database().ref('merchants');
return query.once('value').then(data => {
var result = [];
data.forEach(snapshot => {
var merchantData = snapshot.val();
var merchantName = merchantData.merchantName;
var branchName = merchantData.branches.branchName;
var branchAddress = merchantData.branches.branchAddress;
console.log('check ' + merchantName + ' ' + branchName + ' ' + branchAddress);
result.push({merchantName: merchantName, branchName: branchName, branchAddress: branchAddress});
});
return result;
});
resolve(result);
}
However, when I printed out the branchName and branchAddress, I am getting undefined. I managed to print out merchantName though. Any ideas?
Thanks!
You're not iterating over the branches of the merchant.
data.forEach(merchantSnapshot => {
var merchantData = snapshot.val();
var merchantName = merchantData.merchantName;
console.log('check ' + merchantName);
merchantSnapshot.child("branches").forEach(brancheSnapshot => {
var branchName = brancheSnapshot.val().branchName;
var branchAddress = brancheSnapshot.val.branchAddress;
console.log(' ' + branchName + ' ' + branchAddress);
});
});

Datatables row reorder event issue

var table =$("#exampleList").DataTable({
paging:false,
rowReorder:false
}
});
table.on('row-reorder',function(e, diff,edit){
for(var i=0, ien = diff.length ; i<ien ; i++){
var rowData = table.row(diff[i].node).data();
sequence = sequence + "_" + rowData[0];
}
var data = table.rows().data();
data.each(function (value, index) {
alert('Data in index: ' + index + ' is: ' + value);
});
});
Hi,
I am new to datatables. Issue I am having right now is I cant get the latest value in my table after the user reorder the row. The code above only shows the value before the reorder occurs. I need to get the latest reorder sequence so I can update the database.
What you need to do is wait a few milliseconds before trying to read the data.
table.on('row-reorder',function(e, diff, edit){
for(var i=0, ien = diff.length ; i<ien ; i++){
var rowData = table.row(diff[i].node).data();
sequence = sequence + "_" + rowData[0];
}
setTimeout(()=> { lookAtData() }, 10);
});
function lookAtData() {
var data = table.rows().data();
data.each(function (value, index) {
alert('Data in index: ' + index + ' is: ' + value);
});
}
You should use column-reorder not row-reorder.
Please try :
var rdata = table .columns().order().data();
console.log(rdata);
It will get the data after columns ordering.

How can I pass values from javascript var to executeSql?

How can I pass values from javascript var to executeSql?
I wrote this code:
function BuscaRegistros() {
var mydata=document.getElementById("lugar").value;
var mydata2=document.getElementById("fecha").value;
var db = openDatabase('sightings03', '1.0', 'Test DB', 2 * 1024 * 1024);
db.transaction(function (tx) {
tx.executeSql('SELECT CommonName FROM Sights03 WHERE location=**"mydata"** AND datte="**mydata2**"' , [], function (tx, results) {
var len = results.rows.length, i;
for (i = 0; i < len; i++){
document.despliega.desp.options[i]=new Option(results.rows.item(i).CommonName,i);
}
}, null);
});
}
Are you asking how to concatenate strings? You do that with the + operator.
In general it's like this:
var mydata = "hello";
var string = "the value is: " + mydata;
console.log(string); // the value is: hello
So you might do:
tx.executeSql('SELECT CommonName FROM Sights03 WHERE location="' + mydata + '" AND datte="' + mydata2 + "' , [], function (tx, results) {

Categories