I am calling function in order to insert some values into a database table using mysql module of node.js. The function is called inside a for loop and I can get the last insertId of each INSERT query inside the function.
MY PURPOSE
I want to store these id's to an array in to order to use them later in my code but I cannot do that.
- FUNCTION CALL
for(var i=0;i<some_number;i++){
if (<something is true>){
var lastid = [];
//Function calling
function_a(x, y, z, w, function(error, result) {
if(!error){
lastid.push(result.insertId);
}
//To Do
else{}
});
}
}
- FUNCTION
function function_a(a, b, c, d, callback){
//Connection to the DB
const con = mysql.createConnection({
host: 'host',
user: 'user',
password: 'password',
database: 'database'
});
//DB query
var query = "INSERT INTO my_table (column1, column1, column1, column1) VALUES ('"+a+"', '"+b+"', '"+c+"', '"+d+"')";
con.query(query, (err, rows) => {
var e = rows.insertId;
return callback(err, e);
});
con.end();
}
The problem is that I cannot use the id's outside the function.
Any idea how can I do that?Is this about variable scope?
Several problems here, and yes, they are partly about variable scope.
Put var lastid = []; before the for loop, so you can use it after the loop. However, as you have callback mechanism inside the loop, you must wait for the whole loop to finish (all callbacks called from 0 to some_number-1, in order to have the right values in lastid array. It's a messy thing with callbacks, but you can do it with them or with promises. You can Google for callback handling in for loops.
The function function_a is a asynchronous function. In your case you can use async module to do this task:
var taskIds = Array.apply(null, {length: some_number}).map(Number.call, Number);
var lastid = [];
async.eachLimit(taskIds, 32, function (taskId, done) {
if (<something is true>) {
function_a(x, y, z, w, function(error, result) {
if(!error){
lastid.push(result.insertId);
}
//To Do
else{}
done()
});
} else {
done()
}
}, function () {
// Do some thing with lastid
console.log(lastid);
});
Related
I am writing a small Node js application for automatic vehicle location system.
Here is the code for where I am getting trouble.
markerData contains 4 rows but only in the log I can see the last row.
for (var i = 0, len = markerData.length; i < len; i++) {
var thisMarker = markerData[i];
sql.connect(config, function (err) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
}
});
});
}
Please help me to solve the issue.
As var is not a block level variable in terms of scope, when `sql' module takes time to connect to the database asynchronously, the synchronous loop may change the value of the variable that's why you have the last row printed since the variable holds the reference to the last object at the time of successful connection.
Instead of _.each, I would recommend to use async module with async.each since you have few asynchronous operation to get rid of a synchronous loop.
You can check for samples here,
http://justinklemm.com/node-js-async-tutorial/
Here is your updated code with async.each
-> Install async module with npm install async --save
-> Then add the below reference in the required place,
// Reference
var async = require('async');
-> Modified code:
sql.connect(config, function (err) {
if(err) {
console.log('Connection error: ');
console.log(err);
} else {
async.each(markerData, function(thisMarker, callback) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if(err) {
console.log(err);
callback();
} else {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
} else {
console.log('Recordset empty for id: ' + thisMarker.id);
}
callback();
}
});
}, function(err){
if(err) {
console.log(err);
}
});
}
});
I'm not entirely sure how your library works, but presumably recordset2 is an array of records. recordset2[0] is therefore the first record. If you want the next one you should probably try recordset2[1] and so on and so forth.
Arrays: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array
You'll probably need to loop through all the elements in the array at some point. use a for loop for that:
for (var i = 0; i < recordset2.length; i++ {
console.log(recordset2[i])
}
That will print out everything your query returns.
Is any possible to increment var licznik in this block of code?
I try sth like this, But always receives 0. Could someone explain me what I'm doing wrong?
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
if(item.market_hash_name == record.real_name) {
var asid = item.assetid;
(function(licznik){
connection.query('SELECT count(id) as wynik FROM used where asset_id = \'' + asid + '\'', function(err, wiersze) {
if (wiersze[0].wynik == 0) {
var employee = {
asset_id: asid,
trans_id: record.tid
};
connection.query('INSERT INTO used SET ?', employee, function(err, res) {
if (err) throw err;
offer.addMyItem(item);
console.log(licznik);
&licznik++;
});
}
});
})(licznik);
}
}
});
});
As the comment on your original question points out, I have no context for what this code is actually trying to accomplish. What I can tell you is that your callbacks supplied to connection.query are NOT fired on each iteration of the forEach. The whole reason connection.query takes a callback is because you don't know when the operation will complete. Node is designed to be asynchronous so all it does on each iteration of the forEach loop is begin the query. The callback supplied to the query could be invoked at any time which also means that a query that fired after another query could potentially fire its callback before the callback from the first query. It just depends on how long each query takes.
If you need licznik to be incremented on every iteration of the forEach then you need to increment it after your if statement.
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
// .... omitted for brevity
}
licznik++; // <-- increment here, outside of the closure.
});
});
Again, I have zero clue what you're actually trying to do with that variable so this may not solve your real problem, but that's the way to get it to increment in that loop.
PS - You may not be understanding that you actually have two licznik variables here. You create a new one when you wrap all your logic in a closure function like you did. If you change the variable declared at the top of your closure function you'll see that it's not the same variable as the one outside the closure function.
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
if(item.market_hash_name == record.real_name) {
var asid = item.assetid;
(function(licznik2) { // <-- notice this is a new variable
connection.query('SELECT count(id) as wynik FROM used where asset_id = \'' + asid + '\'', function(err, wiersze) {
if (wiersze[0].wynik == 0) {
var employee = {
asset_id: asid,
trans_id: record.tid
};
connection.query('INSERT INTO used SET ?', employee, function(err, res) {
if (err) throw err;
offer.addMyItem(item);
console.log(licznik2);
licznik2++;
});
}
});
})(licznik);
}
}
});
});
I finally figured out how callbacks work in node.js, but I'm trying now to get my code to execute in order.
The goal is to (in order):
Load the URL into cheerio
Parse through each <td> in the <tbody> on the page.
Once text elements are loaded into the data array, callback.
Call loopThroughData on the full Data array.
Loop through the data array and call the lookForPlayer array on each
one, which:
Runs a SELECT in my db that matches the player name passed from the
text element, and if there is no match in my db, INSERT them (I have
it just printing to the console for now for testing purposes).
The end goal is to go through every page (there is a separate URL for each date, so I am looping through the dates) and INSERT players that aren't in my database ONCE. The problem is that it goes through each SELECT before the INSERT queries are executed, so it's inserting them multiple times.
Here is the page I'm parsing, if it helps: http://www.basketball-reference.com/friv/dailyleaders.cgi?month=12&day=29&year=2014
Here is my code:
function loadPage (url, callback){
request(url, function(err, response, body){
if(!err && response.statusCode ==200){
var $ = cheerio.load(body);
rowsRemaining = $.length;
$('td', 'tbody').each(function(){
var text = $(this).text();
data.push(text);
rowsRemaining -= 1;
console.log('rows left: ',rowsRemaining);
});
}
if (rowsRemaining == 0){
console.log('$ length: ',$.length);
callback(data);
}
});
}
function loopThroughData (data, callback){
for(i=1;i<data.length;i+=26){
lookForPlayer(data[i].replace("'",""),function(name){
/* var insertPlayer = connection.query(
'INSERT INTO player (provider_id, team_id, position_id, name) VALUES (1, (SELECT id FROM team WHERE slug = "'+data[i+1]+'"),1,"'+name+'");',function(err,result,fields){
}); */
console.log('i is currently = ',i);
});
}
callback();
}
function lookForPlayer(name, callback){
console.log('Looking for Player...');
var selectPlayer = connection.query(
"SELECT * FROM player WHERE name = '"+name+"'", function(err, rows, fields){
if(err) throw err;
if(rows.length==0){
callback(name);
}
});
}
//loop through every day since the season started
for (d = seasonStart; d <= Date.now(); d.setDate(d.getDate() + 1)){
console.log('d = ',d);
loadPage(baseURL+(d.getMonth()+1)+'&day='+d.getDate()+'&year='+d.getFullYear(),function(data){
console.log('Page loaded...');
loopThroughData(data,function(){
});
});
}
As you can see, I tried adding a rowsRemaining variable that is meant to make sure I've parsed the whole file before calling the callback in the loadPage function, but it never gets to that point. Note that I initialize a lot of these variables before these functions (rowsRemaining, data, etc).
It also seems to loop through every date before fully loading, parsing, and INSERTing the first page, which it should not be doing.
Here is the updated code based off of #Brant's answer
function loadPage (url, callback){
request(url, function(err, response, body){
if(!err && response.statusCode ==200){
var $ = cheerio.load(body);
console.log(url);
$('td', 'tbody').each(function(){
var text = $(this).text();
data.push(text);
});
}
callback(data);
});
}
function loopThroughData (data, callback){
for(i=1;i<data.length;i+=26){
lookForPlayer(data[i].replace("'",""),function(name){
var insertPlayer = connection.query(
'INSERT INTO player (provider_id, team_id, position_id, name) VALUES (1, (SELECT id FROM team WHERE slug = "'+data[i+1]+'"),1,"'+name+'");',function(err,result,fields){
});
});
}
callback(data);
}
function lookForPlayer(name, callback){
var selectPlayer = connection.query(
"SELECT * FROM player WHERE name = '"+name+"'", function(err, rows, fields){
if(err) throw err;
if(rows.length==0){
console.log(name,' was not found in DB!');
callback(name);
}
});
}
//loop through every day since the season started
for (d = seasonStart; d <= Date.now(); d.setDate(d.getDate() + 1)){
validDatesArr.push(d);
}
async.eachSeries(validDatesArr,
function(validDatesArr, callback){
loadPage(baseURL+'/month='+validDatesArr.getMonth()+1+'&day='+validDatesArr.getDate()+'&year='+validDatesArr.getFullYear(),function(data){
loopThroughData(data, function(){
callback();
});
});
}, function(err){
if(!err){
console.log('We processed each date requests one by one');
}
}
);
So now it's loading the pages one by one, but it isn't executing the INSERT function in the loopThroughData function on that data. I would think I would just add another function to the async list, but this particular one is calling a function as opposed to using an anonymous one.
Modify your for loop to be as follows:
//loop through every day since the season started
var validDatesArr = [];
for (var d = seasonStart; d <= Date.now(); d.setDate(d.getDate() + 1)){
validDatesArr.push(d);
}
async.eachSeries(validDatesArr,
function(d, callback) {
loadPage(baseURL+(d.getMonth()+1)+'&day='+d.getDate()+'&year='+d.getFullYear(),function(data){
console.log('Page loaded...');
loopThroughData(data,function(){
callback();
});
});
}, function(err) {
if(!err) {
console.log('We processed each date request one by one')
}
}
);
And require async which can be found here: https://github.com/caolan/async
npm install async
You can nested the Async function to control the execute flow like in a sequence programming, be careful at the Pyramid of doom, the other solution is to use the Sync version of the async functions you used (if exist). You are not forced to write Async function if you do NOT need them, Node.js use a lot of Async function because is a Non-bloking language very powerful for web development. So do NOT use the asyn style and the callback in your functions !
I'm trying to write a Node program that populates my MySQL database with data from files I have on disk. I may or may not be going about this the right way, but it's working. What I'm having trouble with is understanding how I should be handling allowing asynchronous functions to finish before the connection to the DB is ended. Ultimately, I'll be reading lots of data files, and insert them into the database like I did below. I could just use readFileSync instead of the asynchronous version, but I need to get a better handle on asynchronous functions.
When I insert the wine categories below, it works fine since it's not using an asynchronous function. However, when I use readFile to get data from a file, I get an error that connection ended before any of the queries were executed:
connection.connect( function(err) {
if(err) {
console.log(err);
}
});
// Take a table and the values, and insert a new row into a table
function insert_into( table, values ) {
if( values instanceof Array ) {
values = values.map( function( value ) {
return '"' + value + '"';
}).join(', ');
} else {
values = '"' + values + '"';
}
var statement = 'INSERT INTO ' + table + ' VALUES (NULL, ' + values + ')';
connection.query( statement, function(err, rows, fields) {
if (err) throw err;
console.log( values + " successfully added.");
});
};
// Populate the wine_categories table
var wine_categories = [
'red', 'white', 'rose', 'sparkling', 'fortified'
];
// Works fine when used alone
wine_categories.forEach( function( element ) {
insert_into( 'wine_categories', element );
});
// Populate the countries table
// connection.end() runs before this finishes its job
fs.readFile( countries, 'utf8', function (err, data) {
if (err) {
throw err;
} else {
var codes = Array.prototype.map.call(
data.split('\n'), function( country ) {
return country.split('\t');
});
codes.forEach( function( country ) {
if( country[1].length > 25 ) {
country[1] = country[1].substring(0, 25);
}
insert_into( 'countries', country );
});
}
});
connection.end();
Obviously, connection.end() needs to happen after all of the inserts have completed, but I'm not sure how to handle that. I don't want it to be a callback for the readFile call because I'll ultimately have many of similar calls in this file.
How should I structure my code so that all of the queries execute and connection.end() runs when they're all finished? The answer is probably obvious to an asynchronous wiz...
Using promises it would be like this:
pool.getConnectionAsync().then(function(connection) {
// Populate the wine_categories table
var wine_categories = [
'red', 'white', 'rose', 'sparkling', 'fortified'
];
var wineQueries = wine_categories.map(function(wine){
return insert_into(connection, "wine_categories", wine);
});
var countryQueries = fs.readFileAsync(countries, "utf-8").then(function(data) {
return data.split("\n").map(function(country) {
country = country.split("\t")[1];
if (country.length > 25) {
country = country.substring(0, 25);
}
return insert_into(connection, "countries", country);
});
});
Promise.all(wineQueries.concat(countryQueries))
.then(function() {
console.log("all done");
})
.catch(function(e) {
console.log("error", e);
})
.finally(function() {
connection.release();
})
});
Pre-requisite code for the above
var Promise = require("bluebird");
var fs = Promise.promisifyAll(require("fs"));
Promise.promisifyAll(require("mysql/lib/Connection").prototype);
var pool = Promise.promisifyAll(require("mysql").createPool({
"user": "...",
"password": "...",
"database": "...",
"host": "localhost",
"port": 3306,
"debug": false
}));
function insert_into(connection, table, values) {
if( values instanceof Array ) {
values = values.map(connection.escape, connection).join(', ');
} else {
values = connection.escape(values);
}
return connection
.queryAsync('INSERT INTO ' + table + ' VALUES (NULL, ' + values + ')')
.then(function() {
console.log(values + " successfully added.");
});
}
Assuming that insert_into is also asynchronous, you may want to use something like async.each to handle inserting your records. It has a convenient callback that will be called when all records are inserted, because only at that point do you want to close the connection:
async.each(codes, function(country, callback) {
if ( country[1].length > 25 ) {
country[1] = country[1].substring(0, 25);
}
insert_into( 'countries', country, callback ); // !! read below
}, function(err) {
// TODO: handle any errors
...
// Here, all countries are inserted.
connection.end();
});
However, this means that insert_into should also be made to accept a callback (using the common Node convention function(err, result)) that will be called when the record has been inserted. In the code above, I'm using the callback provided by async directly, meaning that once your insert_into is done, it will call the async callback signaling that this iteration of each is done.
EDIT: you can rewrite insert_into so it looks like this:
function insert_into( table, values, callback ) {
...
connection.query(..., function(err) {
callback(err);
});
}
Since you don't need the actual result from connection.query, you only have to pass err (instead of throwing it).
Tip: assuming that you're using node-mysql, you may want to take a look at the docs on how it can help you with escaping.
I have a function in my express app that makes multiple queries within a For Loop and I need to design a callback that responds with JSON when the loop is finished. But, I'm not sure how to do this in Node yet. Here is what I have so far, but it's not yet working...
exports.contacts_create = function(req, res) {
var contacts = req.body;
(function(res, contacts) {
for (var property in contacts) { // for each contact, save to db
if( !isNaN(property) ) {
contact = contacts[property];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err) { console.log(err) };
}); // .save
}; // if !isNAN
}; // for
self.response();
})(); // function
}; // contacts_create
exports.response = function(req, res, success) {
res.json('finished');
};
There are a few problems with your code besides just the callback structure.
var contacts = req.body;
(function(res, contacts) {
...
})(); // function
^ you are redefining contacts and res in the parameter list, but not passing in any arguments, so inside your function res and contacts will be undefined.
Also, not sure where your self variable is coming from, but maybe you defined that elsewhere.
As to the callback structure, you're looking for something like this (assuming contacts is an Array):
exports.contacts_create = function(req, res) {
var contacts = req.body;
var iterator = function (i) {
if (i >= contacts.length) {
res.json('finished'); // or call self.response() or whatever
return;
}
contact = contacts[i];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err)
console.log(err); //if this is really a failure, you should call response here and return
iterator(i + 1); //re-call this function with the next index
});
};
iterator(0); //start the async "for" loop
};
However, you may want to consider performing your database saves in parallel. Something like this:
var savesPending = contacts.length;
var saveCallback = function (i, err) {
if (err)
console.log('Saving contact ' + i + ' failed.');
if (--savesPending === 0)
res.json('finished');
};
for (var i in contacts) {
...
newContact.save(saveCallback.bind(null, i));
}
This way you don't have to wait for each save to complete before starting the next round-trip to the database.
If you're unfamiliar with why I used saveCallback.bind(null, i), it's basically so the callback can know which contact failed in the event of an error. See Function.prototype.bind if you need a reference.