I am closing my writing connection in a "finally" block but even after going into this block the program seems to go back into the try block.
Here is the code :
function printAllAssetsAndThreats(startX, startY) {
try {
con.query("SELECT * FROM Kunde1Assets;", function (err, result, fields) {
for (var i in result) {
console.log("b");
myDoc.font('Helvetica-Bold')
.fontSize(20)
.text('str', startX, startY);
var xxx = result[i].KundenAssetID;
startY = startY + 50;
//jetzt jeweils alle gefährdungen:
var sql = "SELECT DISTINCT c.AID, b.GID, b.Name, c.Name AS Asset FROM Kunde1Verbindungen a, Gefährdungen b, Kunde1Assets c WHERE a.KundenAssetID = \"" + xxx + " \"AND a.GID = b.GID AND c.KundenAssetID = a.KundenAssetID";
con.query(sql, function (err, result2, fields) {
for (var i2 in result2) {
}
});
///////////////////
}
});
} catch (e) {
} finally {
console.log("a");
end();
}
}
So even after printing "a" there´ll be "b" printed in the console.
Thank you for your help !!!
Tim
EDIT
I have tried rewriting the code f.e. with callbacks but it´s still not working as it is supposed to do
start(()=>{
myDoc.end();
});
function start(callback){
console.log("hello");
myDoc.font('Helvetica-Bold')
.fontSize(20)
.text('str', 44,44);
con.query("SELECT * FROM Kunde1Assets;", function (err, result, fields) {
console.log("hello");
var counter=0;
for(var i in result){
console.log("hello");
start2(result[i].KundenAssetID, ()=>{});
counter++;
if (counter==result.length){
console.log("yo");break;callback();
}
if (counter==result.length){
console.log("yo");callback();
}
}
//
});
}
function start2(kaid, callback){
var sql = "SELECT DISTINCT c.AID, b.GID, b.Name, c.Name AS Asset FROM Kunde1Verbindungen a, Gefährdungen b, Kunde1Assets c WHERE a.KundenAssetID = \"" + kaid + " \"AND a.GID = b.GID AND c.KundenAssetID = a.KundenAssetID";
con.query(sql, function (err, result2, fields) {
for (var i2 in result2) {
console.log(kaid +"---"+ result2[i2].Name);
myDoc.font('Helvetica-Bold')
.fontSize(20)
.text('str', 44,44);
}
});
}
EDIT:
Tried it with promises (not working yet)
myDoc.pipe(fs.createWriteStream('node.pdf'));
var promise = start();
promise.then(function(result){
console.log(result);
myDoc.end();
})
function start(){
return new Promise (function(resolve, reject){
con.query("SELECT * FROM Kunde1Assets;", function (err, result, fields) {
console.log(result);
for (var i in result){
myDoc.font('Helvetica-Bold')
.fontSize(20)
.text(result[i].Name, 30, 20+(i*30));
var promise2 = start2 (result[i].KundenAssetID);
promise.then(function(name){
for (var i2 in name){
myDoc.font('Helvetica-Bold')
.fontSize(20)
.text('result[i].Name', 30, 20+(i2*30));
}
});
}
resolve(result);
if (Error) reject();
});
});
}
function start2(kaid){
return new Promise(function( resolve, reject){
var sql = "SELECT DISTINCT c.AID, b.GID, b.Name, c.Name AS Asset FROM Kunde1Verbindungen a, Gefährdungen b, Kunde1Assets c WHERE a.KundenAssetID = \"" + kaid + " \"AND a.GID = b.GID AND c.KundenAssetID = a.KundenAssetID";
con.query(sql, function (err, result2, fields) {
for (var i2 in result2) {
//console.log(kaid +"---"+ result2[i2].Name);
}
resolve(result2);
if (Error) reject();
});
});
}
Your try/catch/finally block is running synchronously. It will hit the finally within the first tick of the JS event loop. However (I assume, based on the structure of the callback), the con.query() function is asynchronous. That means that it will run in at least the tick after your finally. Therefore, this will always run after everything inside the finally.
If you want it to work as expected, look into asynchronous methods of achieving the try/catch/finally structure. There are million ways to skin a cat in regards to this, the async library, promises, etc. Or just keep any code you want to run after the connection inside the connection callback.
Either way, the issue is a combination of sync and async code running alongside each other.
Related
I'm trying to generate a hashchain using the following code:
var async = require('async');
var _ = require('lodash');
var offset = 1e7;
var games = 1e7;
var game = games;
var serverSeed = '238asd1231hdsad123nds7a182312nbds1';
function loop(cb) {
var parallel = Math.min(game, 1000);
var inserts = _.range(parallel).map(function() {
return function(cb) {
serverSeed = genGameHash(serverSeed);
game--;
query('INSERT INTO `hash` SET `hash` = ' + pool.escape(serverSeed));
};
});
async.parallel(inserts, function(err) {
if (err) throw err;
// Clear the current line and move to the beginning.
var pct = 100 * (games - game) / games;
console.log('PROGRESS: ' + pct.toFixed(2) + '%')
if (game > 0){
loop(cb);
}else {
console.log('Done');
cb();
}
});
}
loop(function() {
console.log('Finished with SEED: ', serverSeed);
});
When I run this code it generates a hash chain of 1k hash's, while I'm trying to generate a chain of 1m hash's. It seems like async isn't working properly, but I have no idea why, there are no errors in console, nothing that points out a flaw.
Any ideas?
Do you can run it with smaller games (about 3000)?
Your parallel function nerver send done signal because the callback of inserts item never trigged. I think query function has two pramasters query(sql: string, callback?: (err, result) => void) (Typescript style).
I suggest you change your logic and flow like below block code:
var inserts = _.range(parallel).map(function() {
return function(cb) {
serverSeed = genGameHash(serverSeed);
query('INSERT INTO `hash` SET `hash` = ' + pool.escape(serverSeed), function(err, result) {
if(result && !err) {
game--;
}
cb(); // remember call the callback
});
};
});
In your code, you have used async.parallel, I think it is not good idea, too many connection has be open(1m). Recommeded for this case is parallelLimit
I am writing a small Node js application for automatic vehicle location system.
Here is the code for where I am getting trouble.
markerData contains 4 rows but only in the log I can see the last row.
for (var i = 0, len = markerData.length; i < len; i++) {
var thisMarker = markerData[i];
sql.connect(config, function (err) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
}
});
});
}
Please help me to solve the issue.
As var is not a block level variable in terms of scope, when `sql' module takes time to connect to the database asynchronously, the synchronous loop may change the value of the variable that's why you have the last row printed since the variable holds the reference to the last object at the time of successful connection.
Instead of _.each, I would recommend to use async module with async.each since you have few asynchronous operation to get rid of a synchronous loop.
You can check for samples here,
http://justinklemm.com/node-js-async-tutorial/
Here is your updated code with async.each
-> Install async module with npm install async --save
-> Then add the below reference in the required place,
// Reference
var async = require('async');
-> Modified code:
sql.connect(config, function (err) {
if(err) {
console.log('Connection error: ');
console.log(err);
} else {
async.each(markerData, function(thisMarker, callback) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if(err) {
console.log(err);
callback();
} else {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
} else {
console.log('Recordset empty for id: ' + thisMarker.id);
}
callback();
}
});
}, function(err){
if(err) {
console.log(err);
}
});
}
});
I'm not entirely sure how your library works, but presumably recordset2 is an array of records. recordset2[0] is therefore the first record. If you want the next one you should probably try recordset2[1] and so on and so forth.
Arrays: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array
You'll probably need to loop through all the elements in the array at some point. use a for loop for that:
for (var i = 0; i < recordset2.length; i++ {
console.log(recordset2[i])
}
That will print out everything your query returns.
I posted a question before and realized my problem actually was async functions. I managed to work out most of it, but I got one little problem left. Using async I used waterfall to create an order for the some queries...
exports.getMenu = function(id_restaurant, callback){
async.waterfall([
async.apply(firstQuery, id_restaurant),
secondQuery,
thirdQuery,
fourthQuery,
formMenu
], function(err, result){
if(err){
console.log(err);
}
callback(result);
});
};
Everything works until fourthQuery, where I have to loop to get all dishes of a menu.
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if(err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes[i] = results2;
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
};
So what i can see that happens from printing the value of 'i' each loop is that it always has the value of 2. Because that's 'size' value. Also, even though it's saving results of index '2' I believe the callback is being done even before the for loop is done, because my fifth function is recieving an empty array.
How can i make my code wait to callback until my for loop is done?
NOTE: Sorry, part of my code is in spanish, tried to translate the important parts of it.
There are a few ways to handle this, one is to look into promise architecture. Promise.all will let you supply one callback to handle the values from each child promise.
To use what you've already got, however, I'd push the values into your dishes array, rather than assigning them specifically to i indexes, then check the size of that array at the end of each connection. When the array length matches the size, fire the callback. (as seen below)
If you need a way to tie each result to that specific i value, I'd recommend pushing them as an object
dishes.push({'index': i, 'dish': results2})
Afterward, if you need the array of just dishes, you can sort the array by that index value and run a map function.
dishes.sort(function(a,b){ return a.index - b.index; })
dishes = dishes.map(function(a){ return a.dish })
Here's the code adjusted:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback) {
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if (err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if (err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2)
if(dishes.length == size){
console.log("I'm sending " + dishes);
callback(null, dishes, array_nombresSecc)
}
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
;
};
Since you're already using the async, I would suggest replacing the for() loop in fourthQuery with async.each().
The updated fourthQuery would look like this:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
async.each(array_secciones,
function(item, itemCallback) {
// Function fun for each item in array_secciones
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [item],
function(err, results2) {
if(err) {
console.log(err);
return itemCallback(true);
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2);
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[dishes.length-1]));
// this prints the result but only if it has a value over 2
return itemCallback();
});
},
function(err) {
// Function run after all items in array are processed or an error occurs
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
});
}); // pool
};
Alternatively, you can use async.map(), which handles gathering the results in the final callback so doesn't rely on the dishes variable.
I'm trying to scrape data from a word document with node.js.
My current problem is that the below console log will return the value inside the juice block as the appropriate varaible. If I move that to outside the juice block it is completely lost. I tried putting return
function getMargin(id, content){
var newMargin = content.css("margin-left");
if(newMargin === undefined){
var htmlOfTarget = content.toString(),
whereToCut = theRaw.indexOf("<div class=WordSection1>");
fs.writeFile("bin/temp/temp_" + id + ".htm", theRaw.slice(0, whereToCut) + htmlOfTarget + "</body> </html>", function (err){
if (err) {
throw err;
}
});
juice("bin/temp/temp_" + id + ".htm", function (err, html) {
if (err) {
throw err;
}
var innerLoad = cheerio.load(html);
newMargin = innerLoad("p").css("margin-left");
console.log(newMargin); // THIS newMargin AS VALUE
});
}
console.log(newMargin);//THIS RETURNS newMargin UNDEFINED
return newMargin;
}
I think the problem lies with fs.write and juice being Asyc functions. I just have no idea how to get around it. I have to be able to call getMargin at certain points, in a sequential order.
As mentioned in comment, change your program flow to run in callbacks, after async code has completed...
// accept callback as parameter, and run it after async methods complete...
function getMargin(id, content, callback){
var newMargin = content.css("margin-left");
if(newMargin === undefined){
var htmlOfTarget = content.toString(),
whereToCut = theRaw.indexOf("<div class=WordSection1>");
fs.writeFile("bin/temp/temp_" + id + ".htm", theRaw.slice(0, whereToCut) + htmlOfTarget + "</body> </html>", function (err){
if (err) {
throw err;
}
// move the juice call inside the callback of the file write operation
juice("bin/temp/temp_" + id + ".htm", function (err, html) {
if (err) {
throw err;
}
var innerLoad = cheerio.load(html);
newMargin = innerLoad("p").css("margin-left");
console.log(newMargin); // THIS newMargin AS VALUE
// now run the callback passed in the beginning...
callback();
});
});
}
}
// call getMargin with callback to run once complete...
getMargin("myId", "myContent", function(){
// continue program execution in here....
});
I'm trying to create a NodeJS application to pull SQL records and insert them into MongoDB. The tables I'm interested in are somewhat large (1million+ records). For small datasets (< 200,000) my app works great, but running against the full table starts to eat up RAM and bring the server to a crawl.
It looks like Node is running through my "for" loop, branching off processes for each SQL sub select, and then running the MongoDB updates.
I never see "Mongo Connected!" until the last "Getting Responses for Activity #" is written to the screen.
#!/var/node/bin/node
var odbc = require("odbc");
var db = new odbc.Database();
var MongoClient = require('mongodb').MongoClient;
var format = require('util').format;
db.open("DSN=<DSN>;SERVER=<SERVER>;DATABASE=<DB>;UID=<UID>;PWD=<PWD>", function (err) {
if(err) throw err;
console.log("SQL Connected!");
var sqlstr = "SELECT TOP 1000 * FROM tbl_A NOLOCK";
console.log("Executing '" + sqlstr + "' against SQL Server");
db.query(sqlstr, function (sql1err, rows, moreResults) {
if (sql1err) throw sql1err;
for (var i = 0; i < rows.length; i++) {
InsertActivity(db, rows[i], i, rows.length, function () {});
}
});
});
function InsertActivity(sql, activity, cur, total, callback) {
console.log("Getting Responses for Activity #" + activity.ActivityID);
var rsql = "SELECT * FROM tbl_Responses NOLOCK WHERE ActivityID = " + activity.ActivityID;
sql.query(rsql, function (sqlerr, rows, moreResults) {
if (sqlerr) console.log(sqlerr);
activity.resonses = rows;
MongoClient.connect('mongodb://localhost:27017/m', function (merr, mdb) {
console.log("Mongo Connected!");
mdb.collection("activity").insert(activity, function () {
console.log("Inserted Activity #" + activity.ActivityID + " inserted into Mongo");
mdb.close(function () { console.log("Mongo Disconnected!"); });
callback();
});
});
if (cur == total - 1) sql.close(function () { console.log("SQL Disconnected!"); });
});
console.log(rsql);
}
What you need is unfortunately an undocumented function (I'll fix that). The function is db.queryResult which returns the result object that allows you to fetch rows individually. That will avoid buffering the entire result set into memory.
https://github.com/wankdanker/node-odbc/blob/master/test/test-query-select-fetch.js
var db = require('odbc')();
db.open(connectionString, function (err) {
db.queryResult('select * from tbl_A NOLOCK', function (err, result) {
fetchMore();
function fetchMore() {
result.fetch(function (err, data) {
if (!data) {
//we're all done, clean up
}
doStuffWithData(data, function (err) {
fetchMore();
});
});
}
});
});
function doStuffWithData(data, cb) {
//do stuff
cb(null);
}