I'm trying to use async.each function to get an array with my results from two queries. After that, I need to render this results in a web page.
The async.each function calcule the variable results properly, but, I am not be able to export this variable outside the function and render it and I don't understand why.
Here I attached the code, where I tested it. I realized that when I call "callback1" the function(error) is not working and I don't get the variable list in the console (so I won't be able to render it later on). Please I would be grateful if someone could help me with that. Thanks a lot.
var list = [];
async.each(data,
function(elem, callback1){
var classgene = '';
var custom_gene = {};
custom_gene = {Name_Gene: elem['Name_Gene']};
if (elem['Type_Gene'] == "reference") {
async.waterfall([
function(callback2){
var id = elem['Id_Genes'];
geneModel.getGenesRefClass(id, function(error, data2){
classgene = data2[0]['Class_Name'];
custom_gene['classgene'] = classgene;
callback2(custom_gene);
});
},
], function(custom_gene, err){
list.push(custom_gene);
console.log(list);
callback1();
});
}
}, function(err){
// if any of the saves produced an error, err would equal that error
if(err){
console.log(list);
}else{
console.log(list);
}
});
Your code has a few problems:
It's not calling callback2() properly. It should be callback2(null, custom_gene) (the first argument is reserved for errors, or null if there aren't any). Preferably, you should also check for error being returned by geneModel.getGenesRefClass();
The previous issue also means that you need to swap the argument of function(custom_gene, err) (it should become function(err, custom_gene));
When elem['Type_Gene'] does not equal "reference", you should still call callback1(), otherwise async.each() doesn't know that the code is done;
So the code would become something like this:
var list = [];
async.each(data, function(elem, callback1) {
var classgene = '';
var custom_gene = { Name_Gene : elem['Name_Gene'] };
if (elem['Type_Gene'] == "reference") {
async.waterfall([
function(callback2) {
var id = elem['Id_Genes'];
geneModel.getGenesRefClass(id, function(error, data2){
if (error) return callback2(error);
classgene = data2[0]['Class_Name'];
custom_gene['classgene'] = classgene;
callback2(null, custom_gene);
});
},
], function(err, custom_gene) {
// If you want to propagate errors, uncomment the following:
// if (err) return callback1(err);
list.push(custom_gene);
console.log(list);
callback1();
});
} else {
callback1();
}
}, function(err){
// if any of the saves produced an error, err would equal that error
if (err) {
console.log('An error occurred!', err);
}
console.log(list);
});
Related
I posted a question before and realized my problem actually was async functions. I managed to work out most of it, but I got one little problem left. Using async I used waterfall to create an order for the some queries...
exports.getMenu = function(id_restaurant, callback){
async.waterfall([
async.apply(firstQuery, id_restaurant),
secondQuery,
thirdQuery,
fourthQuery,
formMenu
], function(err, result){
if(err){
console.log(err);
}
callback(result);
});
};
Everything works until fourthQuery, where I have to loop to get all dishes of a menu.
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if(err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes[i] = results2;
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
};
So what i can see that happens from printing the value of 'i' each loop is that it always has the value of 2. Because that's 'size' value. Also, even though it's saving results of index '2' I believe the callback is being done even before the for loop is done, because my fifth function is recieving an empty array.
How can i make my code wait to callback until my for loop is done?
NOTE: Sorry, part of my code is in spanish, tried to translate the important parts of it.
There are a few ways to handle this, one is to look into promise architecture. Promise.all will let you supply one callback to handle the values from each child promise.
To use what you've already got, however, I'd push the values into your dishes array, rather than assigning them specifically to i indexes, then check the size of that array at the end of each connection. When the array length matches the size, fire the callback. (as seen below)
If you need a way to tie each result to that specific i value, I'd recommend pushing them as an object
dishes.push({'index': i, 'dish': results2})
Afterward, if you need the array of just dishes, you can sort the array by that index value and run a map function.
dishes.sort(function(a,b){ return a.index - b.index; })
dishes = dishes.map(function(a){ return a.dish })
Here's the code adjusted:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback) {
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if (err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if (err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2)
if(dishes.length == size){
console.log("I'm sending " + dishes);
callback(null, dishes, array_nombresSecc)
}
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
;
};
Since you're already using the async, I would suggest replacing the for() loop in fourthQuery with async.each().
The updated fourthQuery would look like this:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
async.each(array_secciones,
function(item, itemCallback) {
// Function fun for each item in array_secciones
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [item],
function(err, results2) {
if(err) {
console.log(err);
return itemCallback(true);
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2);
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[dishes.length-1]));
// this prints the result but only if it has a value over 2
return itemCallback();
});
},
function(err) {
// Function run after all items in array are processed or an error occurs
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
});
}); // pool
};
Alternatively, you can use async.map(), which handles gathering the results in the final callback so doesn't rely on the dishes variable.
I tried to scrap for thousands of pages. So I used async.timesSeries and async.waterfall. Each of functions work synchronously very well but they don't work together. What can I do?
The logic is simple.
Because I want to scrape pages are "http://udb.kr/local/category/390101?page="1~1167, async.timesSeries loop 1 to 1167
async.waterfall scraps components of pages
but messages that console shows me looks like this
info.NM values // just for explain, It shows me each attires of obj because I insert console.log(info.NM) for verifying.
info.NM values
info.NM values
info.NM values and randomly ----- page number -----
...
['done',
'done',
'done',
'done',
'done',
...
'done']
info.NM values again
.../Users/Snark/Dev/job_apply/cheerio_job_app_list.js:29
if (tObj[m+1].children != 0) {info.nAddr = tObj[m+1].firstChild.data}else{info.nAddr = null};
^
TypeError: Cannot read property 'children' of undefined
at /Users/Snark/Dev/job_apply/cheerio_job_app_list.js:29:17
at fn (/Users/Snark/node_modules/async/lib/async.js:746:34)
at /Users/Snark/node_modules/async/lib/async.js:1212:16
at /Users/Snark/node_modules/async/lib/async.js:166:37
at /Users/Snark/node_modules/async/lib/async.js:706:43
at /Users/Snark/node_modules/async/lib/async.js:167:37
at /Users/Snark/node_modules/async/lib/async.js:1208:30
at Request._callback (/Users/Snark/Dev/job_apply/cheerio_job_app_list.js:21:6)
at Request.self.callback (/Users/Snark/node_modules/request/request.js:198:22)
at emitTwo (events.js:87:13)
And this is js code.
var request = require("request"),
cheerio = require("cheerio"),
jsonfile = require("jsonfile"),
fs = require("fs"),
async = require("async");
var info = {},
dbArray = [];
var url = "http://udb.kr/local/category/390101?page=";
async.timesSeries(1166, function(n, next) {
var page = n + 1
async.waterfall([
function(callback) {
request(url + page, function(error, response, html) {
if (error) {
throw error
};
var $ = cheerio.load(html),
tObj = $('tbody tr td');
callback(null, tObj);
});
},
function(tObj, callback) {
for (var m = 0; m < 150; m = m + 5) {
if (tObj[m]) {
info.NM = tObj[m].firstChild.children[0].data
} else {
info.NM = null
};
if (tObj[m + 1].children != 0) {
info.nAddr = tObj[m + 1].firstChild.data
} else {
info.nAddr = null
};
console.log(info.NM);
dbArray.push(info);
}
callback(dbArray, callback);
},
function(dbArray, callback) {
fs.appendFile('./jobDB_l.json', JSON.stringify(dbArray), function (err) {
if (err)
throw err;
});
callback(null, 'done');
}
], function(err, result) {
console.log('----- ' +page+ '-----');
});
next(null, 'done');
}, function(err, result) {
console.log(result)
});
To get these to work together where you are using waterfall inside of each timesSeries iteration, you need to call the timesSeries done callback from the completion callback for the waterfall call. Right now, you are calling it long before that which means that timesSeries won't wait for the waterfall to be done.
You can do that by changing this:
], function(err, result) {
console.log('----- ' +page+ '-----');
});
next(null, 'done');
to this:
], function(err, result) {
console.log('----- ' +page+ '-----');
next(null, 'done');
});
It also seems odd that you have a hard-coded for loop limit of m < 150 rather than using the actual length of the content. You can easily run off the end of the content and potentially cause problems.
And, your error handling probably won't work well either. If you throw inside of the async request() callback, that's not going to go anywhere. You need much better error handling such as calling callback(error) to pass the error on to async.waterfall().
You also may want to surround all your DOM walking in a try/catch so if you throw any exceptions there, you can catch them yourself, analyze them and then fix the code.
if (tObj[m+1] && tObj[m+1].children != 0)
I have the code snippet bellow which actually is a function that makes a query on a SQL database(used tedious for that).
All I want is to get the data from DB and use them on a web-page.
The issue is that: inside the request.on()... I'm calling date.setMyData() function. Still inside the request.on('row', function(columns)....if calling console.log(date.getMyData()) it successfully returns my data.
BUT, if trying to call date.getMyData outside the request.on('row', function(columns) ...I get NULL...
Is there anything that I'm missing here?
var date = new DBData();
function executeStatement() {
request = new Request(queryString, function(err, rowCount) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
connection.close();
});
request.on('row', function(columns) {
columns.forEach(function(column) {
if (column.value === null) {
console.log('NULL');
} else {
date.setMyData(columns);
//WHY THIS CODE RETURNS MY DATA?
console.log(date.getMyData());
}
});
});
request.on('done', function(rowCount, more) {
console.log(rowCount + ' rows returned');
});
connection.execSql(request);
}
function DBData(){
var myData = null;
this.setMyData = function(obiect){
myData = obiect;
};
this.getMyData = function(){
return myData;
};
};
console.log(date.getMyData()); //WHY I GET 'NULL' HERE?
When you run that code, the order of execution will be
var date = new DBData()
console.log(date.getMyData()) // null at this point
// end of execution
The function executeStatement() gets defined, but it is not executed right away. It will be presumably executed at a later time when you call it, at which point it will populate your 'var date', hence it will display the correct data correctly when you console.log(date.getMydata()) inside the callback of executeStatement().
This question already has an answer here:
How to return value from Node.js function which contains DB query [duplicate]
(1 answer)
Closed 8 years ago.
I have a function inside of which there is a nested callback function structure. I want that first "mother" function to return a certain value, after it's been calculated in the callback functions sequence. However. it doesn't really work. Here's a simplified version of the code, do you think you could help?
console.log(finalResult());
function finalResult() {
var finalanswer = firstFunction(secondFunction);
function firstFunction (callback) {
var notion = 1;
callback(null, notion);
}
function secondFunction (err, notion) {
if (err) console.log(err);
var answer = notion + 1
return answer;
}
return finalanswer;
}
Thank you!
**UPDATE - THE ORIGINAL CODE**
return getContexts(makeQuery);
function getContexts (callback) {
dbneo.cypherQuery(context_query, function(err, cypherAnswer){
if(err) {
err.type = 'neo4j';
return callback(err);
}
// No error? Pass the contexts to makeQuery function
return callback(null,cypherAnswer);
});
}
function makeQuery (err,answer) {
// Error? Display it.
if (err) console.log(err);
// Define where we store the new contexts
var newcontexts = [];
// This is an array to check if there are any contexts that were not in DB
var check = [];
// Go through all the contexts we received from DB and create the newcontexts variable from them
for (var i=0;i<answer.data.length;i++) {
newcontexts.push({
uid: answer.data[i].uid,
name: answer.data[i].name
});
check.push(answer.data[i].name);
}
// Now let's check if there are any contexts that were not in the DB, we add them with a unique ID
contexts.forEach(function(element){
if (check.indexOf(element) < 0) {
newcontexts.push({
uid: uuid.v1(),
name: element
});
}
});
return newcontexts;
}
You're setting finalanswer equal to the return of firstFunction however, if you look at the body of firstFunction, it does not return anything. The last line of firstFunction should be:
return callback(null, notion);
I quickly tested this in the Chrome console and it seems to work as expected and logs 2 to the console.
UPDATE
Now that the original code has been posted I would update the code as such:
// call getContexts with a callback when complete
getContexts(function(err, contexts){
console.log(err);
console.log(contexts);
});
function getContexts (callback) {
dbneo.cypherQuery(context_query, function(err, cypherAnswer){
if(err) {
err.type = 'neo4j';
return callback(err);
}
// No error? Pass the contexts to makeQuery function
var contexts = makeQuery(null,cypherAnswer);
// we have our answer, call the callback
callback(null, contexts);
});
}
function makeQuery (err,answer) {
// Error? Display it.
if (err) console.log(err);
// Define where we store the new contexts
var newcontexts = [];
// This is an array to check if there are any contexts that were not in DB
var check = [];
// Go through all the contexts we received from DB and create the newcontexts variable from them
for (var i=0;i<answer.data.length;i++) {
newcontexts.push({
uid: answer.data[i].uid,
name: answer.data[i].name
});
check.push(answer.data[i].name);
}
// Now let's check if there are any contexts that were not in the DB, we add them with a unique ID
contexts.forEach(function(element){
if (check.indexOf(element) < 0) {
newcontexts.push({
uid: uuid.v1(),
name: element
});
}
});
return newcontexts;
}
I'm new to node.js. I need to display Name in jqgrid, but I stored only id of one document into another document.
Example
I have 2 documents like Student master and student mark document. I have to display mark details in jqgrid. In mark document I stored student id instead of name. How do I fetch the name and send a new object to jqgrid?
My code is as follows:
exports.getAllstudentsmark = function(req, callback)
{
studentsmarks.find(function(error, studentsmarks_collection) {
if( error ) callback(error)
else {
studentsmarks_collection.toArray(function(error, results) {
if( error ) callback(error)
else {
newresult = results;
for(i=0;i<results.length;i++)
{
newresult[i]['studentname'] = getStudentName(results[i].studentid);
}
console.log(newresult);
callback(null, newresult)}
});
}
});
}
var getstudentObjectId = function(id)
{
return student.db.bson_serializer.ObjectID.createFromHexString(id);
}
var getStudentName = function(id)
{
student.findOne({_id: getstudentObjectId (id)}, function(e, o){
console.log(o.name);
return o.name;
});
}
newresult[i]['studentname'] is always getting undefined. But if I log into getStudentName function I can get answer into getStudentName function.
My callback function is only getting this problem. How to resolve and get my result in an easy way. Please help any one.
try this inside your for loop
newresult.push({'studentname': getStudentName(results[i].studentid) });
exlpanation:
by the time you access newresult[i] it doesn't exist, so accessing studentname field of it is impossible
Your problem here is that you are not setting the name of the user into the array, but the return value of student.findOne, since this is an asynchronous method. Maybe try this thing
exports.getAllstudentsmark = function(req, callback)
{
studentsmarks.find(function(error, studentsmarks_collection) {
if( error ) callback(error)
else {
studentsmarks_collection.toArray(function(error, results) {
if( error ) callback(error)
else {
newresult = [];
for(i=0;i<results.length;i++)
{
getStudentName(results[i].studentid, function (studentName) {
newresult.push({studentname: studentName});
})
}
console.log(newresult);
callback(null, newresult)}
});
}
});
}
var getstudentObjectId = function(id)
{
return student.db.bson_serializer.ObjectID.createFromHexString(id);
}
var getStudentName = function(id, callback)
{
student.findOne({_id: getstudentObjectId (id)}, function(e, o){
console.log(o.name);
callback(o.name);
});
}
I hope it helps