I'm trying to query a database, then make an array of objects from the results, turn them into a JSON object.
I'm not used to Javascript's asynchronous nature and I'm really confused about how to implement something that needs to work synchronously. I know I probably need to use callbacks, but after looking at many tutorials I'm only left more confused.
This is the code without callbacks:
var foreignTable = (tablename,idArr)=>{
var dataArray = [];
//call a query for each of the ids
var objectToAdd;
for(var id of idArr){
objectToAdd = queryForeignTable(tablename,id);
dataArray.push(objectToAdd);
}
return dataArray;
connection.end();
};
var queryForeignTable = (tablename,id)=>{
connection.query("SELECT * FROM "+tablename+" WHERE id="+id, function (error, results, fields) {
if(error)throw error;
var objectToAddToArray={};
//Go through each field in a result and make the object
for(packet of fields){
var label = packet.name;
objectToAddToArray[label] = results[0][label];
}
return objectToAddToArray;
});
};
var arrayOfDrivers = foreignTable("driver",[1,2]);
outputJson["drive"]=arrayOfDrivers;
console.log(outputJson); // { drive: [ undefined, undefined ] }
I attempted foreignTable(tablename, idArr, callback) with the callback calling queryForeignTable with no luck.
Can someone explain how I can get this piece of code working with callbacks?
A callback function is a function passed into another function as an argument, which is then invoked inside the outer function to complete some kind of routine or action.
from MDN: Callback function
Callbacks are a way of telling a function what to do next, as in "after you're done, run this function".
For example:
first = function (callback) {
console.log("First!");
callback();
}
second = function () {
console.log("Second!");
}
first(second);
Will produce:
First!
Second!
You can also use anonymous function to product the same result:
first(function() {
console.log("Second!")
});
Regarding the specific example from your question, you are correct that you need to use callbacks a bit differently. Rather than using a return statement in each of your two functions, you'll need to use a callback. connection.query is asynchronously coming back with your results as results, but you can't return them to your queryForeignTable function. Instead, give queryForeignTable a callback function to run. The same idea goes for your foreignTable function.
I'm not connected to your database, obviously, so I stubbed a DB connection and simplified what you're trying to do, but it should look something like this:
// Stubbed DB connection
var connection = {};
connection.query = (id, cb) => {
var results = [
{
id: id,
name: 'Name of ' + id,
},
];
cb(null, results);
};
var foreignTable = (ids, cb) => {
var data = [];
for (var i = 0; i < ids.length; i++) {
queryForeignTable(ids[i], (error, obj) => {
data.push(obj);
if (i == ids.length - 1) {
cb(null, data);
}
});
}
};
var queryForeignTable = (id, cb) => {
connection.query(id, (error, results) => {
if (error) {
cb(error, null);
}
cb(null, results[0]);
});
};
foreignTable([1, 2], (error, data) => {
if (error) {
console.error(error);
}
console.log(data);
});
That produces:
[ { id: 1, name: 'Name of 1' }, { id: 2, name: 'Name of 2' } ]
In essence, when you have an urge to return some value(s) from a function in an asynchronous way, give the function a callback parameter, then invoke that callback with your return values.
You can run the code here: https://repl.it/K0YI/3
When you have an asynchronous call, like connection.query(statement, callback), then whatever you want to do with the results of that call needs to be done within the callback.
Bear in mind that async functions don't return the final value that you generally want (usually they return undefined). Instead of using a return value, you pass a callback as a way of saying, "when you're finished, carry on and do this with the results", aka. continuation-passing style.
One of the challenges in your code is that you're issuing separate queries for each ID, and then aggregating the results into an array of responses. This means you'll need to check when all the queries have completed, and only then proceed to display the final results.
Here's your example, re-written, commented, and simplified. Hopefully this helps to explain how the control flow works.
// Read row IDs 1 and 2 from the "driver" table.
readTable("driver", [1, 2], displayData);
// Print the results.
function displayData (arrayOfDrivers) {
console.log(arrayOfDrivers);
}
// Read all rows matching IDs in `idArray` from `tableName`,
// put results into an array, and finally invoke `callback`.
function readTable (tablename, idArray, callback) {
var resultsArray = [];
// Queue up all the async queries.
for (var id of idArray){
queryTable(table, id, handleResponse);
}
// A query finished, so handle the result.
function handleResponse (error, results, fields) {
if (error) {
throw error;
}
// Add the query result to array of results.
resultsArray.push(results[0]);
// Check if all queries are done.
if (resultsArray.length === idArray.length) {
// Invoke the callback with the resultsArray.
// The callback is in fact the `displayData` function.
callback(resultsArray);
}
}
}
// Execute a query, using the `cb` callback to handle the response.
function queryForeignTable (tablename, id, cb) {
var query = "SELECT * FROM " + tablename + " WHERE id=" + id;
connection.query(query, cb);
}
Note that the handleResponse function is defined within the scope of the readTable function, so it can access the variables in readTables scope, such as resultsArray and callback.
Hope that helps.
Related
i am using socket.io to fetch data about a user using his uid when you run this function
function getUserData(uid){
"use strict"
socket.emit('getUserData',uid, function(callback){
console.log('callback' + callback)
for (var i = 0; i < callback.length; i++) {
var row = callback[i];
var username = row.username;
var about = row.about;
var uid = row.uid;
}
})
return {
username: username,
uid: uid,
// about: about
};
}
and it does this on the server side
socket.on('getUserData',function(uid, callback){
connection.query('SELECT * FROM users WHERE uid = ?', [uid], function(err, rows) {
callback(rows)
})
})
but when i do console.log(getUserData(uid)) i get undefined but i do get the object from the first callback what am i doing wrong here?
The callback from .emit() is asynchronous. That means it happens sometime LATER, long after your getUserData() function has already returned. That means you have to communicate back the result using either a callback or a promise. In addition, it makes no sense that you're trying to iterate an array and return one result. You should either return all the results or pick one particular item from the array as your final value. This resolves with the whole array of data, letting the caller decide which one they want to pick from the array.
Here's how you could do so with a promise:
function getUserData(uid){
"use strict"
return new Promise(resolve => {
socket.emit('getUserData',uid, function(returnData){
console.log('returnData', returnData)
resolve(returnData);
});
});
}
// usage
getUserData(someUID).then(results => {
// use results in here
});
I have a list things that I'm sending to PHP one at a time via $.post. I want to wait for each to complete before calling the next. I want to do this with JS not doing the looping with PHP as I want the return value from each to display.
var list = ["a", "b", "c"];
for (i = 0; i < list.length; i++) {
$.post(con, {
callScript: list[i],
}, function(data, status) {
//Do stuff here with the data on success
});
}
I have looked at $.when but just can't sort out how to use it. Ever example assumes that there is a set number of functions and not the same function n times. I also know that async false is not allowed.
Is there a way to get that to run?
Recursion is your good friend here. You can create a function that invokes itself for each item, calling the next one after the async operation of the current one is finished. The recursion stops when we run out of items.
function postInOrder(list, index = 0) {
if (index < list.length) {
$.post(con, {
callScript: list[index],
}, function success(data, status) {
// do stuff here with the data on success
// ...
postInOrder(list, index + 1); // <-- run next script on completion
});
}
}
postInOrder(["a", "b", "c"])
Here's an example with a fake post method:
function postInOrder(list, index = 0) {
if (index < list.length) {
console.log(`Start: ${list[index]}`)
post(function success() {
console.log(`Finish: ${list[index]}`)
postInOrder(list, index + 1); // <-- run next script on completion
});
}
}
postInOrder(["a", "b", "c"])
function post(cb) { setTimeout(cb, 1000); }
You may also reduce it to a promise queue:
list.reduce(function(prom,listEl){
return prom.then(function(){
return new Promise(function(next){
$.post(con, {
callScript: listEl,
}, function(data, status) {
//Do stuff here with the data on success
next();
});
});
});
},Promise.resolve());
(I think wrapping into a promise is not neccessary, may someone whos shure about jquerys syntax feels free to edit ;))
There is a VERY widely used library called "async" that makes this sort of thing very easy. Here are the docs for async series which is probably what you want to do here.
Here is the example from the docs:
async.series([
function(callback) {
// do some stuff ...
callback(null, 'one');
},
function(callback) {
// do some more stuff ...
callback(null, 'two');
}
],
// optional callback
function(err, results) {
// results is now equal to ['one', 'two']
});
The pattern for the callback is pass an error as the first parameter if something went wrong, otherwise pass a null as the first parameter then the actual return value as the second parameter. That pattern is used all over asynchronous JavaScript on the server and in the browser.
That final function is what gets executed once the series is complete.
You can also get tricky with this knowing that variables can be functions in JavaScript and build your queue like this:
var processMe = [];
processMe.push(callScript('two'));
processMe.push(callScript('four'));
processMe.push(callScript('six'));
async.series([ processMe ],
function(err, results) {
// results is now equal to ['two', 'four', 'six']
});
function callScript(value, callback) {
// do something here
callback(null, value);
}
You can also use waterfall if you need to pass results from one step to another.
If it truly is the same code multiple times, use the times operator to simply iterate the same function N times:
// Pretend this is some complicated async factory
var createUser = function(id, callback) {
callback(null, {
id: 'user' + id
});
};
// generate 5 users
async.times(5, function(n, next) {
createUser(n, function(err, user) {
next(err, user);
});
}, function(err, users) {
// we should now have 5 users
});
I am trying to append the results of a db query to a table like so:
function foo() {
var result = {};
pool.getConnection(function(err, connection) {
if(err) {
console.log("Problem establishing connection with the database");
return;
}
var tables = ["first", "second", "third"];
_.forEach(tables, function(table) {
var query = "SELECT * FROM " + table;
connection.query(query, function(err, data) {
if(!err) {
result[table] = data;
} else {
console.log("Problem performing query \"%s\"", query);
}
});
});
});
return result;
}
The returned result is empty, but inside of the forEach, it is being populated. I can prove this by adding a log message in the loop.
I can only think this is a scoping problem but when I changed result to a property (this.result) and tried to assign to that from in the loop I get the same results. I made sure I was using the correct instance of this by adding var that = this; at the start of the function and assigning to that.result inside of the loop.
It's not a scoping problem, it's a timing problem.
You're trying to return result before result is filled in, because your code inside the anonymous function doesn't run until after your foo function returns. It's asynchronous. I expect the query calls are asynchronous as well.
Since foo relies on something asynchronous to do its work, it cannot return the result. It has to do what getConnection and query do: Accept a callback function that it will call, later, with the result.
Here's one way you might do that, see the code comments:
function foo(callback) { // <== Note `callback`
pool.getConnection(function(err, connection) {
if(err) {
console.log("Problem establishing connection with the database");
callback(null); // Do the callback, with a flag value for failure
return;
}
var tables = ["first", "second", "third"];
var result = {};
var results = 0; // Remember how many results we've seen
_.forEach(tables, function(table) {
var query = "SELECT * FROM " + table;
connection.query(query, function(err, data) {
if(!err) {
result[table] = data;
} else {
console.log("Problem performing query \"%s\"", query);
result[table] = null; // Flag value for failure
}
if (++results === tables.length) {
// We're done, issue the callback
callback(results);
}
});
});
});
}
Because the query functions are asynchronous, when the code first runs through, it sends off the queries, then returns your blank object because the queries have not returned yet.
When the queries do return, the properties of that object should be filled out, but you will need to have some code to run at that time in order to "see" it. If you are expecting your code to return the populated object from that function, you will be disappointed. :-)
So basically I am making a database query, to get all posts with a certain id, then add them to a list, so I can return. But the list is returned, before the callback has finished.
How do I prevent it from being returned before callback has finished?
exports.getBlogEntries = function(opid) {
var list12 =[];
Entry.find({'opid' : opid}, function(err, entries) {
if(!err) {
console.log("adding");
entries.forEach( function(currentEntry){
list12.push(currentEntry);
});
}
else {
console.log("EEEERROOR");
}
//else {console.log("err");}
});
console.log(list12);
return list12;
};
ALL callback is asynchronous, so we don't have any guarantee if they will run exactly in the order we have leave them.
To fix it and make the process "synchronous" and guarantee an order executation you have two solutions:
First: make all process in nested list:
instead of this:
MyModel1.find({}, function(err, docsModel1) {
callback(err, docsModel1);
});
MyModel2.find({}, function(err, docsModel2) {
callback(err, docsModel2);
});
use this:
MyModel1.find({}, function(err, docsModel1) {
MyModel2.find({}, function(err, docsModel2) {
callback(err, docsModel1, docsModel2);
});
});
The last snippet above guarantee us that MyModel2 will be executed AFTER MyModel1 is executed.
Second: Use some framework as Async. This framework is awesome and have several helper functions to execute code in series, parallels, whatever way we want.
Example:
async.series(
{
function1 : function(callback) {
//your first code here
//...
callback(null, 'some result here');
},
function2 : function(callback) {
//your second code here (called only after the first one)
callback(null, 'another result here');
}
},
function(err, results) {
//capture the results from function1 and function2
//if function1 raise some error, function2 will not be called.
results.function1; // 'some result here'
results.function2; // 'another result here'
//do something else...
}
);
You could use sync database calls but that would work around the concept of node.js.
The proper way is to pass a callback to the function that queries the database and then call the provided callback inside the database query callback.
How do I prevent it from being returned before callback has finished?
The callback is asynchronous, and you cannot avoid that. Hence, you must not return a list.
Instead, offer a callback for when it's filled. Or return a Promise for the list. Example:
exports.getBlogEntries = function(opid, callback) {
Entry.find({'opid': opid}, callback); // yes, that's it.
// Everything else was boilerplate code
};
There is an alternate way to handle this scenario. You can use the async module and when the forEach has finished then make the return call. Please find the code snippet below for the same:
var async = requires('async');
exports.getBlogEntries = function(opid) {
var list12 =[];
Entry.find({'opid' : opid}, function(err, entries) {
if(!err) {
console.log("adding");
async.forEachSeries(entries,function(entry,returnFunction){
list12.push(entry);
},function(){
console.log(list12);
return list12;
});
}
else{
console.log("EEEERROOR");
}
});
};
I want to pass into Seq([644511,340755]) an response from async function getProjects.
So I tried
...
var ids = pivotal.getProjects(function (err, data) {
var ids = data.project.map(function(x) { return parseInt(x.id); });
console.log("IDS_i: ".red + ids);
});
console.log("IDS_e: ".red + ids);
Seq(ids)
.parEach(function(project_id) {
....
Logs:
IDS_e: undefined
GET /stories 200 34ms
GET /favicon.ico 404 2ms
IDS_i: 644511,340755
I am wondering maybe I should put this into Seq:
Seq()
.seq(function() {
pivotal.getProjects(function (err, data) {
data.project.map(function(x) { return parseInt(x.id); });
});
}
but how to return ids as array in that case?
getProjects is also async. A basic rule: You can't return any values from an async function. You have to do all processing in the callback function. Execution will continue before your arrays have been aggregated. So your seq approach is what you need:
Seq()
.seq(function() {
pivotal.getProjects(this);
})
.flatten()
.seqEach(function(project) {
var projectId = project.id;
myService.someOtherAsyncAction(projectId, this);
});
node-seq will take care of passing the result of the callback to the next seq step by passing this as the callback function to your async function. This is how flow and results are passed to the next step. flatten will make sure each project is available as individual elements on the stack so you can do seqEach in the next step.