I am searching for an elegant way to sync indepentent callbacks result invoked in unknown order.
function callback1() {
var result;
};
function callback2() {
var result;
};
//When done then call
function success(res1, res2) {
// do whatever
}
I know I can do something like:
var res = {};
var dfd = $.Deferred();
function callback1() {
var result;
res.res1 = result;
(res.res1 && res.res2) && (dfd.resolve(res));
};
function callback1() {
var result;
res.res2 = result;
(res.res1 && res.res2) && (dfd.resolve(res));
};
dfd.done(function(result){
// do whatever
});
but I would appreciate if somebody comes up with more elegant solution
If you return promises (builtin promises, not jQuery deferreds) and you don't care about order, then you can use Promise.all:
function callback1() {
return Promise.resolve(1)
}
function callback2() {
return Promise.resolve(2)
}
var ps = [callback1(), callback2()]
function add(x, y) {
return x + y
}
Promise.all(ps).then(function(result) {
return result.reduce(add)
}).then(console.log) // => 3
If you want to sequence them you can do it in such a way that you can apply a curried function that expects as many arguments as resolved promises there are by lifting it into the promise world. In other words:
function apply(pa, pf) {
return pf.then(function(f) {
return pa.then(f)
})
}
function lift(f, ps) {
return ps.reduce(function(pa, pb) {
return apply(pb, pa)
}, Promise.resolve(f))
}
function add(x) {
return function(y) {
return x + y
}
}
lift(add, ps).then(console.log) //=> 3
You can also sequence them in such a way that you don't need a curried function, by collecting the results in an array first then reducing it:
function sequence(ps) {
return ps.reduceRight(function(pa, pb) {
return pa.then(function(a) {
return pb.then(function(b) {
return [b].concat(a)
})
})
}, Promise.resolve([]))
}
function add(x, y) {
return x + y
}
// This looks similar to the Promise.all approach
// but they run in order
sequence(ps).then(function(result) {
return result.reduce(add)
}).then(console.log) // => 3
There are libraries that do this, such as the async library, but here's a "from scratch" solution. I'm also avoiding promises to avoid overwhelming you, but you should really read about them as they are the most elegant solution, albeit complicated for first timers.
function runInParallel(jobs, done) {
// Store all our results in an array.
var results = [];
// If one job fails, set this to true and use it to
// ignore all job results that follow.
var failureOccurred = false;
// Iterate over each of our registered jobs.
jobs.forEach(function (runJob, index) {
// Create a jobDone callback to pass to the job.
var jobDone = function (err, result) {
// If another job failed previously, abort processing
// this job's result. We no longer care.
if (failureOccurred) return;
// If this job passed in an error, set failure to true
// and pass the error to the final done callback.
if (err) {
failureOccurred = true;
done(err);
return;
}
// If we made it this far then push the job result into
// the results array at the same position as the job in
// the jobs array.
results[index] = result;
// If the results array contains as many results as the
// jobs array had jobs then we have finished processing
// them all. Invoke our done callback with an array of
// all results.
if (results.length === jobs.length) {
done(null, results);
}
};
// Begin the job and pass in our jobDone callback.
runJob(jobDone);
});
}
This will call all of your job functions in the array, passing in a jobDone callback the job should call when finished. If any job passes an error in then the function will immediately invoke the result callback with the error and ignore everything else. If the jobs succeed then you'll end up with an array of job results in the same positions as the jobs were in the jobs array. Simply modify your job functions to accept the jobDone callback.
var jobs = [
function job1(done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
},
function job2(done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
}
];
runInParallel(jobs, function (err, results) {
if (err) {
console.error(err);
return;
}
// results[0] = jobs[0] result
// results[1] = jobs[1] result
// etc...
});
Instead of an array of jobs you could modify this code to accept an object with property names. Then instead of assigning the results to the same position as the jobs in the jobs array you could assign the results to an object using the same property names.
Example (without comments this time):
function runInParallel(jobs, done) {
var results = {};
var failureOccurred = false;
Object.keys(jobs).forEach(function (jobName) {
var jobDone = function (err, result) {
if (failureOccurred) return;
if (err) {
failureOccurred = true;
done(err);
return;
}
results[jobName] = result;
if (results.length === jobs.length) {
done(null, results);
}
};
jobs[jobName](jobDone);
});
}
Then you can consume it like this:
var jobs = {
job1: function (done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
},
job2: function (done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
}
};
runInParallel(jobs, function (err, results) {
if (err) {
console.error(err);
return;
}
// results.job1 = job1 result
// results.job2 = job2 result
// etc...
});
The parallel function in the async library does almost exactly what we've done above. It even accepts an array of jobs or an object of named jobs like we did :)
Assuming your tasks (callback1() and callback2()) are synchronous, you might choose to write a reusable generalisation of the code in the question, in the form of a function that returns a function, trapping a couple of private vars in a closure :
function resultAggregator(n, fn) {
var results = {},
count = 0;
return function(id, res) {
count++;
results[id] = res;
if (count == n) {
fn(results);
}
}
}
So after calling resultAccumulator(), you have a function that can kept in-scope of other functions or passed to other parts of your code base. It makes no assumptions about ids or the nature of the results, except that they are synchronously derived. It will fire its callback when n results have been delivered.
var myResults = resultAggregator(2, function(results) {
// do whatever;
});
//The following commands may be in different parts of your code base
myResults('id1', synchTask1());
...
myResults('id2', synchTask2());
...
myResults('id3', synchTask3());
//The second tasks to deliver its data (ostensibly `synchTask1()` and `synchTask2()`, but not necessarily) will trigger the callback.
Demo
This is just one way to perform result aggregation. You might do something different depending on the exact scenario. Here's a slightly different formulation which records the order in which the results arrived :
Demo
Whatever you write, Deferreds/Promises are not necessary for the aggregation of synchronously derived data.
However, if any one task is, or may be, asynchronous then you may need a promise aggregator, eg jQuery.when() or Promise.all(), somewhere in the pattern.
Related
I'm trying to query a database, then make an array of objects from the results, turn them into a JSON object.
I'm not used to Javascript's asynchronous nature and I'm really confused about how to implement something that needs to work synchronously. I know I probably need to use callbacks, but after looking at many tutorials I'm only left more confused.
This is the code without callbacks:
var foreignTable = (tablename,idArr)=>{
var dataArray = [];
//call a query for each of the ids
var objectToAdd;
for(var id of idArr){
objectToAdd = queryForeignTable(tablename,id);
dataArray.push(objectToAdd);
}
return dataArray;
connection.end();
};
var queryForeignTable = (tablename,id)=>{
connection.query("SELECT * FROM "+tablename+" WHERE id="+id, function (error, results, fields) {
if(error)throw error;
var objectToAddToArray={};
//Go through each field in a result and make the object
for(packet of fields){
var label = packet.name;
objectToAddToArray[label] = results[0][label];
}
return objectToAddToArray;
});
};
var arrayOfDrivers = foreignTable("driver",[1,2]);
outputJson["drive"]=arrayOfDrivers;
console.log(outputJson); // { drive: [ undefined, undefined ] }
I attempted foreignTable(tablename, idArr, callback) with the callback calling queryForeignTable with no luck.
Can someone explain how I can get this piece of code working with callbacks?
A callback function is a function passed into another function as an argument, which is then invoked inside the outer function to complete some kind of routine or action.
from MDN: Callback function
Callbacks are a way of telling a function what to do next, as in "after you're done, run this function".
For example:
first = function (callback) {
console.log("First!");
callback();
}
second = function () {
console.log("Second!");
}
first(second);
Will produce:
First!
Second!
You can also use anonymous function to product the same result:
first(function() {
console.log("Second!")
});
Regarding the specific example from your question, you are correct that you need to use callbacks a bit differently. Rather than using a return statement in each of your two functions, you'll need to use a callback. connection.query is asynchronously coming back with your results as results, but you can't return them to your queryForeignTable function. Instead, give queryForeignTable a callback function to run. The same idea goes for your foreignTable function.
I'm not connected to your database, obviously, so I stubbed a DB connection and simplified what you're trying to do, but it should look something like this:
// Stubbed DB connection
var connection = {};
connection.query = (id, cb) => {
var results = [
{
id: id,
name: 'Name of ' + id,
},
];
cb(null, results);
};
var foreignTable = (ids, cb) => {
var data = [];
for (var i = 0; i < ids.length; i++) {
queryForeignTable(ids[i], (error, obj) => {
data.push(obj);
if (i == ids.length - 1) {
cb(null, data);
}
});
}
};
var queryForeignTable = (id, cb) => {
connection.query(id, (error, results) => {
if (error) {
cb(error, null);
}
cb(null, results[0]);
});
};
foreignTable([1, 2], (error, data) => {
if (error) {
console.error(error);
}
console.log(data);
});
That produces:
[ { id: 1, name: 'Name of 1' }, { id: 2, name: 'Name of 2' } ]
In essence, when you have an urge to return some value(s) from a function in an asynchronous way, give the function a callback parameter, then invoke that callback with your return values.
You can run the code here: https://repl.it/K0YI/3
When you have an asynchronous call, like connection.query(statement, callback), then whatever you want to do with the results of that call needs to be done within the callback.
Bear in mind that async functions don't return the final value that you generally want (usually they return undefined). Instead of using a return value, you pass a callback as a way of saying, "when you're finished, carry on and do this with the results", aka. continuation-passing style.
One of the challenges in your code is that you're issuing separate queries for each ID, and then aggregating the results into an array of responses. This means you'll need to check when all the queries have completed, and only then proceed to display the final results.
Here's your example, re-written, commented, and simplified. Hopefully this helps to explain how the control flow works.
// Read row IDs 1 and 2 from the "driver" table.
readTable("driver", [1, 2], displayData);
// Print the results.
function displayData (arrayOfDrivers) {
console.log(arrayOfDrivers);
}
// Read all rows matching IDs in `idArray` from `tableName`,
// put results into an array, and finally invoke `callback`.
function readTable (tablename, idArray, callback) {
var resultsArray = [];
// Queue up all the async queries.
for (var id of idArray){
queryTable(table, id, handleResponse);
}
// A query finished, so handle the result.
function handleResponse (error, results, fields) {
if (error) {
throw error;
}
// Add the query result to array of results.
resultsArray.push(results[0]);
// Check if all queries are done.
if (resultsArray.length === idArray.length) {
// Invoke the callback with the resultsArray.
// The callback is in fact the `displayData` function.
callback(resultsArray);
}
}
}
// Execute a query, using the `cb` callback to handle the response.
function queryForeignTable (tablename, id, cb) {
var query = "SELECT * FROM " + tablename + " WHERE id=" + id;
connection.query(query, cb);
}
Note that the handleResponse function is defined within the scope of the readTable function, so it can access the variables in readTables scope, such as resultsArray and callback.
Hope that helps.
Title says it all, I am using the async-library eachSeries method
let
valueOne = null,
validItem = null;
const asyncEachSeries = require('async/eachSeries');
const getContent = function (item) {
contentResponse = fetchContent(item);
return contentResponse;
}
if (recommendationsArr.length > 0) {
asyncEachSeries(recommendationsArr, function (item, callback) {
getApiContent(item).then(function getValidResult(response) {
try {
valueOne = response.content.valueOne || null; //may or may not be present
recommendationsArr.splice(recommendationsArr.indexOf(item), 1);
validItem = item;
console.log("##### valueOne is: ", valueOne); // I get data here
return true;
} catch (err) {
valueOne = null;
return false;
}
});
//I am guessing I need a return statement here, though not sure what
});
console.log("##### Outside promise, valueOne is: ", valueOne); // no data here
// Access valueOne, validItem, recommendationsArr to be able to pass as props for the component
return {
component: <Layout><ComponentName
//pass other non-dependent props
validItem={validItem}
valueOne={valueOne}
recommendationsArr={recommendationsArr}
/></Layout>,
title: `Page Title`,
};
}
return null;
Scenario: recommendationsArr is an array of items (Its 99.9% not null, but since its an external api call I prefer to have a check). The purpose is to have values for valueOne, validItem, and an updated recommendationsArr.
The validity depends on the existence of valueOne, so if the recommendationsArr[0] has valid valueOne then I don't need to fetch api results for the rest of the array. I am using eachSeries as it runs only a single async operation at a time, hence if that async gets me the valid result I don't have to iterate over other items.
Sample recommendationsArr: recommendationsArr = ["blue", "white", "green", "red"]; //usually the array is anywhere between 12-50 elements
Now in order to be able to pass the updated values to the component I need to be able to access the values set in try block outside of the asyncEachSeries iteration loop. I understand I will have to return the processed/new values but am not sure at what point and how to return those values.
As is usual with any question concerning asynchronous code you return values by using the callback. But eachSeries does not collect return values. What you want is async.mapSeries:
async.mapSeries(recommendationsArr, function (item, callback) {
getApiContent(item).then(function (response) {
try {
valueOne = response.content.valueOne || null; //may or may not be present
recommendationsArr.splice(recommendationsArr.indexOf(item), 1);
validItem = item;
callback(null, valueOne); // This is how you return a value
// return true; // return does absolutely nothing
} catch (err) {
valueOne = null;
callback(err, valueOne);
// return false;
}
});
},function (err, result) {
// this is where you can process the result
// note that the result is an array
});
However, since you are using promises you can use async-q instead which wraps caolan's async library in promises:
var asyncQ = require('async-q');
asyncQ.mapSeries(recommendationsArr, function (item) {
// Note: the return in the line below is important
return getApiContent(item).then(function (response) {
try {
valueOne = response.content.valueOne || null; //may or may not be present
recommendationsArr.splice(recommendationsArr.indexOf(item), 1);
validItem = item;
return valueOne; // this is how you return a value in a promise
} catch (err) {
return null
}
});
})
.then(function(result) {
// you can process the result here
});
Personally I'd prefer to rewrite the code and process the results at the end because the code is cleaner:
asyncQ.mapSeries(recommendationsArr, getApiContent)
.then(function (result) {
for(var i=0;i<result.length;i++) {
var response = result[i];
// do what needs to be done with each response here
}
});
Are both my examples the same in terms of functionality considering the fact that in error handeling I'm terminating by res.json(400, err)? Also I would like to know that in my second example the second async.each always run after the first async.each, so using results1 in the second async.each is safe? Sorry I'm new to Node and async!
Example1: where I'm using the results of each async.each in the last block as an input of the other async.each:
var results1 = {};
var results2 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
}
});
or Example2: where I have separate async.each blocks
var results1 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
}
});
var results2 = {};
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
UPDATED:
Since the second approach is not right way and it is not guaranteed that the second async.each runs after the first one the problem is: Does it mean I cannot have a simple for loop like the following example either? If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well and not a for loop, do you know how I can have this recursive functionality here?
var results1 = {};
var results2 = [];
var results3 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
// So in this case that I need to have nested function, does it mean I cannot have a simple for loop like this as well?
// If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well, do you know how I can have this recursive functionality here?
for (var input in inputs) {
inferFromUnion(inputs[input], results1);
results2.push(inputs[input]);
}
async.each(results2, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results3);
}
});
}
});
// Here just checking each object schema and if they are missing any fields from results1 we add that field with a value of null
function inferFromUnion(obj, allFields) {
Object.keys(allFields).forEach(function (key) {
if (lodash.isUndefined(obj[key])) {
if (lodash.isPlainObject(allFields[key])) {
obj[key] = {};
inferFromUnion(obj[key], allFields[key]);
} else {
obj[key] = null;
}
}
});
}
The first example is the way to go, if you want to use results of the first bunch of calls in the second bunch. The second example won't work, because the second async.each() is guaranteed to run before the callbacks bound to your asynchronous operations.
Asynchronous recursion with loops is very much possible:
(function doSomeAsyncRecursion (results) {
async.each(someItems, function (item, callback) {
// ...
}, function () {
if (results /* ... (are incomplete) */) {
doSomeAsyncRecursion(results);
} else {
// ... (results are complete now, do something with them)
}
});
})(/* initial value of results */);
These two examples are different in desing. First example will run second async after first async is successful. But second example will run second async everytime, if theres an error or not.
I have a array of ids:
var ids = ['53asd3','53asd2','53asd5'];
Each id has a corresponding document in the mongodb.
I want to generate a object by populating data from each of them and save in some other document. Like this:
{
person: { /* data from some collection populated with first id */},
company : { /* data from some collection populated with first id */},
employee : {/* data from some collection populated with first id */}
}
WHAT I DID
var document = {}
models.persons.find({_id:'53asd3'},function(err,data){
if(!err) {
document['persons']=data;
models.company.find({_id:'53asd2'},function(err,data){
if(!err) {
document['company']=data;
models.employee.find({_id:'53asd2'},function(err,data){
if(!err) {
document['employee']=data;
document.save(function(err){ });
}
});
}
});
}
});
So I just use nested calls using callbacks and somewhat make it synchronous. Is there a chance where I can execute all these three find queries in parallel and then execute the save command? I actually want to leverage the async nature of node.js. Any suggestions?
You could build something like async.parallel yourself if you don't want to include an external library. Here's what a simple parallel function might look like. It could be a nice exercise to implement the other functions in the async library.
var parallel = function () {
var functs = arguments[0];
var callback = arguments[1];
// Since we're dealing with a sparse array when we insert the results,
// we cannot trust the `length` property of the results.
// Instead we count the results separately
var numResults = 0;
var results = [];
var getCallback = function (i) {
return function (err, res) {
if (err) {
callback(err)
}
else {
results[i] = res;
numResults += 1;
if (numResults === functs.length) {
callback(null, results);
}
}
}
}
functs.forEach(function (fn, i) {
fn(getCallback(i));
});
};
var getTest = function (timeout) {
return function (callback) {
setTimeout(function () {
callback(null, timeout);
}, timeout);
}
};
parallel([getTest(99), getTest(1000), getTest(199)], console.log.bind(console));
>> null [99, 1000, 199]
Then in your case you can do something like
var findItem = function (collection, id) {
return function (callback) {
collection.find({
_id: id
}, callback);
};
};
parallel([
findItem(models.persons, '53asd3'),
findItem(models.company, '53asd2'),
findItem(models.employee, '53dsa2')
], function (err, results) {
document.persons = results[0];
document.company = results[1];
document.employee = results[2];
document.save(function (err) {
// and so on
});
});
This is super simple task to do in Java but the asynchronous nature of javascript makes this task(for me) almost impossible, at least with my knowledge now.(I'm not trying to bash javascript. Love the language!).
It's very basic. A top level tree has a parent of null in my mysql database. It's easy finding children. The children have lines available to them. The depth of the tree is variable.
private static Set<Tree> getBranches( Tree trunk ) {
Set<Tree> treeSet = new HashSet<Tree>();
if ( trunk != null ) {
if ( trunk.hasLines() ) { //queries if tree has lines. returns true or false
treeSet.add( trunk );
}
for ( Tree tree : trunk.treeList ) {
treeSet.addAll( getBranches( tree ) );
}
}
return treeSet;
}
Basically the method tests if the tree has lines available. If it does it adds all of those to a set. If not it continues until it finds lines.
The asynchronous nature of the mysql node library turns this task into hell.
Here is what I have now
function hasLines(tree_id, callback) {
var ret;
pool.query('SELECT * from pkg_line_tree where tree_id = ?', [tree_id], function (err, rows) {
if (rows.length > 0) {
ret = true;
} else {
ret = false;
}
callback(ret);
});
}
function dig(tree_id, treeArray, callback) {
pool.query('SELECT * from tree where parent_id = ?', [tree_id], function (err, rows) {
if (rows) {
for (var i in rows) {
hasLines(rows[i].tree_id, function (t) {
if (t) {
treeArray.push(rows[i].tree_id);
} else {
treeArray.concat(dig(rows[i].tree_id, treeArray));
}
});
}
if (callback) {
callback(treeArray);
}
}
});
return treeArray;
}
var treeArray = [];
dig(52, treeArray, function (t) {
res.json(t);
});
I really just need to output all the children available in this root tree.
Please let me know if this doesn't make sense. I'll try to refactor. I'm hoping I got some kind of point across. I'd hate to use something like Fibers to get this done but I'm out of options. Thanks.
Your use of dig() isn't currently consistent:
// asynchronous with callback
dig(52, treeArray, function (t) {
res.json(t);
});
// then synchronous with `return`?
treeArray.concat(dig(rows[i].tree_id, treeArray));
Also, the concat in the last line isn't actually doing much, since it doesn't alter the array it's called on. You probably wouldn't actually want it to as dig passes around the treeArray rather than defining a new treeSet like in getBranches. So, if it did, it would append treeArray onto the end of itself each time.
You could still use concat with multiple treeSets, but you'd have to store its return value:
treeSet = treeSet.concat(subSet);
And, you'll have to replace the for loop this with an asynchronous iterator as the loop won't wait for asynchronous operations before continuing. The async library has a few options for this, if you're up for trying it.
So, with multiple treeSets, concat, and async.forEachSeries, you could try:
function dig(tree_id, callback) {
var treeSet = [];
hasLines(tree_id, function (yep) {
if (yep) {
treeSet.push(tree_id);
}
pool.query('SELECT * from tree where parent_id = ?', [tree_id], function (err, rows) {
function each(row, next) {
dig(row.tree_id, function (subSet) {
treeSet = treeSet.concat(subSet);
next(null);
});
}
function done() {
callback(treeSet);
}
async.forEachSeries(rows, each, done);
});
});
}
dig(52, function (treeSet) {
res.json(treeSet);
});
you have to use async https://github.com/caolan/async
I have modified your dig function to use async's forEach method
function dig(tree_id, treeArray, AllDone) {
pool.query('SELECT * from tree where parent_id = ?', [tree_id], function (err, rows) {
if (rows) {
async.forEach(
rows,
function(row, callback) {
hasLine(row.tree_id, function(t){
if (t) {
treeArray.push(row.tree_id);
callback();
}
else {
dig(row.tree_id, treeArray, callback);
}
});
},
function(err) {
if (err) AllDone(err, treeArray);
else AllDone(null, treeArray);
});
}
else
AllDone(null, treeArray)
});
}
treeArray = [];
dig(52, treeArray, function(err, t) {
res.json(t);
});
assuming rows is an array.. forEach go through each row and perform hasLine, each iteration will call the callback function when it finish, and AllDone will be called when all callback functions are called. the tricky part here is the recursion, each recursive call will have a forEach loop, and it will call the AllDone method only when all callbacks are finish.
however forEach execute in parallel, so order is not perserved
I think this should work, if you don't care about order.
Edit : you can use forEachSeries to solve the order problem.