We Keep Coding

Loops and Callback hell

Suppose you have an Array/Object that contains a list of values. Lets say those a mysql commands or urls or filespaths. Now you want to iterate over all of them and execute some code over every entry.
for(let i = 0; i < urls.length; i++){
doSthWith(urls[i]);
}
No Problem so far. But now lets say each function has a callback and needs the result of the last execution. e.g. you request something from one website and you want to use the results of this request for one of your following requests.
for(let i = 0; i < urls.length; i++){
if(resultOfLastIteration.successful){ //or some other result besides the last one
doSthWith(urls[i]);
}
}
Now lets say the length of urls (or sth similar) is over 100. Thats why you normaly use a loop so you dont need to write the same function a 100 times. That also means that Promises wont do the trick either (except Im unaware trick a trick), because you have the same problem:
doSthWith(urls[0]).then(...
doSthWith(urls[1]).then(... //either put them inside each other
).then(...
doSthWith(urls[i]) //or in sequence
...
).catch(err){...}
Either way I dont see a way to use a loop.
A way that I found but isnt really "good" is to use the package "wait.for"(https://www.npmjs.com/package/wait.for). But what makes this package tricky is to launch a fiber each time you want to use wait.for:
//somewhere you use the function in a fiber Context
wait.for(loopedExecutionOfUrls, urls);
//function declaration
function loopedExecutionOfUrls(urls, cb){
//variables:
for(let i = 0; i < urls.length; i++){
if(someTempResultVar[i-1] === true){
someTempResultVar = wait.for(doSthWith,urls[i]);
} else if(...){...}
}
}
But Im not sure if this approach is really good, besides you always have to check if you have wrapped the whole thing in a Fiber so for each function that has loops with functions that have callbacks. Thus you have 3 levels: the lauchFiber level, wait.for(loopedFunction) level and the wait.for the callback function level. (Hope I that was formulated understandable)
So my questions is: Do you guys have a good approach where you can loop throw callback functions and can use results of those whenever you like?
good = easy to use, read, performant, not recursive,...
(Im sorry if this question is stupid, but I really have problems getting along with this asynchronous programming)

If you want to wait for doSthWith to finish before doing the same but with the nex url, you have to chain your promises and you can use array.prototype.reduce to do that:
urls = ["aaa", "bbb", "ccc", "ddd"];
urls.reduce((lastPromise, url) => lastPromise.then((resultOfPreviousPromise) => {
console.log("Result of previous request: ", resultOfPreviousPromise); // <-- Result of the previous request that you can use for the next request
return doSthWith(url);
}), Promise.resolve());
function doSthWith(arg) { // Simulate the doSthWith promise
console.log("do something with: ", arg);
return new Promise(resolve => {
setTimeout(() => resolve("result of " + arg), 2000);
});
}

Use async, specifically async.each:
const async = require('async');
function doSthWith(url, cb) {
console.log('doing something with ' + url);
setTimeout(() => cb(), 2000);
}
const urls = ['https://stackoverflow.com/', 'https://phihag.de/'];
async.each(urls, doSthWith, (err) => {
if (err) {
// In practice, likely a callback or throw here
console.error(err);
} else {
console.log('done!');
}
});
Use async.map if you are interested in the result.

When I need to loop over promises I use my handy dandy ploop function. Here is an example:
// Function that returns a promise
var searchForNumber = function(number) {
return new Promise(function(resolve, reject) {
setTimeout(function() {
var min = 1;
var max = 10;
var val = Math.floor(Math.random()*(max-min+1)+min);
console.log('Value is: ' + val.toString());
return resolve(val);
}, 1000);
});
};
// fn : function that should return a promise.
// args : the arguments that should be passed to fn.
// donefn : function that should check the result of the promise
// and return true to indicate whether ploop should stop or not.
var ploop = function(fn, args, donefn) {
return Promise.resolve(true)
.then(function() {
return(fn.apply(null, args));
})
.then(function(result) {
var finished = donefn(result);
if(finished === true){
return result;
} else {
return ploop(fn, args, donefn);
}
});
};
var searchFor = 4;
var donefn = function(result) {
return result === searchFor;
};
console.log('Searching for: ' + searchFor);
ploop(searchForNumber, [searchFor], donefn)
.then(function(val) {
console.log('Finally found! ' + val.toString());
process.exit(0);
})
.catch(function(err) {
process.exit(1);
});

Access value that promise .then returns outside of promise

Title says it all, I am using the async-library eachSeries method
let
valueOne = null,
validItem = null;
const asyncEachSeries = require('async/eachSeries');
const getContent = function (item) {
contentResponse = fetchContent(item);
return contentResponse;
}
if (recommendationsArr.length > 0) {
asyncEachSeries(recommendationsArr, function (item, callback) {
getApiContent(item).then(function getValidResult(response) {
try {
valueOne = response.content.valueOne || null; //may or may not be present
recommendationsArr.splice(recommendationsArr.indexOf(item), 1);
validItem = item;
console.log("##### valueOne is: ", valueOne); // I get data here
return true;
} catch (err) {
valueOne = null;
return false;
}
});
//I am guessing I need a return statement here, though not sure what
});
console.log("##### Outside promise, valueOne is: ", valueOne); // no data here
// Access valueOne, validItem, recommendationsArr to be able to pass as props for the component
return {
component: <Layout><ComponentName
//pass other non-dependent props
validItem={validItem}
valueOne={valueOne}
recommendationsArr={recommendationsArr}
/></Layout>,
title: `Page Title`,
};
}
return null;
Scenario: recommendationsArr is an array of items (Its 99.9% not null, but since its an external api call I prefer to have a check). The purpose is to have values for valueOne, validItem, and an updated recommendationsArr.
The validity depends on the existence of valueOne, so if the recommendationsArr[0] has valid valueOne then I don't need to fetch api results for the rest of the array. I am using eachSeries as it runs only a single async operation at a time, hence if that async gets me the valid result I don't have to iterate over other items.
Sample recommendationsArr: recommendationsArr = ["blue", "white", "green", "red"]; //usually the array is anywhere between 12-50 elements
Now in order to be able to pass the updated values to the component I need to be able to access the values set in try block outside of the asyncEachSeries iteration loop. I understand I will have to return the processed/new values but am not sure at what point and how to return those values.

As is usual with any question concerning asynchronous code you return values by using the callback. But eachSeries does not collect return values. What you want is async.mapSeries:
async.mapSeries(recommendationsArr, function (item, callback) {
getApiContent(item).then(function (response) {
try {
valueOne = response.content.valueOne || null; //may or may not be present
recommendationsArr.splice(recommendationsArr.indexOf(item), 1);
validItem = item;
callback(null, valueOne); // This is how you return a value
// return true; // return does absolutely nothing
} catch (err) {
valueOne = null;
callback(err, valueOne);
// return false;
}
});
},function (err, result) {
// this is where you can process the result
// note that the result is an array
});
However, since you are using promises you can use async-q instead which wraps caolan's async library in promises:
var asyncQ = require('async-q');
asyncQ.mapSeries(recommendationsArr, function (item) {
// Note: the return in the line below is important
return getApiContent(item).then(function (response) {
try {
valueOne = response.content.valueOne || null; //may or may not be present
recommendationsArr.splice(recommendationsArr.indexOf(item), 1);
validItem = item;
return valueOne; // this is how you return a value in a promise
} catch (err) {
return null
}
});
})
.then(function(result) {
// you can process the result here
});
Personally I'd prefer to rewrite the code and process the results at the end because the code is cleaner:
asyncQ.mapSeries(recommendationsArr, getApiContent)
.then(function (result) {
for(var i=0;i<result.length;i++) {
var response = result[i];
// do what needs to be done with each response here
}
});

Callback problems

I am new into javascript, and currently I'm trying to learning callback to my script. This script should return reduced words in array of objects
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = []
list.forEach(function (val) {
readFile(val, function(error,data){
txtObj.push(data)
})
})
function readFile(src, cb){
fs.readFile(src,'utf8', function (error,data) {
if (error) return callback(error,null)
return mapred(data)
})
}
return callback(null,txtObj)
}
But it returns empty array. Any help would be appreciated.
Thanks!

`fs.readFile`
is an asynchronous function, before it's done and result callback is invoked, you are returning the empty txtObj array.
how to fix it ?
call return callback(null,txtObj) after fs.readFile is finished running.
and also, as you are running asynchronous function on an array of items one-by-one, it might not still work the way you want. might want to use modudles like async in nodejs
Here comes an asynchronous version using module async. synchronous file operation is strongly objected :)
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = [],
async = require('async');
async.each(list, readFile, function(err) {
callback(err,txtObj)
});
function readFile(src, cb) {
fs.readFile(src,'utf8', function (error,data) {
if (error) {
cb(error);
}
else {
txtObj.push(mapred(data));
cb(null);
}
})
}
}
EDIT : You can do this without async, but it is little bit dirty isn't it ? also its OK if you remove the self invoking function inside the forEach, i included so that you can access the val, even after the callback is done
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = [],
counter = list.length,
start = 0;
list.forEach(function (val) {
(function(val)
readFile(val, function(error,data) {
txtObj.push(data);
start++;
if(error || (start === counter)) {
callback(error,txtObj);
}
}))(val);
})
function readFile(src, cb) {
fs.readFile(src,'utf8', function (error,data) {
if (error) {
cb(error);
}
else {
txtObj.push(mapred(data));
cb(null);
}
})
}
}

The reason you are getting an empty array result is that you are performing the callback before the readFile function has a chance to populate the array. You are performing multiple asynchronous actions but not letting them to complete before continuing.
If there was only one async action, you would call callback() in the callback function of readFile, but as you need to perform multiple async actions before calling callback(), you should consider using fs.readFileSync().
Sometimes sync cannot be avoided.
function mapping(list, callback)
{
var txtObj = []
list.forEach(function(val)
{
try { txtObj.push(mapred(fs.readFileSync(val, 'utf8'))) }
catch(err) { callback(err) }
})
callback(null, txtObj)
}

sync independet callbacks result

I am searching for an elegant way to sync indepentent callbacks result invoked in unknown order.
function callback1() {
var result;
};
function callback2() {
var result;
};
//When done then call
function success(res1, res2) {
// do whatever
}
I know I can do something like:
var res = {};
var dfd = $.Deferred();
function callback1() {
var result;
res.res1 = result;
(res.res1 && res.res2) && (dfd.resolve(res));
};
function callback1() {
var result;
res.res2 = result;
(res.res1 && res.res2) && (dfd.resolve(res));
};
dfd.done(function(result){
// do whatever
});
but I would appreciate if somebody comes up with more elegant solution

If you return promises (builtin promises, not jQuery deferreds) and you don't care about order, then you can use Promise.all:
function callback1() {
return Promise.resolve(1)
}
function callback2() {
return Promise.resolve(2)
}
var ps = [callback1(), callback2()]
function add(x, y) {
return x + y
}
Promise.all(ps).then(function(result) {
return result.reduce(add)
}).then(console.log) // => 3
If you want to sequence them you can do it in such a way that you can apply a curried function that expects as many arguments as resolved promises there are by lifting it into the promise world. In other words:
function apply(pa, pf) {
return pf.then(function(f) {
return pa.then(f)
})
}
function lift(f, ps) {
return ps.reduce(function(pa, pb) {
return apply(pb, pa)
}, Promise.resolve(f))
}
function add(x) {
return function(y) {
return x + y
}
}
lift(add, ps).then(console.log) //=> 3
You can also sequence them in such a way that you don't need a curried function, by collecting the results in an array first then reducing it:
function sequence(ps) {
return ps.reduceRight(function(pa, pb) {
return pa.then(function(a) {
return pb.then(function(b) {
return [b].concat(a)
})
})
}, Promise.resolve([]))
}
function add(x, y) {
return x + y
}
// This looks similar to the Promise.all approach
// but they run in order
sequence(ps).then(function(result) {
return result.reduce(add)
}).then(console.log) // => 3

There are libraries that do this, such as the async library, but here's a "from scratch" solution. I'm also avoiding promises to avoid overwhelming you, but you should really read about them as they are the most elegant solution, albeit complicated for first timers.
function runInParallel(jobs, done) {
// Store all our results in an array.
var results = [];
// If one job fails, set this to true and use it to
// ignore all job results that follow.
var failureOccurred = false;
// Iterate over each of our registered jobs.
jobs.forEach(function (runJob, index) {
// Create a jobDone callback to pass to the job.
var jobDone = function (err, result) {
// If another job failed previously, abort processing
// this job's result. We no longer care.
if (failureOccurred) return;
// If this job passed in an error, set failure to true
// and pass the error to the final done callback.
if (err) {
failureOccurred = true;
done(err);
return;
}
// If we made it this far then push the job result into
// the results array at the same position as the job in
// the jobs array.
results[index] = result;
// If the results array contains as many results as the
// jobs array had jobs then we have finished processing
// them all. Invoke our done callback with an array of
// all results.
if (results.length === jobs.length) {
done(null, results);
}
};
// Begin the job and pass in our jobDone callback.
runJob(jobDone);
});
}
This will call all of your job functions in the array, passing in a jobDone callback the job should call when finished. If any job passes an error in then the function will immediately invoke the result callback with the error and ignore everything else. If the jobs succeed then you'll end up with an array of job results in the same positions as the jobs were in the jobs array. Simply modify your job functions to accept the jobDone callback.
var jobs = [
function job1(done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
},
function job2(done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
}
];
runInParallel(jobs, function (err, results) {
if (err) {
console.error(err);
return;
}
// results[0] = jobs[0] result
// results[1] = jobs[1] result
// etc...
});
Instead of an array of jobs you could modify this code to accept an object with property names. Then instead of assigning the results to the same position as the jobs in the jobs array you could assign the results to an object using the same property names.
Example (without comments this time):
function runInParallel(jobs, done) {
var results = {};
var failureOccurred = false;
Object.keys(jobs).forEach(function (jobName) {
var jobDone = function (err, result) {
if (failureOccurred) return;
if (err) {
failureOccurred = true;
done(err);
return;
}
results[jobName] = result;
if (results.length === jobs.length) {
done(null, results);
}
};
jobs[jobName](jobDone);
});
}
Then you can consume it like this:
var jobs = {
job1: function (done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
},
job2: function (done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
}
};
runInParallel(jobs, function (err, results) {
if (err) {
console.error(err);
return;
}
// results.job1 = job1 result
// results.job2 = job2 result
// etc...
});
The parallel function in the async library does almost exactly what we've done above. It even accepts an array of jobs or an object of named jobs like we did :)

Assuming your tasks (callback1() and callback2()) are synchronous, you might choose to write a reusable generalisation of the code in the question, in the form of a function that returns a function, trapping a couple of private vars in a closure :
function resultAggregator(n, fn) {
var results = {},
count = 0;
return function(id, res) {
count++;
results[id] = res;
if (count == n) {
fn(results);
}
}
}
So after calling resultAccumulator(), you have a function that can kept in-scope of other functions or passed to other parts of your code base. It makes no assumptions about ids or the nature of the results, except that they are synchronously derived. It will fire its callback when n results have been delivered.
var myResults = resultAggregator(2, function(results) {
// do whatever;
});
//The following commands may be in different parts of your code base
myResults('id1', synchTask1());
...
myResults('id2', synchTask2());
...
myResults('id3', synchTask3());
//The second tasks to deliver its data (ostensibly `synchTask1()` and `synchTask2()`, but not necessarily) will trigger the callback.
Demo
This is just one way to perform result aggregation. You might do something different depending on the exact scenario. Here's a slightly different formulation which records the order in which the results arrived :
Demo
Whatever you write, Deferreds/Promises are not necessary for the aggregation of synchronously derived data.
However, if any one task is, or may be, asynchronous then you may need a promise aggregator, eg jQuery.when() or Promise.all(), somewhere in the pattern.

Recursively traverse tree in Javascript

This is super simple task to do in Java but the asynchronous nature of javascript makes this task(for me) almost impossible, at least with my knowledge now.(I'm not trying to bash javascript. Love the language!).
It's very basic. A top level tree has a parent of null in my mysql database. It's easy finding children. The children have lines available to them. The depth of the tree is variable.
private static Set<Tree> getBranches( Tree trunk ) {
Set<Tree> treeSet = new HashSet<Tree>();
if ( trunk != null ) {
if ( trunk.hasLines() ) { //queries if tree has lines. returns true or false
treeSet.add( trunk );
}
for ( Tree tree : trunk.treeList ) {
treeSet.addAll( getBranches( tree ) );
}
}
return treeSet;
}
Basically the method tests if the tree has lines available. If it does it adds all of those to a set. If not it continues until it finds lines.
The asynchronous nature of the mysql node library turns this task into hell.
Here is what I have now
function hasLines(tree_id, callback) {
var ret;
pool.query('SELECT * from pkg_line_tree where tree_id = ?', [tree_id], function (err, rows) {
if (rows.length > 0) {
ret = true;
} else {
ret = false;
}
callback(ret);
});
}
function dig(tree_id, treeArray, callback) {
pool.query('SELECT * from tree where parent_id = ?', [tree_id], function (err, rows) {
if (rows) {
for (var i in rows) {
hasLines(rows[i].tree_id, function (t) {
if (t) {
treeArray.push(rows[i].tree_id);
} else {
treeArray.concat(dig(rows[i].tree_id, treeArray));
}
});
}
if (callback) {
callback(treeArray);
}
}
});
return treeArray;
}
var treeArray = [];
dig(52, treeArray, function (t) {
res.json(t);
});
I really just need to output all the children available in this root tree.
Please let me know if this doesn't make sense. I'll try to refactor. I'm hoping I got some kind of point across. I'd hate to use something like Fibers to get this done but I'm out of options. Thanks.

Your use of dig() isn't currently consistent:
// asynchronous with callback
dig(52, treeArray, function (t) {
res.json(t);
});
// then synchronous with `return`?
treeArray.concat(dig(rows[i].tree_id, treeArray));
Also, the concat in the last line isn't actually doing much, since it doesn't alter the array it's called on. You probably wouldn't actually want it to as dig passes around the treeArray rather than defining a new treeSet like in getBranches. So, if it did, it would append treeArray onto the end of itself each time.
You could still use concat with multiple treeSets, but you'd have to store its return value:
treeSet = treeSet.concat(subSet);
And, you'll have to replace the for loop this with an asynchronous iterator as the loop won't wait for asynchronous operations before continuing. The async library has a few options for this, if you're up for trying it.
So, with multiple treeSets, concat, and async.forEachSeries, you could try:
function dig(tree_id, callback) {
var treeSet = [];
hasLines(tree_id, function (yep) {
if (yep) {
treeSet.push(tree_id);
}
pool.query('SELECT * from tree where parent_id = ?', [tree_id], function (err, rows) {
function each(row, next) {
dig(row.tree_id, function (subSet) {
treeSet = treeSet.concat(subSet);
next(null);
});
}
function done() {
callback(treeSet);
}
async.forEachSeries(rows, each, done);
});
});
}
dig(52, function (treeSet) {
res.json(treeSet);
});

you have to use async https://github.com/caolan/async
I have modified your dig function to use async's forEach method
function dig(tree_id, treeArray, AllDone) {
pool.query('SELECT * from tree where parent_id = ?', [tree_id], function (err, rows) {
if (rows) {
async.forEach(
rows,
function(row, callback) {
hasLine(row.tree_id, function(t){
if (t) {
treeArray.push(row.tree_id);
callback();
}
else {
dig(row.tree_id, treeArray, callback);
}
});
},
function(err) {
if (err) AllDone(err, treeArray);
else AllDone(null, treeArray);
});
}
else
AllDone(null, treeArray)
});
}
treeArray = [];
dig(52, treeArray, function(err, t) {
res.json(t);
});
assuming rows is an array.. forEach go through each row and perform hasLine, each iteration will call the callback function when it finish, and AllDone will be called when all callback functions are called. the tricky part here is the recursion, each recursive call will have a forEach loop, and it will call the AllDone method only when all callbacks are finish.
however forEach execute in parallel, so order is not perserved
I think this should work, if you don't care about order.
Edit : you can use forEachSeries to solve the order problem.