We Keep Coding

Nodejs setInterval and run first immediatly not after interval

I try to implement a loop in my noejs app that will always wait between the tasks. For this I found the setInterval function and I thought it is the solution for me. But as I found out, the first Interval, means the very first action also wait until the interval is ready. But I want that the first action runs immediatly and then each action with the given interval.
In arry scope:
myArray[0] starts immediatly while myArray[1..10] will start with Interval waiting time.
I tried it with:
function rollDice(profilearray, browserarray, url) {
return new Promise((resolve, reject) => {
var i = 0;
const intervalId = setInterval(
(function exampleFunction() {
console.log(profilearray[i].profil);
//########################################################################
createTestCafe("localhost", 1337, 1338, void 0, true)
.then((tc) => {
testcafe = tc;
runner = testcafe.createRunner();
inputStore.metaUrl = url;
inputStore.metaLogin = teamdataarray[0].email;
inputStore.metaPassword = teamdataarray[0].password;
inputStore.moderator = profilearray[i].profil;
inputStore.message = profilearray[i].template;
inputStore.channelid = profilearray[i].channelid;
})
.then(() => {
return runner
.src([__basedir + "/tests/temp.js"])
.browsers(browserarray)
.screenshots("", false)
.run()
.then((failedCount) => {
testcafe.close();
if (failedCount > 0) {
console.log(profilearray[i].profil);
console.log("No Success. Fails: " + failedCount);
//clearInterval(intervalId);
//reject("Error");
} else {
console.log(profilearray[i].profil);
console.log("All success");
//clearInterval(intervalId);
//resolve("Fertig");
}
});
})
.catch((error) => {
testcafe.close();
console.log(profilearray[i].profil);
console.log("Testcafe Error" + error);
//clearInterval(intervalId);
//reject("Error");
});
//######################################################################
i++;
console.log("Counter " + i);
if (i === profilearray.length) {
clearInterval(intervalId);
resolve("Fertig");
}
return exampleFunction;
})(),
3000
); //15 * 60 * 1000 max time to wait (user input)
});
}
The way I have done works bad because in the first action it will not start the testcafe. But in all other actions it will do.
Anybody knows a better way to do this?
Scope:
Give a array of data and for each array start testcafe with a given waiting time. 3 seconds up to 15 minutes. Because in some cases 15 Minutes is a long time I want to start the first one without any waiting time.
Iam open for any suggestion

For modern JavaScript await and async should be used instead of then and catch.
This will make many things easier, and the code becomes more readable. You e.g. can use a regular for loop to iterate over an array while executing asynchronous tasks within it. And use try-catch blocks in the same way as you would in synchronous code.
// a helperfunction that creates a Promise that resolves after
// x milliseconds
function wait(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds))
}
async function rollDice(profilearray, browserarray, url) {
for (let i = 0; i < profilearray.length; i++) {
// depending how you want to handle the wait you would create
// the "wait"-Promise here
// let timer = wait(3000)
let testcafe = await createTestCafe("localhost", 1337, 1338, void 0, true);
try {
let runner = testcafe.createRunner();
inputStore.metaUrl = url;
inputStore.metaLogin = teamdataarray[0].email;
inputStore.metaPassword = teamdataarray[0].password;
inputStore.moderator = profilearray[i].profil;
inputStore.message = profilearray[i].template;
inputStore.channelid = profilearray[i].channelid;
let failedCount = await runner.src([__basedir + "/tests/temp.js"])
.browsers(browserarray)
.screenshots("", false)
.run()
if (failedCount > 0) {
// ...
} else {
// ...
}
} catch (err) {
console.log(profilearray[i].profil);
console.log("Testcafe Error" + error);
} finally {
testcafe.close();
}
// Here you would wait for the "wait"-Promise to resolve:
// await timer;
// This would have similar behavior to an interval.
// Or you wait here for a certain amount of time.
// The difference is whether you want that the time the
// runner requires to run counts to the waiting time or not.
await wait(3000)
}
return "Fertig"
}

Declare function before setInterval, run setInterval(exampleFunction, time) and then run the function as usual (exampleFunction()). May not be ideal to the millisecond, but if you don't need to be perfectly precise, should work fine.
Ask if you need further assistance
EDIT: now looking twice at it, why are you calling the function you pass as parameter to setInterval inside the setInterval?

Promise not running asynchronously [duplicate]

How can I make a simple, non-block Javascript function call? For example:
//begin the program
console.log('begin');
nonBlockingIncrement(10000000);
console.log('do more stuff');
//define the slow function; this would normally be a server call
function nonBlockingIncrement(n){
var i=0;
while(i<n){
i++;
}
console.log('0 incremented to '+i);
}
outputs
"beginPage"
"0 incremented to 10000000"
"do more stuff"
How can I form this simple loop to execute asynchronously and output the results via a callback function? The idea is to not block "do more stuff":
"beginPage"
"do more stuff"
"0 incremented to 10000000"
I've tried following tutorials on callbacks and continuations, but they all seem to rely on external libraries or functions. None of them answer the question in a vacuum: how does one write Javascript code to be non-blocking!?
I have searched very hard for this answer before asking; please don't assume I didn't look. Everything I found is Node.js specific ([1], [2], [3], [4], [5]) or otherwise specific to other functions or libraries ([6], [7], [8], [9], [10], [11]), notably JQuery and setTimeout(). Please help me write non-blocking code using Javascript, not Javascript-written tools like JQuery and Node. Kindly reread the question before marking it as duplicate.

To make your loop non-blocking, you must break it into sections and allow the JS event processing loop to consume user events before carrying on to the next section.
The easiest way to achieve this is to do a certain amount of work, and then use setTimeout(..., 0) to queue the next chunk of work. Crucially, that queueing allows the JS event loop to process any events that have been queued in the meantime before going on to the next piece of work:
function yieldingLoop(count, chunksize, callback, finished) {
var i = 0;
(function chunk() {
var end = Math.min(i + chunksize, count);
for ( ; i < end; ++i) {
callback.call(null, i);
}
if (i < count) {
setTimeout(chunk, 0);
} else {
finished.call(null);
}
})();
}
with usage:
yieldingLoop(1000000, 1000, function(i) {
// use i here
}, function() {
// loop done here
});
See http://jsfiddle.net/alnitak/x3bwjjo6/ for a demo where the callback function just sets a variable to the current iteration count, and a separate setTimeout based loop polls the current value of that variable and updates the page with its value.

SetTimeout with callbacks is the way to go. Though, understand your function scopes are not the same as in C# or another multi-threaded environment.
Javascript does not wait for your function's callback to finish.
If you say:
function doThisThing(theseArgs) {
setTimeout(function (theseArgs) { doThatOtherThing(theseArgs); }, 1000);
alert('hello world');
}
Your alert will fire before the function you passed will.
The difference being that alert blocked the thread, but your callback did not.

There are in general two ways to do this as far as I know. One is to use setTimeout (or requestAnimationFrame if you are doing this in a supporting environment). #Alnitak shown how to do this in another answer. Another way is to use a web worker to finish your blocking logic in a separate thread, so that the main UI thread is not blocked.
Using requestAnimationFrame or setTimeout:
//begin the program
console.log('begin');
nonBlockingIncrement(100, function (currentI, done) {
if (done) {
console.log('0 incremented to ' + currentI);
}
});
console.log('do more stuff');
//define the slow function; this would normally be a server call
function nonBlockingIncrement(n, callback){
var i = 0;
function loop () {
if (i < n) {
i++;
callback(i, false);
(window.requestAnimationFrame || window.setTimeout)(loop);
}
else {
callback(i, true);
}
}
loop();
}
Using web worker:
/***** Your worker.js *****/
this.addEventListener('message', function (e) {
var i = 0;
while (i < e.data.target) {
i++;
}
this.postMessage({
done: true,
currentI: i,
caller: e.data.caller
});
});
/***** Your main program *****/
//begin the program
console.log('begin');
nonBlockingIncrement(100, function (currentI, done) {
if (done) {
console.log('0 incremented to ' + currentI);
}
});
console.log('do more stuff');
// Create web worker and callback register
var worker = new Worker('./worker.js'),
callbacks = {};
worker.addEventListener('message', function (e) {
callbacks[e.data.caller](e.data.currentI, e.data.done);
});
//define the slow function; this would normally be a server call
function nonBlockingIncrement(n, callback){
const caller = 'nonBlockingIncrement';
callbacks[caller] = callback;
worker.postMessage({
target: n,
caller: caller
});
}
You cannot run the web worker solution as it requires a separate worker.js file to host worker logic.

You cannot execute Two loops at the same time, remember that JS is single thread.
So, doing this will never work
function loopTest() {
var test = 0
for (var i; i<=100000000000, i++) {
test +=1
}
return test
}
setTimeout(()=>{
//This will block everything, so the second won't start until this loop ends
console.log(loopTest())
}, 1)
setTimeout(()=>{
console.log(loopTest())
}, 1)
If you want to achieve multi thread you have to use Web Workers, but they have to have a separated js file and you only can pass objects to them.
But, I've managed to use Web Workers without separated files by genering Blob files and i can pass them callback functions too.
//A fileless Web Worker
class ChildProcess {
//#param {any} ags, Any kind of arguments that will be used in the callback, functions too
constructor(...ags) {
this.args = ags.map(a => (typeof a == 'function') ? {type:'fn', fn:a.toString()} : a)
}
//#param {function} cb, To be executed, the params must be the same number of passed in the constructor
async exec(cb) {
var wk_string = this.worker.toString();
wk_string = wk_string.substring(wk_string.indexOf('{') + 1, wk_string.lastIndexOf('}'));
var wk_link = window.URL.createObjectURL( new Blob([ wk_string ]) );
var wk = new Worker(wk_link);
wk.postMessage({ callback: cb.toString(), args: this.args });
var resultado = await new Promise((next, error) => {
wk.onmessage = e => (e.data && e.data.error) ? error(e.data.error) : next(e.data);
wk.onerror = e => error(e.message);
})
wk.terminate(); window.URL.revokeObjectURL(wk_link);
return resultado
}
worker() {
onmessage = async function (e) {
try {
var cb = new Function(`return ${e.data.callback}`)();
var args = e.data.args.map(p => (p.type == 'fn') ? new Function(`return ${p.fn}`)() : p);
try {
var result = await cb.apply(this, args); //If it is a promise or async function
return postMessage(result)
} catch (e) { throw new Error(`CallbackError: ${e}`) }
} catch (e) { postMessage({error: e.message}) }
}
}
}
setInterval(()=>{console.log('Not blocked code ' + Math.random())}, 1000)
console.log("starting blocking synchronous code in Worker")
console.time("\nblocked");
var proc = new ChildProcess(blockCpu, 43434234);
proc.exec(function(block, num) {
//This will block for 10 sec, but
block(10000) //This blockCpu function is defined below
return `\n\nbla bla ${num}\n` //Captured in the resolved promise
}).then(function (result){
console.timeEnd("\nblocked")
console.log("End of blocking code", result)
})
.catch(function(error) { console.log(error) })
//random blocking function
function blockCpu(ms) {
var now = new Date().getTime();
var result = 0
while(true) {
result += Math.random() * Math.random();
if (new Date().getTime() > now +ms)
return;
}
}

For very long tasks, a Web-Worker should be preferred, however for small-enough tasks (< a couple of seconds) or for when you can't move the task to a Worker (e.g because you needs to access the DOM or whatnot, Alnitak's solution of splitting the code in chunks is the way to go.
Nowadays, this can be rewritten in a cleaner way thanks to async/await syntax.
Also, instead of waiting for setTimeout() (which is delayed to at least 1ms in node-js and to 4ms everywhere after the 5th recursive call), it's better to use a MessageChannel.
So this gives us
const waitForNextTask = () => {
const { port1, port2 } = waitForNextTask.channel ??= new MessageChannel();
return new Promise( (res) => {
port1.addEventListener("message", () => res(), { once: true } );
port1.start();
port2.postMessage("");
} );
};
async function doSomethingSlow() {
const chunk_size = 10000;
// do something slow, like counting from 0 to Infinity
for (let i = 0; i < Infinity; i++ ) {
// we've done a full chunk, let the event-loop loop
if( i % chunk_size === 0 ) {
log.textContent = i; // just for demo, to check we're really doing something
await waitForNextTask();
}
}
console.log("Ah! Did it!");
}
console.log("starting my slow computation");
doSomethingSlow();
console.log("started my slow computation");
setTimeout(() => console.log("my slow computation is probably still running"), 5000);
<pre id="log"></pre>

Using ECMA async function it's very easy to write non-blocking async code, even if it performs CPU-bound operations. Let's do this on a typical academic task - Fibonacci calculation for the incredible huge value.
All you need is to insert an operation that allows the event loop to be reached from time to time. Using this approach, you will never freeze the user interface or I/O.
Basic implementation:
const fibAsync = async (n) => {
let lastTimeCalled = Date.now();
let a = 1n,
b = 1n,
sum,
i = n - 2;
while (i-- > 0) {
sum = a + b;
a = b;
b = sum;
if (Date.now() - lastTimeCalled > 15) { // Do we need to poll the eventloop?
lastTimeCalled = Date.now();
await new Promise((resolve) => setTimeout(resolve, 0)); // do that
}
}
return b;
};
And now we can use it (Live Demo):
let ticks = 0;
console.warn("Calulation started");
fibAsync(100000)
.then((v) => console.log(`Ticks: ${ticks}\nResult: ${v}`), console.warn)
.finally(() => {
clearTimeout(timer);
});
const timer = setInterval(
() => console.log("timer tick - eventloop is not freezed", ticks++),
0
);
As we can see, the timer is running normally, which indicates the event loop is not blocking.
I published an improved implementation of these helpers as antifreeze2 npm package. It uses setImmediate internally, so to get the maximum performance you need to import setImmediate polyfill for environments without native support.
Live Demo
import { antifreeze, isNeeded } from "antifreeze2";
const fibAsync = async (n) => {
let a = 1n,
b = 1n,
sum,
i = n - 2;
while (i-- > 0) {
sum = a + b;
a = b;
b = sum;
if (isNeeded()) {
await antifreeze();
}
}
return b;
};

If you are using jQuery, I created a deferred implementation of Alnitak's answer
function deferredEach (arr, batchSize) {
var deferred = $.Deferred();
var index = 0;
function chunk () {
var lastIndex = Math.min(index + batchSize, arr.length);
for(;index<lastIndex;index++){
deferred.notify(index, arr[index]);
}
if (index >= arr.length) {
deferred.resolve();
} else {
setTimeout(chunk, 0);
}
};
setTimeout(chunk, 0);
return deferred.promise();
}
Then you'll be able to use the returned promise to manage the progress and done callback:
var testArray =["Banana", "Orange", "Apple", "Mango"];
deferredEach(testArray, 2).progress(function(index, item){
alert(item);
}).done(function(){
alert("Done!");
})

I managed to get an extremely short algorithm using functions. Here is an example:
let l=($,a,f,r)=>{f(r||0),$((r=a(r||0))||0)&&l($,a,f,r)};
l
(i => i < 4, i => i+1, console.log)
/*
output:
0
1
2
3
*/
I know this looks very complicated, so let me explain what is really going on here.
Here is a slightly simplified version of the l function.
let l_smpl = (a,b,c,d) => {c(d||0);d=b(d||0),a(d||0)&&l_smpl(a,b,c,d)||0}
First step in the loop, l_smpl calls your callback and passes in d - the index. If d is undefined, as it would be on the first call, it changes it to 0.
Next, it updates d by calling your updater function and setting d to the result. In our case, the updater function would add 1 to the index.
The next step checks if your condition is met by calling the first function and checking if the value is true meaning the loop is not done. If so, it calls the function again, or otherwise, it returns 0 to end the loop.

Implementing exponential backoff with Node.js and Promises

I'm making about 70 requests to an API in my code. I'm getting an error response telling me that I'm making requests too quickly one after the other and I have decided to use the idea of exponential backoff to get through this problem.
Currently, this is what my code looks like:
let backoffTime = 1;
for (let i = 0; i < fileIds.length; i++) {
let fileId = fileIds[i];
getFileName(fileId, auth)
.then((name) => {
// do some work
})
.catch((err) => {
// assumes that the error is "request made too soon"
backoff(backoffTime);
backoffTime *= 2;
i--;
console.log(err);
});
}
function backoff(time) {
let milliseconds = time * 1000;
let start = (new Date()).getTime();
while (((new Date()).getTime() - start) < milliseconds) {
// do nothing
}
}
My getFileName function makes the request to the API and returns a Promise.
Currently this does not work because Promises are async (kinda). My for loop runs really fast and calls the getFileName function which makes those API requests really fast. Then, it gets the error for some of the API calls in which case it updates the backoffTime. This implementation doesn't work.
Any idea how I can implement this correctly?

First of all blocking the browser with a nearly infinite loop is a very very bad idea, just use promises:
const delay = ms => new Promise(res => setTimeout(res, ms));
Then just await the promise before continuing the loop and use the delay:
(async function() {
for (let i = 0; i < fileIds.length; i++) {
let fileId = fileIds[i];
await getFileName(fileId, auth)
.then((name) => {
// do some work
})
.catch((err) => {
// assumes that the error is "request made too soon"
backoffTime *= 2;
i--;
console.log(err);
return delay(backoffTime);
});
}
})();

The easiest way is to use async/await and then either await each request, or if it is too slow for you, then create chunks with i.e. 15 requests and Promise.all such chunks.
You can also use this: https://caolan.github.io/async/parallelLimit.js.html
It will require some extra work to transition promises into callbacks and vice versa, but it will do the best job.
This is the function: parallelLimit(tasks, limit, callback)
const tasks = [];
// This for-cycle will just prepare the tasks into array of functions
for (let i = 0; i < fileIds.length; i++) {
tasks.push(callback => doWorkThatReturnsPromise(fileIds[i])
.then(val => callback(null, val))
.catch(callback));
}
async.parallelLimit(tasks, 15, (err, values) => {
// do something after it finishes
})

You can fix that using closures, here is an example:
for (let i = 0; i < fileIds.length; i++) {
let fileId = fileIds[i];
doCall(fileId);
}
function doCall(fileId, backoffTime = 1){
getFileName(fileId, auth)
.then((name) => {
// do some work
})
.catch((err) => {
setTimeout(() => {
doCall(fileId, (backoffTime * 2));
}, backoffTime * 1000);
});
}
I've replaced the backoff function which pauses the execution using a while loop with a setTimeout call.
This code can cause an infinite recursion, you should probably add some checks to prevent that.

Loops and Callback hell

Suppose you have an Array/Object that contains a list of values. Lets say those a mysql commands or urls or filespaths. Now you want to iterate over all of them and execute some code over every entry.
for(let i = 0; i < urls.length; i++){
doSthWith(urls[i]);
}
No Problem so far. But now lets say each function has a callback and needs the result of the last execution. e.g. you request something from one website and you want to use the results of this request for one of your following requests.
for(let i = 0; i < urls.length; i++){
if(resultOfLastIteration.successful){ //or some other result besides the last one
doSthWith(urls[i]);
}
}
Now lets say the length of urls (or sth similar) is over 100. Thats why you normaly use a loop so you dont need to write the same function a 100 times. That also means that Promises wont do the trick either (except Im unaware trick a trick), because you have the same problem:
doSthWith(urls[0]).then(...
doSthWith(urls[1]).then(... //either put them inside each other
).then(...
doSthWith(urls[i]) //or in sequence
...
).catch(err){...}
Either way I dont see a way to use a loop.
A way that I found but isnt really "good" is to use the package "wait.for"(https://www.npmjs.com/package/wait.for). But what makes this package tricky is to launch a fiber each time you want to use wait.for:
//somewhere you use the function in a fiber Context
wait.for(loopedExecutionOfUrls, urls);
//function declaration
function loopedExecutionOfUrls(urls, cb){
//variables:
for(let i = 0; i < urls.length; i++){
if(someTempResultVar[i-1] === true){
someTempResultVar = wait.for(doSthWith,urls[i]);
} else if(...){...}
}
}
But Im not sure if this approach is really good, besides you always have to check if you have wrapped the whole thing in a Fiber so for each function that has loops with functions that have callbacks. Thus you have 3 levels: the lauchFiber level, wait.for(loopedFunction) level and the wait.for the callback function level. (Hope I that was formulated understandable)
So my questions is: Do you guys have a good approach where you can loop throw callback functions and can use results of those whenever you like?
good = easy to use, read, performant, not recursive,...
(Im sorry if this question is stupid, but I really have problems getting along with this asynchronous programming)

If you want to wait for doSthWith to finish before doing the same but with the nex url, you have to chain your promises and you can use array.prototype.reduce to do that:
urls = ["aaa", "bbb", "ccc", "ddd"];
urls.reduce((lastPromise, url) => lastPromise.then((resultOfPreviousPromise) => {
console.log("Result of previous request: ", resultOfPreviousPromise); // <-- Result of the previous request that you can use for the next request
return doSthWith(url);
}), Promise.resolve());
function doSthWith(arg) { // Simulate the doSthWith promise
console.log("do something with: ", arg);
return new Promise(resolve => {
setTimeout(() => resolve("result of " + arg), 2000);
});
}

Use async, specifically async.each:
const async = require('async');
function doSthWith(url, cb) {
console.log('doing something with ' + url);
setTimeout(() => cb(), 2000);
}
const urls = ['https://stackoverflow.com/', 'https://phihag.de/'];
async.each(urls, doSthWith, (err) => {
if (err) {
// In practice, likely a callback or throw here
console.error(err);
} else {
console.log('done!');
}
});
Use async.map if you are interested in the result.

When I need to loop over promises I use my handy dandy ploop function. Here is an example:
// Function that returns a promise
var searchForNumber = function(number) {
return new Promise(function(resolve, reject) {
setTimeout(function() {
var min = 1;
var max = 10;
var val = Math.floor(Math.random()*(max-min+1)+min);
console.log('Value is: ' + val.toString());
return resolve(val);
}, 1000);
});
};
// fn : function that should return a promise.
// args : the arguments that should be passed to fn.
// donefn : function that should check the result of the promise
// and return true to indicate whether ploop should stop or not.
var ploop = function(fn, args, donefn) {
return Promise.resolve(true)
.then(function() {
return(fn.apply(null, args));
})
.then(function(result) {
var finished = donefn(result);
if(finished === true){
return result;
} else {
return ploop(fn, args, donefn);
}
});
};
var searchFor = 4;
var donefn = function(result) {
return result === searchFor;
};
console.log('Searching for: ' + searchFor);
ploop(searchForNumber, [searchFor], donefn)
.then(function(val) {
console.log('Finally found! ' + val.toString());
process.exit(0);
})
.catch(function(err) {
process.exit(1);
});

Node.js synchronously loop or iterate over asynchronous statements

I want to do a for each loop but have it run synchronously. Each iteration of the loop will do an http.get call and that will return json for it to insert the values into a database. The problem is that the for loop runs asynchronously and that causes all of the http.gets to all run at once and my database doesn't end up inserting all of the data.I am using async-foreach to try to do what I want it to do, but I don't have to use it if I can do it the right way.
mCardImport = require('m_cardImport.js');
var http = require('http');
app.get('/path/hi', function(req, res) {
mCardImport.getList(function(sets) {
forEach(sets, function(item, index, arr) {
theUrl = 'http://' + sets.set_code + '.json';
http.get(theUrl, function(res) {
var jsonData = '';
res.on('data', function(chunk) {
jsonData += chunk;
});
res.on('end', function() {
var theResponse = JSON.parse(jsonData);
mCardImport.importResponse(theResponse.list, theResponse.code, function(theSet) {
console.log("SET: " + theSet);
});
});
});
});
});
});
and my model
exports.importResponse = function(cardList, setCode, callback) {
mysqlLib.getConnection(function(err, connection) {
forEach(cardList, function(item, index, arr) {
var theSql = "INSERT INTO table (name, code, multid, collector_set_num) VALUES "
+ "(?, ?, ?, ?) ON DUPLICATE KEY UPDATE id=id";
connection.query(theSql, [item.name, setCode, item.multid, item.number], function(err, results) {
if (err) {
console.log(err);
};
});
});
});
callback(setCode);
};

With recursion the code is pretty clean. Wait for the http response to come back then fire off next attempt. This will work in all versions of node.
var urls = ['http://stackoverflow.com/', 'http://security.stackexchange.com/', 'http://unix.stackexchange.com/'];
var processItems = function(x){
if( x < urls.length ) {
http.get(urls[x], function(res) {
// add some code here to process the response
processItems(x+1);
});
}
};
processItems(0);
A solution using promises would also work well, and is more terse. For example, if you have a version of get that returns a promise and Node v7.6+, you could write an async/await function like this example, which uses some new JS features.
const urls = ['http://stackoverflow.com/', 'http://security.stackexchange.com/', 'http://unix.stackexchange.com/'];
async function processItems(urls){
for(const url of urls) {
const response = await promisifiedHttpGet(url);
// add some code here to process the response.
}
};
processItems(urls);
Note: both of these examples skip over error handling, but you should probably have that in a production app.

To loop and synchronously chain asynchronous actions, the cleanest solution is probably to use a promise library (promises are being introduced in ES6, this is the way to go).
Using Bluebird, this could be
Var p = Promise.resolve();
forEach(sets, function(item, index, arr) {
p.then(new Promise(function(resolve, reject) {
http.get(theUrl, function(res) {
....
res.on('end', function() {
...
resolve();
}
}));
});
p.then(function(){
// all tasks launched in the loop are finished
});

I found out that I wasn't releasing my mysql connections after I was done with each call and this tied up the connections causing it to fail and appear to be an issue with synchronization.
After explicitly calling connection.release(); it caused my code to work 100% correctly even in an asynchronous fashion.
Thanks for those who posted to this question.

"use strict";
var Promise = require("bluebird");
var some = require('promise-sequence/lib/some');
var pinger = function(wht) {
return new Promise(function(resolve, reject) {
setTimeout(function () {
console.log('I`ll Be Waiting: ' + wht);
resolve(wht);
}, Math.random() * (2000 - 1500) + 1500);
});
}
var result = [];
for (var i = 0; i <= 12; i++) {
result.push(i);
}
some(result, pinger).then(function(result){
console.log(result);
});

Just wrap the loop in an async function. This example illustrates what I mean:
const oneSecond = async () =>
new Promise((res, _) => setTimeout(res, 1000));
This function completes after just 1 second:
const syncFun = () => {
for (let i = 0; i < 5; i++) {
oneSecond().then(() => console.log(`${i}`));
}
}
syncFun(); // Completes after 1 second ❌
This one works as expected, finishing after 5 seconds:
const asyncFun = async () => {
for (let i = 0; i < 5; i++) {
await oneSecond();
console.log(`${i}`);
}
}
asyncFun(); // Completes after 5 seconds ✅

var urls = ['http://stackoverflow.com/', 'http://security.stackexchange.com/', 'http://unix.stackexchange.com/'];
for (i = 0; i < urls.length; i++){
http.get(urls[i], function(res) {
// add some code here to process the response
});
}