We Keep Coding

How do I make a nested loop continue only after a asynchronous function has been resolved or how do I extend ".then" beyond the scope

I tried to prevent async problems with promises in the following code. By using a .then function everything within that function gets called after the function has been resolved. But now I have the problem that neither can I extend the scope of the ".then function" enough to include the bits after the second loop nor can I to my knowledge easily pause the code until the function has been properly resolved and THEN continue with the loop iteration.
Here's my main code(simplified):
let total = []
$.each(element, function(data) {
//Some other code
let out;
$.each(element2, function(data2) {
getZip(data2).then(function(txt){ //after everything has finished this get's called
out = someFunction(txt,data2);
total.push(out);
});
)};
console.log(total)//this gets called first
//some other code that does some stuff with total
)};
Here's the getZip code which is asynchronous:
function getZip(zipFile) {
return new Promise(function (resolve, reject){
zip = new JSZip()
JSZipUtils.getBinaryContent("someURL/" + zipFile, function (err, data) {
if (err) {
reject(err)
}
JSZip.loadAsync(data).then(function (zip) {
return zip.file(zipFile.replace(".zip", "")).async("text"); //gets the file within the zip andoutputs as text
}).then(function (txt) {
resolve(txt)
});
});
});
}
I'd be happy if either the getZip code could be made synchronous or if the before mentioned could be done.

I do not think I fully understand the code you have written. However, I recommend you use Promise.all. Here is an example I have written that I hope helps guide you:
let total = [];
$.each([1,2,3,4], function (data) {
// Some other code.
let out;
// Create a new promise so that we can wait on the getZip method.
new Promise(function (resolve, reject) {
// Create a holder variable. This variable with hold all the promises that are output from the getZip method you have.
let gZipPromises = [];
$.each([5,6,7,8], function (data2) {
// Your getZip method would go here. wrap the call to getZip in gZipPromises.push to push all the returned promises onto the holding variable.
gZipPromises.push(new Promise(function (resolve2, reject2) {
// Sample Code
setTimeout(function () {
total.push(data2);
resolve2("");
}, 10);
// End Sample Code.
}));
});
// Pass the holding variable to Promise.all so that all promises in the holding variable are executed before resolving.
Promise.all(gZipPromises).then(function() {
resolve()
});
}).then(function () {
// This will be called only when all getZip promises are completed in the second loop.
console.log(total);
});
});
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
With that said, I could not test your code. But I think this would work:
(Please note that based on the code you provided, the variable total would be logged for each iteration of the top most $.each
let total = []
$.each(element, function(data) {
//Some other code
let out;
// Define a new promise.
new Promise(function (resolve, reject) {
let gZipPromises = [];
$.each(element2, function(data2) {
gZipPromises.push(
getZip(data2).then(function(txt){ //after everything has finished this get's called
out = someFunction(txt,data2);
total.push(out);
});
);
)};
Promise.all(gZipPromises).then(function() {
resolve()
});
}).then(function () {
console.log(total)
});
)};

const elements = [["foo.zip"],["bar.zip"],["baz.zip"]];
const totalOut = getAllZips(elements)
.then(text => console.info(text))
.catch(error => console.error(error))
function someFunction(text, data) {
return `${text}\nLength: ${data.length}`;
}
async function getAllZips(elements) {
let promises = [];
for(const element of elements) {
for(const data of element) {
promises.push(getZip(data).then(text => {
return someFunction(text, data);
}));
}
}
return Promise.all(promises);
}
async function getZip(file) {
return new Promise((resolve, reject) => {
JSZipUtils.getBinaryContent(`someURL/${file}`, async (err, data) => {
try {
if (err) throw err;
const zip = await JSZip.loadAsync(data);
const name = file.replace(".zip", "");
resolve(await zip.file(name).async('text'));
} catch(error) {
reject(error);
}
});
});
}
<script>/*IGNORE*/const JSZipUtils = {getBinaryContent:(p,c)=>errs.gbc?c(new Error('gbc'),null):c(null,{foo:true})};const JSZip = {loadAsync:(d)=>errs.la?Promise.reject(new Error('la')):({file:n=>({async:a=>errs.a?Promise.reject(new Error('a')):Promise.resolve('Hello World')})})};const errs = {gbc:false,la:false,a:false};/*IGNORE*/</script>

This kind of sounds like a use case for async iterator generators, but maybe I'm just over-engineering. You have a bunch of resources that you want to iterate over and their contents are asynchronous. You want it to "look" synchronous, so you can leverage async/await:
function getZip(zipFile) {
/*
* Theres no point in simplifying this function since it looks like
* the JSZip API deals with callbacks and not Promises.
*/
return Promise.resolve(zipFile);
}
function someFn(a, b) {
return `${a}: ${b.length}`;
}
async function* zipper(elements) {
for (const element of elements) {
for (const data of element) {
const txt = await getZip(data);
yield someFn(txt, data);
}
}
}
(async() => {
const elements = [
["hello"],
["world"],
["foo"],
["bar"]
];
let total = [];
for await (const out of zipper(elements)) {
total.push(out);
}
console.log(total);
})();

How to wait for each iteration in for loop and return response as API response in nodeJS

I'm using for loop to iterate over an array of elements and to call the same function with different parameters inside the for loop. Here is my code:
exports.listTopSongs = function(query) {
return new Promise(function(resolve, reject) {
var str = query.split(","), category,
for(var i=0; i<str.length; i++) {
sampleFn(str[i], 'sample', resolve, reject);
}
});
};
function sampleFn(lang, cat, resolve, reject) {
client.on("error", function (err) {
console.log(err);
var err = new Error('Exception in redis client connection')
reject(err);
});
client.keys(lang, function (err, keys){
if (err) return console.log(err);
if(keys.length != 0) {
client.hgetall(keys, function (error, value) {
var objects = Object.keys(value);
result['title'] = lang;
result[cat] = [];
var x =0;
for(x; x<objects.length; x++) {
var val = objects[x];
User.findAll({attributes: ['X', 'Y', 'Z'],
where: {
A: val
}
}).then(data => {
if(data != null) {
//some actions with data and stored it seperately in a Json array
if(result[cat].length == objects.length) {
resolve(result);
}
} else {
console.log(""+cat+" is not avilable for this value "+data.dataValues['X']);
}
});
}
});
});
}
Here it won't wait for completion of first iteration. It just run asyncronously before completing first iteration function. I need to return the result as result:[{ 1, 2}, {3,4}]. but it runs seamlessly and returns empty or only one object before completing all. How to resolve it.
I used node-async-loop. But it uses next and i can't able to send my parameteres while using that package. Please help me

Async provides control flow methods allowing to do so.
Using async.each:
async.each(openFiles, function(file, callback) {
// Perform operation on file here.
console.log('Processing file ' + file);
if( file.length > 32 ) {
console.log('This file name is too long');
callback('File name too long');
} else {
// Do work to process file here
console.log('File processed');
callback();
}
}, function(err) {
// if any of the file processing produced an error, err would equal that error
if( err ) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('A file failed to process');
} else {
console.log('All files have been processed successfully');
}
});

If you don't want to use a library, you can code it yourself. It would also be very instructive. I took your issue and coded a dummy async loop :
function listTopSongs(query) {
return new Promise(async(resolve, reject) => { //add async here in order to do asynchronous calls
const str = query.split(",") //str is const, and the other variable was not used anyway
for( let i = 0;i < str.length; i++) {
const planet = await sampleFn(str[i], 'sample', resolve, reject)
console.log(planet)
}
});
};
function sampleFn(a, b, c, d) {
return fetch(`https://swapi.co/api/planets/${a}/`)
.then(r => r.json())
.then(rjson => (a + " : " + rjson.name))
}
listTopSongs("1,2,3,4,5,6,7,8,9")
I used some dummy star wars API to fake a long promise but it should work with your sampleFn. Be careful, it is very, very slow if you have network call like the one in the example.
EDIT: I ran your code and I noticed there are a few mistakes: there is no resolve in your promise so it's not a thenable (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/resolve see thenable )
Here is a fully working code. The nice part : no library needed, no dependencies.
//for node.js, use node-fetch :
//const fetch = require("node-fetch")
function listTopSongs(query) {
return new Promise(async(resolve, reject) => { //add async here in order to do asynchronous calls
const str = query.split(",") //str is const, and the other variable was not used anyway
const planets = []
for (let i = 0; i < str.length; i++) {
const planet = await sampleFn(i + 1, str[i], resolve, reject)
planets[i] = planet
console.log(planet)
}
resolve(planets)
});
};
function sampleFn(a, b, c, d) {
return fetch(`https://swapi.co/api/planets/${a}/`)
.then(r => r.json())
.then(rjson => (a + b + " : " + rjson.name))
}
listTopSongs("a,b,c,d").then(planets => console.log(planets))

Since you are using promise, you can do something like this
exports.listTopSongs = function(query) {
return Promise.resolve(true).then(function(){
var str = query.split(",");
var promises = str.map(function(s){
return sampleFn(str[i], 'sample');
});
return Promise.all(promises);
}).then(function(results){
//whatever you want to do with the result
});
};
For this to work you have to change your sampleFn to not to depend on external resolve and reject functions. I don't see a reason using external resolve and reject. why just not use Promise.Resolve, Promise.Reject;

Pattern for dynamic Javascript promises

Inside a promise, I need to call and process an indeterminate number of asynch API responses after individually calling them either inside another promise, or after said promise, but before another so the order of execution is respected.
var promiseA = function() {
return new Promise(function(resolve, reject) {
// 1. Establish objects needed from one API endpoint
// 2. Call API endpoint for each object and parse
// 3. Only then continue to next promise
}
}
var finalPromise = function() {
return new Promise(function(resolve, reject) {
//
}
}
promiseA()
.then(finalPromise)
So inside promiseA, I find out how many objects I'll need to poll individually from an API. Each request is of course asynchronous. I need to make these calls and process the response before the final promise is called.
I am struggling to determine a pattern for this with promises, where I can dynamically create these promises and only allow the final promise to execute after the indeterminate and asynchronous have executed and processed. I've worked with other languages where this is possible, but I'm struggling to see it here with Promises.
Any help is appreciated.

I have changed the answer to incorporate the comments below. Since, you mentioned ES6 promises I shall stick to that. There are two basic types of callbacks that we might care about.
DOM load or other one time event callbacks (window.onload and so on)
Async method callback (AJAX call, setTimout and so on)
Since,
1.DOM load or other one time event
var p = new Promise(function(res, rej) {
window.onload = res();
};
2.Plain callback: these are callbacks that don't conform to a convention. e.g. setTimeout
var p = new Promise(function(res, rej){
setTimeout(function() {
//your business/view logic
success? res():rej(); //if successful resolve else reject
}, 2000);
});
In each of the above case the promise (var p) can be wrapped to be returned by a function.
var myAsyncMethod = function () {
var p = new ... // as mentioned in 1 or 2
return p;
}
Then the usage:
myAsyncMethod()
.then(function(){/* success-handler */})
.catch(function(/* failure-handler */));
Specific to your question you may have many such methods:
function baseAJAXCall (url) {
new Promise(functoin(rej, res) {
$.get(url, function(err, data){
if(err) {
rej();
}
else {
resolve(data);
}
});
}
};
function callAPIEndpoint(url) {
return baseAJAXCall(url);
}
function finalPromiseHandler () {
//your final business/view logic
}
//USAGE
callAPIEndpoint('/my-first-call')
.then(function(data){
var promiseArray = data.map(function(item){
return baseAJAXCall(item.url);
});
return Promise.all(promiseArray);
})
.then(finalPromiseHandler)
.catch(function(){
console.log('.error-message.');
});
Ref:
How do I convert an existing callback API to promises?.
http://www.datchley.name/es6-promises/
Links from comments below.
---OLD ANSWER: PLEASE OVERLOOK---
I am familiar with this library : https://github.com/kriskowal/q. And, you can do this using using the q.all and q.allSettled constructs. May be that is what you are looking for.
Normally, the pattern is to create a function that returns a promise.
function someAsyncFuncName1(url) {
var def = q.defer();
//async function
$.get(url, function(err, data){ //suppose
if(err){
def.reject();
}
else {
def.resolve(data); //pass the data to the .then() handler.
}
});
return def.promise;
}
function someAsyncFuncName2() {
var def = q.defer();
//async function
setTimeout(function(){ //suppose
//do something
if(good) {
def.resolve();
} else {
def.reject();
}
}, 1000); //arbitrary timeout of 1 second
return def.promise;
}
USAGE:
q.all([someAsyncFuncName1('/api-1'), someAsyncFuncName2()])
.then(function() {
//final handler
});
On a similar line of thought one can use q.allSettled() if you want to wait for all promises to return.
Hope this helps.
---EOF OLD ANSWER---

First of all, if async functions used in PromiseA don't return promises, you need to promisify them. You can do that with Promise constructor, but it's much better to use libraries, such as bluebird with their promisify methods.
Let's imagine, that we have two functions getUserIdsAsync and getUserAsync. The first on returns a list of user ids, getUserAsync returns an user data by userId. And you need to get a list of users by their ids. The code of PromiseA could look so:
var promiseA = function() {
return getUserIdsAsync()
.then(userIds => {
let ops = users.map(uid => getUserAsync(uid));
return Promise.all(ops);
});
}

The following snippet shows a solution without using any external library like bluebird. It follows the code snippet in your question (which seems to be more complicate than needed).
You have to collect all api promisses in an array. Then you can call Promise.all() to get a Promise for the end of all api promisses. Then you can do some final stuff, like parsing the result of each promise and continue afterwards.
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
var apiEndpoint = function (name) {
return new Promise( (resolve, reject) => {
setTimeout(() => resolve('API ' + name + ' job done'), 1000);
});
}
var promiseA = function() {
return new Promise( (resolve, reject) => {
const promisses = [];
for (var i=1; i < getRandomInt(3,6); i++) {
// 1. Establish objects needed from one API endpoint
promisses.push(apiEndpoint('This is number ' + i));
}
Promise.all(promisses).then( results => {
// do final stuff
for (const s of results) {
// 2. Call API endpoint for each object and parse
console.log(s);
}
// continue ...
// 3. Only then continue to next promise
resolve('now it is finished');
}).catch( err => reject(err) );
});
}
var finalPromise = function() {
return new Promise( (resolve, reject) => {
console.log('finalPromise');
resolve();
});
}
promiseA()
.then( () => finalPromise())
.catch(err => console.log(err) );
Please hold in mind that this solution is not easy to read. Using external libraries or reducing promisses can improve readability. Maybe you should take a look to the async/await pattern to get a much more better (readable) solution.
Here is a solution with async/await:
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
const apiEndpoint = function (name) {
return new Promise( (resolve, reject) => {
setTimeout(() => resolve('API ' + name + ' job done'), 1000);
});
}
async function promiseParallel () {
const promisses = [];
for (let i = 1; i < getRandomInt(3,6); i++) {
promisses.push(apiEndpoint('This is number ' + i));
}
for (const p of promisses) {
const x = await p;
console.log(x);
}
return ('everything is done');
}
promiseParallel().then( result => {
console.log(result);
}).catch( err => console.log(err) );
If you want call the promisses sequentially you can replace with:
async function promiseSequ () {
for (let i = 1; i < getRandomInt(3,6); i++) {
const x = await apiEndpoint('This is number ' + i);
console.log(x);
}
return ('everything is done');
}

Promise.all() resolving, but it shouldn't (node.js)

I'm trying something like this right now in node.js:
var exec = require('child_process').exec
var write = require('fs-writefile-promise')
function run() {
var myArray = [];
var execs = [];
for (var i = 1; i <= 7; i++) {
(function(cntr) {
write('file-' + i + '.txt', someString)
.then(function (filename) {
execs.push(new Promise(function(resolve, reject) {
exec('cat ' + 'file-' + cntr + '.cnf', function(error, stdout, stderr) {
console.log(cntr + ' ' + stdout);
if (stdout.search(/\bsomeString\b/) > -1) {
myArray.push(cntr);
resolve();
}
else {
resolve();
}
})
}))
})
.catch(function (err) {
console.error(err);
});
})(i);
}
return Promise.all(execs).then(function() {
return new Promise(function(resolve) {
resolve(myArray);
})
})
}
run().then(function(result) {
console.log(result);
});
As you can see, I'm creating multiple Promises that run exec() and each one of them resolves when exec() finishes.
Then I'm waiting for every Promise to resolve in Promise.all(execs) to return myArray as a Promise. Yet, when I execute my run() function at the end, it is returning an empty array. I guess this has something to do with Promise.all() as it resolves even if some Promises in execs haven't resolved yet, but I'm not sure, thats why I really need some help here. Does anyone know where I'm making a mistake in code?
Thank you very much in advance!
#EDIT 1
var exec = require('child_process').exec
var write = require('fs-writefile-promise')
function run() {
var myArray = [];
var execs = [];
for (var i = 1; i <= 7; i++) {
(function(cntr) {
return new Promise(function(resolve, reject) {
fs.writeFile('file-' + i + '.txt', someString, (err) => {
if (err) {
reject();
}
else {
resolve();
}
});
})
.then(function (filename) {
execs.push(new Promise(function(resolve, reject) {
exec('cat ' + 'file-' + cntr + '.cnf', function(error, stdout, stderr) {
console.log(cntr + ' ' + stdout);
if (stdout.search(/\bsomeString\b/) > -1) {
myArray.push(cntr);
resolve();
}
else {
resolve();
}
})
}))
})
.catch(function (err) {
console.error(err);
});
})(i);
}
return Promise.all(execs).then(function() {
return new Promise(function(resolve) {
resolve(myArray);
})
})
}
run().then(function(result) {
console.log(result);
});

There are numerous issues with both your attempts. The issue with the first attempt is that you're populating the execs array after an asynchronous operation so it has nothing in it when you actually pass the array to Promise.all(), thus Promise.all() has nothing to wait for.
In addition, you're not just using promises that are already created, thus you end up making way more promises than needed.
In general, it is best to "promisify" your async operations once outside of your main logic and then have all your logic be promise driven rather than mixing and matching promises with plain callbacks. Here's a version that attempts to fix those issues:
var exec = require('child_process').exec
var write = require('fs-writefile-promise')
// make promisified version of exec
function execP(file, options) {
return new Promise(function(resolve, reject) {
exec(file, options, function(err, stdout, stderr) {
if (err) return resolve(err);
resolve({stdout: stdout, stderr: stderr});
});
});
}
function run() {
var promises = [];
for (var i = 1; i <= 7; i++) {
promises.push(write('file-' + i + '.txt', someString).then(function(filename) {
return execP(filename);
}));
}
return Promise.all(promises).then(function(results) {
// results is an array of {stdout: xxx, stderr: yyy} objects
// process those results into a new array of just indexes
var final = [];
results.forEach(function(data, index) {
if (data.stdout.search(/\bsomeString\b/) > -1) {
final.push(index);
}
});
return final;
});
}
run().then(function(results) {
// array of indexes that contained the desired search string
}, function(err) {
// process error here
});
Note: This runs all your exec operations in parallel which is what your original code did. If you want to run them sequentially, that could be done too, but would require some tweaks.

As write is asynchronous, the program at this stage is passing back to the main thread and that's going straight through to your promise.all and returning that before execs has been loaded.
I suggest you create a function that returns a promise of both a file save followed by exec.

Node.js synchronously loop or iterate over asynchronous statements

I want to do a for each loop but have it run synchronously. Each iteration of the loop will do an http.get call and that will return json for it to insert the values into a database. The problem is that the for loop runs asynchronously and that causes all of the http.gets to all run at once and my database doesn't end up inserting all of the data.I am using async-foreach to try to do what I want it to do, but I don't have to use it if I can do it the right way.
mCardImport = require('m_cardImport.js');
var http = require('http');
app.get('/path/hi', function(req, res) {
mCardImport.getList(function(sets) {
forEach(sets, function(item, index, arr) {
theUrl = 'http://' + sets.set_code + '.json';
http.get(theUrl, function(res) {
var jsonData = '';
res.on('data', function(chunk) {
jsonData += chunk;
});
res.on('end', function() {
var theResponse = JSON.parse(jsonData);
mCardImport.importResponse(theResponse.list, theResponse.code, function(theSet) {
console.log("SET: " + theSet);
});
});
});
});
});
});
and my model
exports.importResponse = function(cardList, setCode, callback) {
mysqlLib.getConnection(function(err, connection) {
forEach(cardList, function(item, index, arr) {
var theSql = "INSERT INTO table (name, code, multid, collector_set_num) VALUES "
+ "(?, ?, ?, ?) ON DUPLICATE KEY UPDATE id=id";
connection.query(theSql, [item.name, setCode, item.multid, item.number], function(err, results) {
if (err) {
console.log(err);
};
});
});
});
callback(setCode);
};

With recursion the code is pretty clean. Wait for the http response to come back then fire off next attempt. This will work in all versions of node.
var urls = ['http://stackoverflow.com/', 'http://security.stackexchange.com/', 'http://unix.stackexchange.com/'];
var processItems = function(x){
if( x < urls.length ) {
http.get(urls[x], function(res) {
// add some code here to process the response
processItems(x+1);
});
}
};
processItems(0);
A solution using promises would also work well, and is more terse. For example, if you have a version of get that returns a promise and Node v7.6+, you could write an async/await function like this example, which uses some new JS features.
const urls = ['http://stackoverflow.com/', 'http://security.stackexchange.com/', 'http://unix.stackexchange.com/'];
async function processItems(urls){
for(const url of urls) {
const response = await promisifiedHttpGet(url);
// add some code here to process the response.
}
};
processItems(urls);
Note: both of these examples skip over error handling, but you should probably have that in a production app.

To loop and synchronously chain asynchronous actions, the cleanest solution is probably to use a promise library (promises are being introduced in ES6, this is the way to go).
Using Bluebird, this could be
Var p = Promise.resolve();
forEach(sets, function(item, index, arr) {
p.then(new Promise(function(resolve, reject) {
http.get(theUrl, function(res) {
....
res.on('end', function() {
...
resolve();
}
}));
});
p.then(function(){
// all tasks launched in the loop are finished
});

I found out that I wasn't releasing my mysql connections after I was done with each call and this tied up the connections causing it to fail and appear to be an issue with synchronization.
After explicitly calling connection.release(); it caused my code to work 100% correctly even in an asynchronous fashion.
Thanks for those who posted to this question.

"use strict";
var Promise = require("bluebird");
var some = require('promise-sequence/lib/some');
var pinger = function(wht) {
return new Promise(function(resolve, reject) {
setTimeout(function () {
console.log('I`ll Be Waiting: ' + wht);
resolve(wht);
}, Math.random() * (2000 - 1500) + 1500);
});
}
var result = [];
for (var i = 0; i <= 12; i++) {
result.push(i);
}
some(result, pinger).then(function(result){
console.log(result);
});

Just wrap the loop in an async function. This example illustrates what I mean:
const oneSecond = async () =>
new Promise((res, _) => setTimeout(res, 1000));
This function completes after just 1 second:
const syncFun = () => {
for (let i = 0; i < 5; i++) {
oneSecond().then(() => console.log(`${i}`));
}
}
syncFun(); // Completes after 1 second ❌
This one works as expected, finishing after 5 seconds:
const asyncFun = async () => {
for (let i = 0; i < 5; i++) {
await oneSecond();
console.log(`${i}`);
}
}
asyncFun(); // Completes after 5 seconds ✅

var urls = ['http://stackoverflow.com/', 'http://security.stackexchange.com/', 'http://unix.stackexchange.com/'];
for (i = 0; i < urls.length; i++){
http.get(urls[i], function(res) {
// add some code here to process the response
});
}