Javascript Async Await Code Execution Order - javascript

I'm stumped why loopCafes runs before buildCafeList in this example. I need to build an array and pass it along for additional modification, but the execution order is reversed.
The output returns:
loopCafes: 0
getCafeList 8
getCafeList 16
const fs = require("fs");
const JSON_FOLDER = "./reports/";
let cafes = [];
const buildCafeList = async () => {
fs.readdir(JSON_FOLDER, function(err, list) {
if (err) throw err;
list.forEach(function(file) {
let thisJSON = JSON_FOLDER + file;
fs.readFile(thisJSON, function(err2, data) {
if (err2) throw err2;
let thisJSON = JSON.parse(data);
for (let i = 0; i < thisJSON.businesses.length; i++) {
let thisCafe = thisJSON.businesses[i];
cafes.push({
alias: thisCafe.alias,
name: thisCafe.name,
url: thisCafe.url,
address1: thisCafe.location.address1,
city: thisCafe.location.city
});
}
console.log("getCafeList", cafes.length); // 8, 16
});
});
});
};
const loopCafes = async () => {
console.log("loopCafes:", cafes.length); // 0
for (let k = 0; k < cafes.length; k++) {
console.log(k, cafes[k].name);
}
};
const buildReport = async () => {
const getCafeList = await buildCafeList();
const showCafeList = await loopCafes();
};
buildReport();

fs.readdir is an async function that accepts a callback.
Consequently, buildCafeList returns immediately (since you marked it async and didn't include an explicit return statement it returns a promise that resolves immediately).
Later on, the callback for fs.readdir is triggered and the value you are logging is reported.
You need to wrap fx.readdir in a promise and resolve it when you get the data you want in the callback. buildCafeList needs to return that promise.

Related

scan duplicates item with DynamoDB

I'd like to scan items and avoid to use duplicates code.
so, I am trying to use for-of asynchronously for it.
async function checkDupl(){
const arr = new Array(10).fill(0);
let code = '';
for(const i of arr){
//generate RANDOM CODE
//for example, it would be '000001' to '000010'
code = (Math.floor(Math.random() * 10) + 1).toString().padStart(6,"0");
const params = { ... }; // it has filterExpression the code I generated randomly
await DYNAMO_DB.scan(params, (err, res) => {
if(res.Items.length === 0) {
/* no duplicate! */
return code;
}
});
}
return code;
}
console.log(checkDupl());
// it always return '';
What I have missed or misunderstood?
await just waits a Promise (or thenable object), but you are using await with a "void" function (You use DYNAMO_DB.scan as a callback styte function).
My suggestion, Use DYNAMO_DB.scan with Promise style (The way)
async function checkDupl() {
const arr = new Array(10).fill(0);
let code = '';
for (const i of arr) {
//generate RANDOM CODE
//for example, it would be '000001' to '000010'
code = (Math.floor(Math.random() * 10) + 1).toString().padStart(6, "0");
const params = { ... }; // it has filterExpression the code I generated randomly
const res = await DYNAMO_DB.scan(params).promise(); // convert to promise
if (res.Items.length === 0) {
/* no duplicate! */
return code;
}
return code;
}
}
(async () => {
console.log(await checkDupl());
})();

parallel async Mongoose queries || eval(return) SyntaxError : Illegal return statement

I'm trying to make a lot of requests to Mongo with Mongoose. I'd like them to be parallel and async. I don't know how many I might have to query so I created this helper function. It works fine, although I'm not sure about my eval usage. The problem is the eval(return${returnStr}) giving me the SyntaxError.
const batchRetrieve = async (query, models) => {
models.forEach((model, i) => {
eval(`task${i} = ${model}.findOne(query)`);
});
let str = [];
for (let i = 0; i < models.length; i++) {
str.push(`res${i}: await task${i}`);
}
const joinedStr = Array.prototype.join.call(str, ', '); //output is 'res0: await task0, res1: await task1'
// return { res0: await task0, res1: await task1 }; works fine
eval(`return { ${joinedStr} };`);
};
Rule of thumb: If you're using eval, you're doing it wrong. Try this:
const batchRetrieve = async (query, models) => {
const promises = [];
models.forEach(model => {
promises.push(model.findOne(query));
});
const results = [];
for (let i = 0; i < models.length; i++) {
results.push(await promises[i]);
}
return results;
};
replacing the eval(return { ${joinedStr} };); with
return eval(`(async () => {
return { ${joinedStr} }
})();`);
resolves my issue.

Nested for loop combined with fetching data from an API in Nodejs?

My scenario is as follows:
There is an API I would like to fetch from. The API returns a json that has an array named "assets". This array size will always be 20.
Now, I call the endpoint like so:
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
where if offset is 0 it will return the array of the 0 - 20 assets, if the offset is 20 it will return 20 to 40 and so on.
I want to check 1000 items which means I would like to call this fetch 1000/20 = 50 times.
Whenever I call the fetch I would like loop over these 20 items and insert them into my DB.
The problem is that I can't do something like this:
let offset=0;
for(let i = 0; i < 50; i++ {
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
for(let j = 0; j < 20; j++){
// Insert into DB
}
offset+=20;
}
Due to JS asynchronous nature. Whenever I try to do this it will call the fetch with the value 0 for offset a couple of times, it won't wait until the nested for loop finishes and than call it for 20 and later 40 and so on...
What is the correct way to achive this behavior?
I've nicked the following from one of my nodeJS repos as it employs async code for solving a very similar problem:
// Throttling is important because you don't want to
// overload the API
const createThrottle = require('async-throttle');
const throttle = createThrottle(2);
// First push all the links into an array using
// the offset
const links = [];
for (let offset = 0; offset < 100; offset += 20) {
links.push(`https://api.example.io/api/v1/assets/?offset=${offset}`);
}
// Next create an array of promises by `map`ing
// over the links using `fetch`.
// Notice I've used throttle here to slow down the hits on
// the API
const promises = links.map(link => throttle(async => () {
const res = await fetch(link);
return await res.json();
}));
// Once all the promises are complete, iterate over their datasets
Promise.all(promises).then(datasets => {
// iterate over datasets
});
Why not use Promise.all?
const promises = [];
for (let i = 0; i < 10; i++) {
promises.push(
fetch(`https://api.example.io/api/v1/assets/?offset=${i}`)
);
}
Promise.all(promises).then( _results => {
// All fetch calls have completed at this point. _results will be in array of your fetch results.
console.log(results);
// Do db insert here
});
One thing you could do is to make a function that is a promise. Within that function, have it execute the fetch and then execute the db insert all in one function (using .then). If you did it this way, the single Promise.all call would handle everything. If you don't, you'll have to loop through the promises again and insert those values into the db. It might look something like this:
const promises = [];
for (let i = 0; i < 10; i++) {
promises.push(fetchAndInsert(i));
}
Promise.all(promises).then( _results => {
console.log(results);
});
function fetchAndInsert(offset) {
return new Promise( (resolve, reject) => {
fetch(`https://api.example.io/api/v1/assets/?offset=${i}`).then (_result => {
// db.insert().then( _result => {
//resolve();
//})
})
})
}
You could use async and await. This should work:
async function fetchExample() {
for (let i = 0; i < 50; i++) {
let fetchedData = await fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
);
for(data of fetchedData) {
// Insert into DB
}
offset+=20;
}
}
Instead of for..loop, you could use recursion or Promises.
Recursion::
let insertToDB = function (records, cb) {
if (!records.length) return cb();
let record = records.shift();//assuming records is an array
// Insert record into DB1
insertToDB(records, cb);
};
let loop = function (count, offset, cb) {
if (!count) return cb();
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
insertToDB(recordsFromFetch, function () {
offset += 20;
--count;
loop(count, offset, cb)
})
};
loop(50, 0, function () {
console.log("All Done");
})
Promise:: I have not run it, so might be some syntactical error, but you get the idea
let insertToDB = function (record) {
return new Promise(function (resolve, reject) {
// Insert record into DB then call resolve
resolve();
})
};
let fetchPhase = function (offset) {
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
let dbAll = [];
for (let j = 0; j < 20; j++) {
// assuming you get records from fetch , pass record to be added to db in the parameter
dbAll.push(insertToDB(records[j]))
}
return Promise.all(dbAll)
};
let all = [];
let offset = 0;
for (let i = 0; i < 50; i++) {
all.push(fetchPhase(i));
}
Promise.all(all)
.then(function () {
console.log("ALL DONe");
})
Promise.all(Array(50).fill(null).map((v,i) => {
const url = `https://api.example.io/api/v1/assets/?offset=${i*20}`;
return fetch(url).then(results => {
for (let result of results) {
// insert into DB
}
);
}).then(() => console.log("done"));

How to use fetch within a for-loop, wait for results and then console.log it

i have this problem: i want to make multiple fetch calls within a for-loop. The number of calls depend on the user input (in my example i have three). How can i make it loop through all the fetch requests and then console.log the number off calls?
function getPosts(){
let url = ["https://www.freecodecamp.org", "https://www.test.de/, http://www.test2.com"];
let array = new Array;
for (let i = 0; i < url.length; i++) {
console.log(url[i]);
fetch(url[i])
.then(res => {return res.text(); })
.then(res => {
let reg = /\<meta name="description" content\=\"(.+?)\"/;
res = res.match(reg);
array.push(res);
console.log(res);
}
)
.catch(status, err => {return console.log(status, err);})
}
console.log (array.length);
}
It console.logs 0 instead of 3, cause it doesn't wait for all the promises to be resolved. How can i make it to console.log 3?
If you know a solution, please help me out.
You can't call console.log(array.length) until after the promises are all done. So why not something like this?
let url = ["https://www.freecodecamp.org", "https://www.test.de/, http://www.test2.com"];
let array = new Array;
var fetches = [];
for (let i = 0; i < url.length; i++) {
console.log(url[i]);
fetches.push(
fetch(url[i])
.then(res => {return res.text(); })
.then(res => {
let reg = /\<meta name="description" content\=\"(.+?)\"/;
res = res.match(reg);
array.push(res);
console.log(res);
}
)
.catch(status, err => {return console.log(status, err);})
);
}
Promise.all(fetches).then(function() {
console.log (array.length);
});
}
Promise.all waits for all the fetches to finish, THEN it'll print the #.
You can use async/await with try/catch:
async function getPosts(){
let array = [];
let url = ["https://www.freecodecamp.org", "https://www.test.de/", "http://www.test2.com"];
let reg = /\<meta name="description" content\=\"(.+?)\"/;
for (let i = 0; i < url.length; i++) {
console.log('fetching',url[i]);
try {
let p1 = await fetch(url[i]);
let p2 = await p1.text();
let res = p2.match(reg);
array.push(res);
console.log('adding',res);
}
catch (e) {
console.error(e.message);
}
};
console.log ('length',array.length);
};
getPosts().then(()=>{console.log('done')});
You can try chaining your promises, Try the following:
function getPosts(){
let url = ["https://www.freecodecamp.org", "https://www.test.de/, http://www.test2.com"];
let array = new Array;
var promise = Promise.resolve();
for (let i = 0; i < url.length; i++) {
console.log(url[i]);
promise = promise.then(fetch(url[i]))
.then(res => {return res.text(); })
.then(res => {
let reg = /\<meta name="description" content\=\"(.+?)\"/;
res = res.match(reg);
array.push(res);
console.log(res);
}
)
.catch(status, err => {return console.log(status, err);})
}
promise.then(function(response){
console.log (array.length);
});
}
You can use async await ( async: function, await: operator ). Await operator simply wait for the promise to be resolved. First promise will be resolved then it will move to another one. Also, if it finds error in any fetch, it will catch the error right away.
You should use promise, A Promise is a proxy for a value not necessarily known when the promise is created. It allows you to associate handlers with an asynchronous action's eventual success value or failure reason. This lets asynchronous methods return values like synchronous methods: instead of immediately returning the final value, the asynchronous method returns a promise to supply the value at some point in the future.
const fetch = require('node-fetch')
let url = ["https://www.freecodecamp.org", "https://www.test.de/, http://www.test2.com"];
let array = new Array;
function get(url) {
return new Promise((resolve, reject) => {
fetch(url)
.then(res => { return res.text(); })
.then(res => {
let reg = /\<meta name="description" content\=\"(.+?)\"/;
res = res.match(reg);
resolve(res)
//console.log(res);
}
)
.catch(err => { reject(err) })
});
}
async function result() {
for (let i = 0; i < url.length; i++) {
const value = await get(url[i]);
array.push(value)
}
console.log(array.length)
}
result()
==> array.length = 2,

Best way to wait for .forEach() to complete

Sometimes I need to wait for a .forEach() method to finish, mostly on 'loader' functions. This is the way I do that:
$q.when(array.forEach(function(item){
//iterate on something
})).then(function(){
//continue with processing
});
I can't help but feel that this isn't the best way to wait for a .forEach() to finish. What is the best way to do this?
If there is no asynchronous code inside the forEach, forEach is not asynchronous, for example in this code:
array.forEach(function(item){
//iterate on something
});
alert("Foreach DONE !");
you will see the alert after forEach finished.
Otherwise (You have something asynchronous inside), you can wrap the forEach loop in a Promise:
var bar = new Promise((resolve, reject) => {
foo.forEach((value, index, array) => {
console.log(value);
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log('All done!');
});
Credit: #rolando-benjamin-vaz-ferreira
The quickest way to make this work using ES6 would be just to use a for..of loop.
const myAsyncLoopFunction = async (array) => {
const allAsyncResults = []
for (const item of array) {
const asyncResult = await asyncFunction(item)
allAsyncResults.push(asyncResult)
}
return allAsyncResults
}
Or you could loop over all these async requests in parallel using Promise.all() like this:
const myAsyncLoopFunction = async (array) => {
const promises = array.map(asyncFunction)
await Promise.all(promises)
console.log(`All async tasks complete!`)
}
var foo = [1,2,3,4,5,6,7,8,9,10];
If you're actually doing async stuff inside the loop, you can wrap it in a promise ...
var bar = new Promise((resolve, reject) => {
foo.forEach((value, index, array) => {
console.log(value);
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log('All done!');
});
If you have an async task inside a loop and you want to wait. you can use for await
for await (const i of images) {
let img = await uploadDoc(i);
};
let x = 10; //this executes after
Use for of instead of forEach. Like this:
for (const item of array) {
//do something
}
console.log("finished");
"finished" will be logged after finishing the loop.
forEach() doesn't return anything, so a better practice would be map() + Promise.all()
var arr = [1, 2, 3, 4, 5, 6]
var doublify = (ele) => {
return new Promise((res, rej) => {
setTimeout(() => {
res(ele * 2)
}, Math.random() ); // Math.random returns a random number from 0~1
})
}
var promises = arr.map(async (ele) => {
// do some operation on ele
// ex: var result = await some_async_function_that_return_a_promise(ele)
// In the below I use doublify() to be such an async function
var result = await doublify(ele)
return new Promise((res, rej) => {res(result)})
})
Promise.all(promises)
.then((results) => {
// do what you want on the results
console.log(results)
})
A universal solution for making sure that all forEach() elements finished execution.
const testArray = [1,2,3,4]
let count = 0
await new Promise( (resolve) => {
testArray.forEach( (num) => {
try {
//some real logic
num = num * 2
} catch (e) {
// error handling
console.log(e)
} fanally {
// most important is here
count += 1
if (count == testArray.length) {
resolve()
}
}
})
})
The idea is same with the answer using index to count. But in real case, if error happened, the index way cannot count correctly. So the solution is more robust.
Thx
const array = [1, 2, 3];
const results = [];
let done = 0;
const asyncFunction = (item, callback) =>
setTimeout(() => callback(item * 10), 100 - item * 10);
new Promise((resolve, reject) => {
array.forEach((item) => {
asyncFunction(item, (result) => {
results.push(result);
done++;
if (done === array.length) resolve();
});
});
}).then(() => {
console.log(results); // [30, 20, 10]
});
// or
// promise = new Promise(...);
// ...
// promise.then(...);
The order of results in the "results" array can be different than the order of items in the original array, depending on the time when the asyncFunction() finishes for each of the items.
Alter and check a counter at the end of every possible unique branch of code, including callbacks. Example:
const fs = require('fs');
/**
* #description Delete files older than 1 day
* #param {String} directory - The directory to purge
* #return {Promise}
*/
async function purgeFiles(directory) {
const maxAge = 24*3600000;
const now = Date.now();
const cutoff = now-maxAge;
let filesPurged = 0;
let filesProcessed = 0;
let purgedSize = 0;
await new Promise( (resolve, reject) => {
fs.readdir(directory, (err, files) => {
if (err) {
return reject(err);
}
if (!files.length) {
return resolve();
}
files.forEach( file => {
const path = `${directory}/${file}`;
fs.stat(path, (err, stats)=> {
if (err) {
console.log(err);
if (++filesProcessed === files.length) resolve();
}
else if (stats.isFile() && stats.birthtimeMs < cutoff) {
const ageSeconds = parseInt((now-stats.birthtimeMs)/1000);
fs.unlink(path, error => {
if (error) {
console.log(`Deleting file failed: ${path} ${error}`);
}
else {
++filesPurged;
purgedSize += stats.size;
console.log(`Deleted file with age ${ageSeconds} seconds: ${path}`);
}
if (++filesProcessed === files.length) resolve();
});
}
else if (++filesProcessed === files.length) resolve();
});
});
});
});
console.log(JSON.stringify({
directory,
filesProcessed,
filesPurged,
purgedSize,
}));
}
// !!DANGER!! Change this line! (intentional syntax error in ,')
const directory = ,'/tmp'; // !!DANGER!! Changeme
purgeFiles(directory).catch(error=>console.log(error));
I'm not sure of the efficiency of this version compared to others, but I used this recently when I had an asynchronous function inside of my forEach(). It does not use promises, mapping, or for-of loops:
// n'th triangular number recursion (aka factorial addition)
function triangularNumber(n) {
if (n <= 1) {
return n
} else {
return n + triangularNumber(n-1)
}
}
// Example function that waits for each forEach() iteraction to complete
function testFunction() {
// Example array with values 0 to USER_INPUT
var USER_INPUT = 100;
var EXAMPLE_ARRAY = Array.apply(null, {length: USER_INPUT}).map(Number.call, Number) // [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, n_final... ] where n_final = USER_INPUT-1
// Actual function used with whatever actual array you have
var arrayLength = EXAMPLE_ARRAY.length
var countMax = triangularNumber(arrayLength);
var counter = 0;
EXAMPLE_ARRAY.forEach(function(entry, index) {
console.log(index+1); // show index for example (which can sometimes return asynchrounous results)
counter += 1;
if (triangularNumber(counter) == countMax) {
// function called after forEach() is complete here
completionFunction();
} else {
// example just to print counting values when max not reached
// else would typically be excluded
console.log("Counter index: "+counter);
console.log("Count value: "+triangularNumber(counter));
console.log("Count max: "+countMax);
}
});
}
testFunction();
function completionFunction() {
console.log("COUNT MAX REACHED");
}
I had to deal with the same problem (forEach using multiple promises inside) and none of the solutions presented at the current date were helpful for me. So I implemented a check array, were each promise updates its complete status. We have a general promise that wraps the process. We only resolve the general promise when each promise completed. Snippet code:
function WaitForEachToResolve(fields){
var checked_fields = new Array(fields.length).fill(0);
const reducer = (accumulator, currentValue) => accumulator + currentValue;
return new Promise((resolve, reject) => {
Object.keys(fields).forEach((key, index, array) => {
SomeAsyncFunc(key)
.then((result) => {
// some result post process
checked_fields[index] = 1;
if (checked_fields.reduce(reducer) === checked_fields.length)
resolve();
})
.catch((err) => {
reject(err);
});
}
)}
}
I like to use async-await instead of .then() syntax so for asynchronous processing of data, modified the answer of #Ronaldo this way -
let finalData = [];
var bar = new Promise(resolve => {
foo.forEach((value, index) => {
const dataToGet = await abcService.getXyzData(value);
finalData[index].someKey = dataToGet.thatOtherKey;
// any other processing here
if (finalData[dataToGet.length - 1].someKey) resolve();
});
});
await Promise.all([bar]);
console.log(`finalData: ${finalData}`);
NOTE: I've modified the if condition where it resolves the promise to meet my conditions. You can do the same in your case.
You can use this, because we are using async/await inside the forEach loop. You can use your own logic inside the loop.
let bar = new Promise((resolve, reject) => {
snapshot.forEach(async (doc) => {
"""Write your own custom logic and can use async/await
"""
const result = await something()
resolve(result);
});
});
let test = []
test.push(bar)
let concepts = await Promise.all(test);
console.log(concepts);
For simple compare code i like use for statement.
doit();
function doit() {
for (var i = 0; i < $('span').length; i++) {
console.log(i,$('span').eq(i).text() );
if ( $('span').eq(i).text() == "Share a link to this question" ) { // span number 59
return;
}
}
alert('never execute');
}
If there are async (observable) method calls inside the for loop, you could use the following method:
await players.reduce(async (a, player) => {
// Wait for the previous item to finish processing
await a;
// Process this item
await givePrizeToPlayer(player);
}, Promise.resolve());
Check here: https://gist.github.com/joeytwiddle/37d2085425c049629b80956d3c618971
I've been using this and it works best .forEach()
//count
var expecting = myArray.length;
myArray.forEach(function(item){
//do logic here
var item = item
//when iteration done
if (--expecting === 0) {
console.log('all done!');
}
})

Categories