We Keep Coding

Multiple awaits in function

I'm struggling a bit with JS promises.
I am using a library to pull data from Spotify that returns promises.
In my main function I can use an await to build an object from the response data and push it to an array (called nodes):
var nodes = [];
main();
async function main() {
var id = '0gusqTJKxtU1UTmNRMHZcv';
var artist = await getArtistFromSpotify(id).then(data => buildArtistObject(data));
nodes.push(artist);
When I debug here then all is good, nodes has my object.
However, when I introduce a 2nd await underneath to make another call:
nodes.forEach((node, i) => {
if (node.done == false) {
console.log(node.toString());
var related_artists = await getRelatedArtists(node.spotify_id);
I get the following error: SyntaxError: await is only valid in async function
I thought the first await statement would be resolved and the execution would continue until the next..?
Any help would be greatly appreciated.
EDIT
The other functions, if that helps, are just as follows:
function getArtistFromSpotify(id) {
let response = spotify
.request('https://api.spotify.com/v1/artists/' + id).then(function (data) {
return data;
})
.catch(function (err) {
console.error('Error occurred: ' + err);
return null;
});
return response;
}
function getRelatedArtists(id) {
let response = spotify
.request('https://api.spotify.com/v1/artists/' + id + '/related-artists').then(function (data) {
return data;
})
.catch(function (err) {
console.error('Error occurred: ' + err);
return null;
});
return response;
}
function buildArtistObject(data) {
var artist = {
node_id: nodes.length,
name: null,
genres: null,
popularity: null,
spotify_id: null,
done: false
}
artist.name = data.name;
artist.genres = data.genres;
artist.popularity = data.popularity > 0 ? data.popularity : 0;
artist.spotify_id = data.id;
return artist;
}

The code below has multiple problems.
var nodes = [];
main();
async function main() {
var id = '0gusqTJKxtU1UTmNRMHZcv';
var artist = await getArtistFromSpotify(id).then(data => buildArtistObject(data));
nodes.push(artist);
// ...
First of all, main mutates global scope nodes. Not only is this an antipattern even in synchronous code (functions should not rely on, or modify, global variable names; use parameters and return values instead), in asynchronous code, nodes will never be available for use anywhere but within main. See How do I return the response from an asynchronous call?.
Secondly, try to avoid combining then and await. It's confusing.
It's also a little odd that an array of nodes is used, yet only one artist is pushed onto it...
As for this code:
nodes.forEach((node, i) => {
if (node.done == false) {
console.log(node.toString());
var related_artists = await getRelatedArtists(node.spotify_id);
// ...
The error is self-explanatory. You must add async to the enclosing function if you want it to be asynchronous: nodes.forEach(async (node, i) => { // .... But that spawns a new promise chain per node, meaning future code that's dependent on the result won't be able to await all of the promises in the loop resolving. See Using async/await with a forEach loop. The likely solution is for..of or Promise.all.
While I'm not 100% sure what your final goal is, this is the general pattern I'd use:
async function main() {
const id = '0gusqTJKxtU1UTmNRMHZcv';
const data = await getArtistFromSpotify(id);
const artist = await buildArtistObject(data);
const nodes = [artist]; // odd but I assume you have more artists somewhere...
for (const node of nodes) {
if (!node.done) {
const relatedArtists = await getRelatedArtists(node.spotify_id);
}
}
/* or run all promises in parallel:
const allRelatedArtists = await Promise.all(
nodes.filter(e => !e.done).map(e => getRelatedArtists(e.spotify_id))
);
*/
// ...
}
main();
Since your code isn't runnable and some of the intent is unclear from the context, you'll likely need to adapt this a bit, so consider it pseudocode.

You have some misunderstandings of how to use promises -
let response = spotify
.request(url)
.then(function(data) { return data }) // this does nothing
.catch(function (err) { // don't swallow errors
console.error('Error occurred: ' + err);
return null;
})
return response
You'll be happy there's a more concise way to write your basic functions -
const getArtist = id =>
spotify
.request('https://api.spotify.com/v1/artists/' + id)
const getRelatedArtists = id =>
spotify
.request('https://api.spotify.com/v1/artists/' + id + '/related-artists')
Now in your main function, we can await as many things as needed. Let's first see how we would work with a single artist ID -
async function main(artistId) {
const artistData = await getArtist(artistId)
const relatedData = await getRelatedArtists(artistId)
return buildArtist(artistData, relatedData)
}
If you have many artist IDs -
async function main(artistIds) {
const result = []
for (const id of artistIds) {
const artistData = await getArtist(artistId)
const relatedData = await getRelatedArtists(artistId)
result.push(buildArtist(artistData, relatedData))
}
return result
}
Either way, the caller can handle errors as
main([693, 2525, 4598])
.then(console.log) // display result
.catch(console.error) // handle errors
Which is the same as -
main([693, 2525, 4598]).then(console.log, console.error)
The pattern above is typical but sub-optimal as the caller has to wait for all data to fetch before the complete result is returned. Perhaps you would like to display the information, one-by-one as they are fetched. This is possible with async generators -
async function* buildArtists(artistIds) {
for (const id of artistIds) {
const artistData = await getArtist(artistId)
const relatedData = await getRelatedArtists(artistId)
yield buildArtist(artistData, relatedData) // <- yield
}
}
async function main(artistIds) {
for await (const a of buildArtists(artistIds)) // <- for await
displayArtist(a)
}
main([693, 2525, 4598]).catch(console.error)

Database transactions happen parallely instead of sequentially in loop

I have an endpoint which loops through an array and updates the database as follows.
app.post('/create', Authenticate, async (req, res) => {
const {
products,
} = req.body;
const trxProvider = knex.transactionProvider();
const trx = await trxProvider();
try {
const formattedProduct = await Promise.all(products.map(async (product) => {
// Get Current value
const warehouseProducts = await trx('warehouse_products')
.select('available_qty as qty')
.where('product_code', product.product.code)
.first();
const finalQty = warehouseProducts.qty - product.orderQty;
// Update database
await trx('warehouse_products')
.update({ available_qty: finalQty })
.where('product_code', product.product.code);
}));
await trx('ordered_products')
.insert(formattedProduct);
trx.commit();
console.log('Transaction successful');
return send(res, 201, { success: true });
} catch (err) {
console.log(err);
trx.rollback();
const errors = {};
errors.message = 'something went wrong';
return send(res, 500, errors);
}
});
The issue arises when i try to update the same row of the warehouse_products table within the loop.
In the loop initially the qty value is taken from the warehouse_products table for a particular product then an arithmetic operation is done and the qty value is updated.
Ideally if both iterations access the same row, the second iteration's initial qty value should be what the first iteration updated. However the issue is that the second iteration too starts with the initial value of the first iteration. Its almost as if both iterations are happening parallel to each other instead of occurring sequentially.

Since you are using Promise.all it is supposed to happen in paralle. For sequential processing change this code
await Promise.all(products.map(async (product) => {
// logic here
});
to
for(const product of products) {
// logic here
}

Have a look at the definition for Promise.all()
It is typically used after having started multiple asynchronous tasks to run concurrently and having created promises for their results, so that one can wait for all the tasks being finished.

if you don't want to use an external library like Bluebird or Async
you can go with simple for a loop as following
let delay = (t) => new Promise((resolve) => {
setTimeout(() => {
return resolve(new Date().getMilliseconds())
}, t*1000);
});
let array = [1,1,1,1];
//using simple for loop
async function test() {
let out = [];
for (let i = 0; i < 4; i++) {
const res = await delay(array[i]);
out.push(res);
}
return out;
}
test().then(res => {
console.log(res)
})

Fetching synchroneously from redis in Node through async/await

I am learning Javascript and Vue from Java from few days and not able to solve a problem with my Node, Express app through async/await. The below code is receiving a list of Stock symbols from request and then checking through a loop if details against any of the symbols is already cached in redis.
var controllers = {
getCurrentPrice: function(req, res) {
var symbolsArray = req.body.symbols;
var results = [];
var tmpArray = [];
_(symbolsArray).each( async function(symbol, iPos) {
client.hget("realtime", symbol, function(err, reply) {
if(reply) {
await results.push(reply);
} else {
await tmpArray.push(symbol);
}
console.log("reply", reply);
});
});
console.log("Results so far ", results);
if( !tmpArray || tmpArray.length == 0 ) { //will be fetching these now }
}
}
Getting output in the inner console statement but not for the outer one. I have tried looking at few solutions through net like through redis-co to promisify the redis calls but could not exactly solve it.

There are several things wrong here:
Doing await on the result of a .push() doesn't do anything useful. You use await on a promise.
Your .each() loop doesn't wait for each asycnhronous operation to finish, therefore you have no way of knowing when all the async operations are done
I'd suggest using a regular for loop where async/await will pause for the loop:
const util = require('util');
client.hgetP = util.promisify(client.hget);
var controllers = {
getCurrentPrice: async function(req, res) {
var symbolsArray = req.body.symbols;
var results = [];
var tmpArray = [];
for (let symbol of symbolsArray) {
let reply = await client.hgetP("realtime", symbol);
if (reply) {
results.push(reply);
} else {
tempArray.push(symbol);
}
}
// do any further processing of tempArray here
console.log(results);
return results; // this will be the resolved value of the returned promise
}
}
Sample usage:
obj.getCurrentPrice.then(results => {
console(results);
}).catch(err => {
console.log(err);
});

Async Javascript: Trying to make AJAX API calls multiple times at interval?

So I have a form, user searches a query.
Trying to do:
I make API calls consistently. The first query is entered by user, then the program makes new queries based on Entity Analysis, and makes API calls itself - over and over.
Issue: The results are the same, which means the code is making the
API calls based on the initial query over and over, instead of making
new queries from API results.
Result in Console:
queries length is zero...
Iterating... undefined
On Queue Searches #1: []
query length is zero
POST / 200 15.747 ms - -
etc. etc. etc.
code
Pushing the query entered by user into array, sending it to onQueueSearch to make API calls at a set interval.
// Global Queries
const queries = new Array();
let counter = 0; // Just to keep track
router.post('/', async (req, res) => {
// Assign first query
queries.push(req.body.searchTerm);
// Invoke the async iterator
const results = await queryIterator(queries);
res.send(results);
});
This function might be the problem?
For each query in the array, do pass it to sentimentAnalyze which calls API's, this function is suppose to make calls to API consistently at set interval. (Note: New queries are added to queries array in the API calls.)
async function onQueueSearch() {
if(queries.length === 0 ) {
console.log("queries length is zero...");
return; // return something if empty
}
// This will extract the next in queue and remove it from queries
return await sentimentAnalyze(queries.pop);
}
// Define functions for Async Iterator -----
async function* queryGenerator() {
while (true) {
yield await onQueueSearch(queries.pop());
}
}
async function queryIterator(queries) {
for await (const result of queryGenerator()) {
console.log('Iterating... ', result);
console.log(`On Queue Searches #${++counter}: `, queries);
if (!queries.length) {
console.log("query length is zero");
return result;
}
}
}
API calls function - Return a set of twitter results, then google api results - this async function try's to wait for returned promises/result
from twitter google api and returns
async function sentimentAnalyze(searchValue) {
try {
const tweetDocumentObject = await searchTwitter(searchValue);
const sentimentResults = await googleSentimentAnalyze(tweetDocumentObject);
const entitiesResults = await googleEntityAnalyze(tweetDocumentObject);
return {sentimentResults, entitiesResults};
} catch(err) {
console.log('sentimentAnalyze Error: ', err);
}
}
This function is where new queries are ADDED from entity analysis.
function googleEntityAnalyze(document) {
return new Promise((resolve, reject) => {
// Detects entities in the document
googleClient
.analyzeEntities({document: document})
.then(results => {
const entities = results[0].entities;
queries.unshift(req.body.searchTerm);
entities.forEach(entity => {
queries.unshift(entity.name);
// console.log("Entitiy name: " +entity.name);
// console.log(` - Type: ${entity.type} `);
resolve({ entity });
});
})
.catch(err => {
console.error('ENTITY ERROR:', err);
});
});
}

Ok, there's a couple of issues with your code, but easy to fix if you're carefull, I'll guide you through them.
First of:
You're pushing to queue always, this is the root cause of you getting the same data in queue:
queries.push(req.body.searchTerm); // searchTerm being '1'
... // Do more logic until googleEntityAnalyze
entities.forEach(entity => {
queries.push(entity.name);
resolve({ entity });
});
// Assume entities as [2, 3]
Now you have your queue as [1,2,3], being 1 as the first argument.
Then you redo the query logic from:
async function onQueueSearch() {
for(i = 0; i < queries.length; i++) {
return await sentimentAnalyze(queries[i]);
// queries[i].splice(i, 1);
}
if(queries.length === 0 ) {
console.log("queries length is zero...");
}
}
The return on the for loop will break the loop and return the first execution, which will be queries[0] === 1, your first argument.
So, to resolve this, keep the array in first-in-first-out, using:
queries.unshift(req.body.searchTerm);
entities.forEach(entity => {
queries.unshift(entity.name);
resolve({ entity });
});
This will keep your queries in order as they arrive [3,2,1], instead of [1,2,3], now you can extract your query using queries.pop() instead.
queries.unshift(4); // where queries was [3,2,1], will be [4,3,2,1]
queries.pop(); // 1
Change the sentiment analyze to:
async function onQueueSearch(){
if(queries.length === 0 ) {
console.log("queries length is zero...");
return; // Return something if empty!
}
return await sentimentAnalyze(queries.pop());
// This will extract the next in queue and remove it from queries
}
Secondly, your re-call logic
You're using an iterval to continuously recall the query, the problem with this:
setInterval(async function() {
results = await onQueueSearch();
}, 3000);
Is that you need to "estimate" the time the query will take to finish before re-executing. With the query unshift and pop above you'll make this work, however, you need a more elegant solution.
With NodeJS version < 10+:
recursion will be useful, here's a small sample of what you're doing in a simple manner:
const queries = [1];
let counter = 0; // Just to keep track
async function reRunQueries(){
counter++;
console.log(`queries #${counter}: `, queries);
results = await onQueueSearch();
console.log('Results: ', results);
if(!!queries.length){
return await reRunQueries();
}else{
return results;
}
}
async function onQueueSearch(){
return await longQuery(queries.pop());
}
async function longQuery(param){
if(param === 6){
// Use this as stop signal!
return param;
}else{
queries.unshift(++param);
return param;
}
}
const RES = reRunQueries();
RES.then(result => {
console.log('RES: ', result);
})
It's important to know your stop signal for recursion, otherwise it will never end.
With NodeJS > 10+
Use a Iterable Generator:
const queries = [];
let counter = 0; // Just to keep track
// EMULATE EXPRESS JS call ========================
const req = { body: { searchTerm: 1 } };
const res = {send: (val) => console.log('RECEIVED: ', val)};
const router = {
post: (route, callback) => callback(req, res)
}
router.post('/', async (req, res) => {
// Assign first query
queries.push(req.body.searchTerm);
// Invoke the async iterator
const results = await queryIterator(queries);
res.send(results);
});
// The Above is for nodeJS direct testing only,
// > you can plug it into your own router.post declaration "as is"
// -----------------------------------------
// Define functions for Async Iterator -----
async function* queryGenerator() {
while (true) {
yield await onQueueSearch(queries.pop());
}
}
async function queryIterator(queries) {
for await (const result of queryGenerator()) {
console.log('Iterating... ', result);
console.log(`On Queue Searches #${++counter}: `, queries);
if (!queries.length) {
return result;
}
}
}
// ------------------------------------------------------------
// Emulate your current async requests using queries array ----
async function onQueueSearch(param) {
return await longQuery(param);
}
async function longQuery(param) {
if (param === 6) {
// Use this as stop signal!
return Promise.resolve(param);
} else {
queries.unshift(++param);
return Promise.resolve(param);
}
}
Returns:
/*
Iterating... 2
On Queue Searches #1: [ 2 ]
Iterating... 3
On Queue Searches #2: [ 3 ]
Iterating... 4
On Queue Searches #3: [ 4 ]
Iterating... 5
On Queue Searches #4: [ 5 ]
Iterating... 6
On Queue Searches #5: [ 6 ]
Iterating... 6
On Queue Searches #6: []
RECEIVED: 6
*/

how to send requests inside loop to backend and use await and make the loop executes sequentially

I have an array list of sizes which i want to create in the database so i loop on the array and send a request for each item in the array using await to get the response , the problem is the async part takes too much time the loop continues execution and when it ends the method returns size_group with an empty array, also the array get filled after the return but the returned data is empty
add(body) {
let sizeGroup = null;
let sizes = null;
return god.post('/size_groups', body)
.then((response) => {
sizeGroup = response.data.size_group;
for (var i = 0; i < body.sizes.length; i++) {
let size = {
'name': body.sizes[i].label,
'size_group': sizeGroup._id
};
(async () => {
let response2 = await god.post('/sizes', size);
if (response2.data.status === true)
sizeGroup.push(response2.data.size);
})();
}
return sizeGroup;
});
}

This is one of few problems that async..await solves. Since async functions are supported in current environment, there's no reason to combine them with promise chains, the entire function should be converted to async:
async add(body) {
const response = await god.post('/size_groups', body)
let sizeGroup = response.data.size_group;
for (var i = 0; i < body.sizes.length; i++) {
let size = {
'name': body.sizes[i].label,
'size_group': sizeGroup._id
};
let response2 = await god.post('/sizes', size);
if (response2.data.status === true)
sizeGroup.push(response2.data.size);
}
return sizeGroup;
}