We Keep Coding

How to correctly use 'async, await and promises' in nodejs, while allocating values to a variable returned from a time-consuming function?

Problem Statement:
Our aim is to allocate values in the array ytQueryAppJs, which are returned from a time consuming function httpsYtGetFunc().
The values in ytQueryAppJs needs to be used many times in further part of the code, hence it needs to be done 'filled', before the code proceeds further.
There are many other arrays like ytQueryAppJs, namely one of them is ytCoverAppJs, that needs to be allocated the value, the same way as ytQueryAppJs.
The values in ytCoverAppJs further require the use of values from ytQueryAppJs. So a solution with clean code would be highly appreciated.
(I am an absolute beginner. I have never used async, await or promises and I'm unaware of the correct way to use it. Please guide.)
Flow (to focus on):
The user submits a queryValue in index.html.
An array ytQueryAppJs is logged in console, based on the query.
Expected Log in Console (similar to):
Current Log in Console:
Flow (originally required by the project):
User submits query in index.html.
The values of arrays, ytQueryAppJs, ytCoverAppJs, ytCoverUniqueAppJs, ytLiveAppJs, ytLiveUniqueAppJs gets logged in the console, based on the query.
Code to focus on, from 'app.js':
// https://stackoverflow.com/a/14930567/14597561
function compareAndRemove(removeFromThis, compareToThis) {
return (removeFromThis = removeFromThis.filter(val => !compareToThis.includes(val)));
}
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
let promise = new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = []
response.on('data', (d) => {
chunks.push(d)
})
// Parsing the chunks
response.on('end', () => {
resultOfYtGetFunc = JSON.parse((Buffer.concat(chunks).toString()))
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < (resultOfYtGetFunc.items).length; i++) {
extractedResultOfYtGetFunc[i] = resultOfYtGetFunc.items[i].id.videoId;
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
})
})
})
let result = await promise;
return result;
}
app.post("/", function(req, res) {
// Accessing the queryValue, user submitted in index.html. We're using body-parser package here.
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Complete app.post method from app.js:
(For better understanding of the problem.)
app.post("/", function(req, res) {
// Accessing the queryValue user submitted in index.html.
query = req.body.queryValue;
// Fetcing top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
// Fetching 'cover' songs related to user's query and putting them in the array.
if (query.includes("cover") == true) {
ytCoverAppJs = httpsYtGetFunc(query);
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else {
ytCoverAppJs = httpsYtGetFunc(query + " cover");
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
}
// Fetching 'live performances' related to user's query and putting them in the array.
if (query.includes("live") == true) {
ytLiveAppJs = httpsYtGetFunc(query);
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else {
ytLiveAppJs = httpsYtGetFunc(query + " live");
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
}
// Emptying all the arrays.
ytQueryAppJs.length = 0;
ytCoverAppJs.length = 0;
ytCoverUniqueAppJs.length = 0;
ytLiveAppJs.length = 0;
ytLiveUniqueAppJs.length = 0;
});

Unfortunately you can use the async/await on http module when making requests. You can install and use axios module . In your case it will be something like this
const axios = require('axios');
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
const promise = axios.get(urlOfYtGetFunc).then(data => {
//do your data manipulations here
})
.catch(err => {
//decide what happens on error
})
Or async await
const data = await axios.get(urlOfYtGetFunc);
//Your data variable will become what the api has returned
If you still want to catch errors on async await you can use try catch
try{
const data = await axios.get(urlOfYtGetFunc);
}catch(err){
//In case of error do something
}

I have just looked at the code I think the issue is how you are handling the async code in the request handler. You are not awaiting the result of the function call to httpsYtGetFunc in the body so when it returns before the promise is finished which is why you get the Promise {Pending}.
Another issue is that the array is not extractedResultOfYtGetFunc is not initialised and you may access indexes that don't exist. The method to add an item to the array is push.
To fix this you need to restructure your code slightly. A possible solution is something like this,
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI";
urlOfYtGetFunc =
"https://www.googleapis.com/youtube/v3/search?key=" +
apiKey +
"&part=snippet&q=" +
queryOfYtGetFunc +
"&maxResults=4&order=relevance&type=video";
return new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = [];
response.on("data", (d) => {
chunks.push(d);
});
// Parsing the chunks
response.on("end", () => {
// Initialising the array
extractedResultOfYtGetFunc = []
resultOfYtGetFunc = JSON.parse(Buffer.concat(chunks).toString());
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < resultOfYtGetFunc.items.length; i++) {
// Adding the element to the array
extractedResultOfYtGetFunc.push(resultOfYtGetFunc.items[i].id.videoId);
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
});
});
});
}
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = await httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Another option would be to use axios,
The code for this would just be,
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
try{
ytQueryAppJs = await axios.get(url); // replace with your URL
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
} catch(e) {
console.log(e);
}
});
Using Axios would be a quicker way as you don't need to write promise wrappers around everything, which is required as the node HTTP(S) libraries don't support promises out of the box.

async functions not executing in the correct order inside a map function

I have created an async function that will extra the data from the argument, create a Postgres query based on a data, then did some processing using the retrieved query data. Yet, when I call this function inside a map function, it seemed like it has looped through all the element to extra the data from the argument first before it proceed to the second and the third part, which lead to wrong computation on the second element and onwards(the first element is always correct). I am new to async function, can someone please take at the below code? Thanks!
async function testWeightedScore(test, examData) {
var grade = [];
const testID = examData[test.name];
console.log(testID);
var res = await DefaultPostgresPool().query(
//postgres query based on the score constant
);
var result = res.rows;
for (var i = 0; i < result.length; i++) {
const score = result[i].score;
var weightScore = score * 20;
//more computation
const mid = { "testID": testID, "score": weightScore, more values...};
grade.push(mid);
}
return grade;
}
(async () => {
const examSession = [{"name": "Sally"},{"name": "Bob"},{"name": "Steph"}]
const examData = {
"Sally": 384258,
"Bob": 718239,
"Steph": 349285,
};
var test = [];
examSession.map(async sesion => {
var result = await testWeightedScore(sesion,examData);
let counts = result.reduce((prev, curr) => {
let count = prev.get(curr.testID) || 0;
prev.set(curr.testID, curr.score + count);
return prev;
}, new Map());
let reducedObjArr = [...counts].map(([testID, score]) => {
return {testID, score}
})
console.info(reducedObjArr);
}
);
})();
// The console log printed out all the tokenID first(loop through all the element in examSession ), before it printed out reducedObjArr for each element

The async/await behaviour is that the code pause at await, and do something else (async) until the result of await is provided.
So your code will launch a testWeightedScore, leave at the postgresql query (second await) and in the meantime go to the other entries in your map, log the id, then leave again at the query level.
I didn't read your function in detail however so I am unsure if your function is properly isolated or the order and completion of each call is important.
If you want each test to be fully done one after the other and not in 'parallel', you should do a for loop instead of a map.

Adding more elements to an object in javascript

Context: I'm fetching 'car' data below (see the code that starts with "for") from a GET request and am pushing it to the 'array' array. And for each car.ID that I get, I need to run another GET request in sequence (the GET uses car.ID as a parameter and I have no problems in doing this).
Problem: after I fetch the results from the second GET, how to push the data to the same object of the array (i.e. I want to "complement" the object above that ended on car.BrandID by adding a few more key: value pairs to the same "line")?
THANK YOU IN ADVANCE.
for (let car of carsJustObtained) {
for (i=0; i<=2; i++){
array.push(
{
timestamp: epoch,
ID : car.ID,
BrandID : car.BrandID
})
//code continues but don't worry
FULL CODE BELOW:
function gotCars(carsJustObtained) {
for (let car of carsJustObtained) {
for (i=0; i<=2; i++){
array.push(
{
timestamp: epoch,
ID : car.ID,
BrandID : car.BrandID,
ModelID : car.ModelID,
}
);
//given car.ID the second GET will be triggered because the path depends on this variable!
let path_get_all_prices = `xxx=${car.ID}?securityToken=xxx&vehiclePriceTypeID=xxx`;
let get = https.get(
{
hostname: 'xxx.com',
path: path_get_all_prices
},
(getRes) => {
console.log(`getting prices for car ${car.ID}...`);
var reply = "";
getRes.on("data", (chunk) => (reply += chunk));
const obj = JSON.parse(reply);
gotPrices(obj.Response);
}
);
function gotPrices(pricesJustObtained) {
for (let price of pricesJustObtained){
array.push(
//how to add results to the same array of the 1st GET? There are three 'prices' for each car.ID
)};
};
};

You have to find the index of your object in your array, then you can add everything you can to this object :
array[index].name = 'Hello';

There are many ways to do this. I recommend you read about array.map()
This function lets you iterate your array and in each iteration perform the get request and extend the current element.

The key is to recognise that you are not pushing, the second time
What you are doing is reading each element of the array, and adding some information to that element.
for (let car of carsJustObtained) {
const newInfo=getFurtherInformationAboutCar(car) // this is your second getting
car.newInfo1 = newInfo.param1;
car.newInfo2 = newInfo.param2;
car.newInfo3 = newInfo.param3;
}
To answer your specific question about "merging" information
If you have one set of properties already defined for the car, and you want to merge in multiple new properties, a simple way to do it is as follows:
car = { ...car, ...objectContainingNewProperties};
If your original car was {a:2, b:3, c:4} and objectContainingNewProperties was {c: 10, d:20, e:30}, the result would be:
{ a:2,
b:3,
c:10,
d:20,
e:30 }
Any same-named properties in the second object will overwrite those in the original object.

Your second request is of course asynchronous, so by the time you get its response, you have already populated your array with all information from the first request.
I would suggest to use a promise-enabled alternative to http.get, as promises are a native feature in JavaScript that makes working with asynchronous events less messy. I will show here how it can work with node-fetch.
As fetch is natively supported in browser agents, you can run the snippet below to see the result. As a demo I have used https://jsonplaceholder.typicode.com/ as a server resource: it returns JSON for several sample datasets, including todos and users. A todo has some properties (like a title) and has a user id. A user has an email and a username. So we could make the todos-request the first request, and the users-request the second one (based on the user id received in the first). So the principle is the same as with your cars and prices.
This relies heavily on promises:
// For demo, we use these two URls:
// They both need a number following it
let url1 = "https://jsonplaceholder.typicode.com/todos/";
let url2 = "https://jsonplaceholder.typicode.com/users/";
let promises = [];
// Let's say we build an array with 5 objects:
for (let i = 1; i <= 5; i++) {
promises.push(
// Make the request
fetch(url1 + i*30)
// Parse the response as JSON
.then(resp => resp.json())
// Process this data
.then(data => {
// Create our own object from this data
let obj = {
user: data.userId,
todo: data.title
};
// Make second request, to get user's email, joining it with obj
return Promise.all([obj, fetch(url2 + obj.user)])
})
.then(([obj, resp2]) => Promise.all([obj, resp2.json()]))
// Merge the new data with the old
.then(([obj, data2]) => Object.assign(obj, {
email: data2.email,
logon: data2.username
}))
);
}
// Wait for all requests to finish...
Promise.all(promises).then(results => {
console.log(results); // The result!
});
With async/await
The above can be made even more readable, if you use the async/await syntax:
let url1 = "https://jsonplaceholder.typicode.com/todos/";
let url2 = "https://jsonplaceholder.typicode.com/users/";
async function getOne(i) {
let resp = await fetch(url1 + i*30);
let data = await resp.json();
// Create object from first request
let obj = {
user: data.userId,
todo: data.title
};
// Make second request, joining it with obj
let resp2 = await fetch(url2 + obj.user);
let data2 = await resp2.json();
return Object.assign(obj, {
email: data2.email,
logon: data2.username
});
}
let promises = [];
for (let i = 1; i <= 5; i++) {
promises.push(getOne(i));
}
Promise.all(promises).then(results => {
console.log(results);
});

asynchronous loop for in Javascript

I'm trying to iterate and print out in order an array in Javascript that contains the title of 2 events that I obtained from doing web scraping to a website but it prints out in disorder. I know Javascript is asynchronous but I'm new in this world of asynchronism. How can I implement the loop for to print the array in order and give customized info?
agent.add('...') is like console.log('...'). I'm doing a chatbot with DialogFlow and NodeJs 8 but that's not important at this moment. I used console.log() in the return just for debug.
I tried the next:
async function printEvent(event){
agent.add(event)
}
async function runLoop(eventsTitles){
for (let i = 0; i<eventsTitles.length; i++){
aux = await printEvent(eventsTitles[i])
}
}
But i got this error error Unexpected await inside a loop no-await-in-loop
async function showEvents(agent) {
const cheerio = require('cheerio');
const rp = require('request-promise');
const options = {
uri: 'https://www.utb.edu.co/eventos',
transform: function (body) {
return cheerio.load(body);
}
}
return rp(options)
.then($ => {
//** HERE START THE PROBLEM**
var eventsTitles = [] // array of event's titles
agent.add(`This mont we have these events available: \n`)
$('.product-title').each(function (i, elem) {
var event = $(this).text()
eventsTitles.push(event)
})
agent.add(`${eventsTitles}`) // The array prints out in order but if i iterate it, it prints out in disorder.
// *** IMPLEMENT LOOP FOR ***
agent.add(`To obtain more info click on this link https://www.utb.edu.co/eventos`)
return console.log(`Show available events`);
}).catch(err => {
agent.add(`${err}`)
return console.log(err)
})
}
I would like to always print out Event's title #1 and after Event's title #2. Something like this:
events titles.forEach((index,event) => {
agent.add(`${index}. ${event}`) // remember this is like console.log(`${index}. ${event}`)
})
Thanks for any help and explanation!

There no async case here but if you still face difficultly than use this loop
for (let index = 0; index < eventsTitles.length; index++) {
const element = eventsTitles[index];
agent.add(${index}. ${element})
}

How to query SQL in a loop in node js

SCENARIO:
Using mssql I'm connecting to sql and retrieving a list of ids, then based on those id I want to run stored procedures. What I'm currently doing is running the first stored proc, storing the id's in an array, then I'm running a for loop calling another module, where I pass the id to run a stored proc. This works fine when I've got a single id, but fails with 'Global connection already exists. Call sql.close() first.' when I try to run multiple ones.
How do I create connect to sql, run my query, then run the next one? What's the best approach?
The code below runs the stored proc with ids and causes the above error.
exports.runStoredProc = function (query,id) {
sql.connect(config.config).then(()=>{
return sql.query`${query} ${id}`
}).then(res=> {
do something with the response
}).catch(error => {
console.log(error)
})
}
Looks like the connection still exists when the below bit of code runs it using next id. I thought that creating a Promise will force to await execution before it runs the above bit of code again?
let toRun = result.recordset.length
let gen = async num => {
for(let i=0;i<num;i++) {
var resp = result.recordset[i].id
console.log(i, resp)
var sp = report
var reportId = await new Promise(() => db.runStoredProc(sp,resp))
}
}
gen(toRun).then(() => console.log("done!"))

You need to return Promise from runStoredProc
exports.runStoredProc = function (query,id) {
return sql.connect(config.config).then(()=>{
return sql.query`${query} ${id}`
}).then(res=> {
do something with the response
}).catch(error => {
console.log(error)
})
}
and no need to wrap db.runStoredProc in loop
let toRun = result.recordset.length
let gen = async num => {
for(let i=0;i<num;i++) {
var resp = result.recordset[i].id
console.log(i, resp)
var sp = report
var reportId = await db.runStoredProc(sp,resp)
}
}
gen(toRun).then(() => console.log("done!"))