We Keep Coding

How to correctly use 'async, await and promises' in nodejs, while allocating values to a variable returned from a time-consuming function?

Problem Statement:
Our aim is to allocate values in the array ytQueryAppJs, which are returned from a time consuming function httpsYtGetFunc().
The values in ytQueryAppJs needs to be used many times in further part of the code, hence it needs to be done 'filled', before the code proceeds further.
There are many other arrays like ytQueryAppJs, namely one of them is ytCoverAppJs, that needs to be allocated the value, the same way as ytQueryAppJs.
The values in ytCoverAppJs further require the use of values from ytQueryAppJs. So a solution with clean code would be highly appreciated.
(I am an absolute beginner. I have never used async, await or promises and I'm unaware of the correct way to use it. Please guide.)
Flow (to focus on):
The user submits a queryValue in index.html.
An array ytQueryAppJs is logged in console, based on the query.
Expected Log in Console (similar to):
Current Log in Console:
Flow (originally required by the project):
User submits query in index.html.
The values of arrays, ytQueryAppJs, ytCoverAppJs, ytCoverUniqueAppJs, ytLiveAppJs, ytLiveUniqueAppJs gets logged in the console, based on the query.
Code to focus on, from 'app.js':
// https://stackoverflow.com/a/14930567/14597561
function compareAndRemove(removeFromThis, compareToThis) {
return (removeFromThis = removeFromThis.filter(val => !compareToThis.includes(val)));
}
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
let promise = new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = []
response.on('data', (d) => {
chunks.push(d)
})
// Parsing the chunks
response.on('end', () => {
resultOfYtGetFunc = JSON.parse((Buffer.concat(chunks).toString()))
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < (resultOfYtGetFunc.items).length; i++) {
extractedResultOfYtGetFunc[i] = resultOfYtGetFunc.items[i].id.videoId;
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
})
})
})
let result = await promise;
return result;
}
app.post("/", function(req, res) {
// Accessing the queryValue, user submitted in index.html. We're using body-parser package here.
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Complete app.post method from app.js:
(For better understanding of the problem.)
app.post("/", function(req, res) {
// Accessing the queryValue user submitted in index.html.
query = req.body.queryValue;
// Fetcing top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
// Fetching 'cover' songs related to user's query and putting them in the array.
if (query.includes("cover") == true) {
ytCoverAppJs = httpsYtGetFunc(query);
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else {
ytCoverAppJs = httpsYtGetFunc(query + " cover");
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
}
// Fetching 'live performances' related to user's query and putting them in the array.
if (query.includes("live") == true) {
ytLiveAppJs = httpsYtGetFunc(query);
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else {
ytLiveAppJs = httpsYtGetFunc(query + " live");
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
}
// Emptying all the arrays.
ytQueryAppJs.length = 0;
ytCoverAppJs.length = 0;
ytCoverUniqueAppJs.length = 0;
ytLiveAppJs.length = 0;
ytLiveUniqueAppJs.length = 0;
});

Unfortunately you can use the async/await on http module when making requests. You can install and use axios module . In your case it will be something like this
const axios = require('axios');
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
const promise = axios.get(urlOfYtGetFunc).then(data => {
//do your data manipulations here
})
.catch(err => {
//decide what happens on error
})
Or async await
const data = await axios.get(urlOfYtGetFunc);
//Your data variable will become what the api has returned
If you still want to catch errors on async await you can use try catch
try{
const data = await axios.get(urlOfYtGetFunc);
}catch(err){
//In case of error do something
}

I have just looked at the code I think the issue is how you are handling the async code in the request handler. You are not awaiting the result of the function call to httpsYtGetFunc in the body so when it returns before the promise is finished which is why you get the Promise {Pending}.
Another issue is that the array is not extractedResultOfYtGetFunc is not initialised and you may access indexes that don't exist. The method to add an item to the array is push.
To fix this you need to restructure your code slightly. A possible solution is something like this,
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI";
urlOfYtGetFunc =
"https://www.googleapis.com/youtube/v3/search?key=" +
apiKey +
"&part=snippet&q=" +
queryOfYtGetFunc +
"&maxResults=4&order=relevance&type=video";
return new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = [];
response.on("data", (d) => {
chunks.push(d);
});
// Parsing the chunks
response.on("end", () => {
// Initialising the array
extractedResultOfYtGetFunc = []
resultOfYtGetFunc = JSON.parse(Buffer.concat(chunks).toString());
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < resultOfYtGetFunc.items.length; i++) {
// Adding the element to the array
extractedResultOfYtGetFunc.push(resultOfYtGetFunc.items[i].id.videoId);
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
});
});
});
}
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = await httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Another option would be to use axios,
The code for this would just be,
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
try{
ytQueryAppJs = await axios.get(url); // replace with your URL
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
} catch(e) {
console.log(e);
}
});
Using Axios would be a quicker way as you don't need to write promise wrappers around everything, which is required as the node HTTP(S) libraries don't support promises out of the box.

How can I properly create an array and use it outside of the scope in this example?

I have written some code using a colleagues package "db". First it gets a list of sessions of a film shoot day, then it lists the captured shots in the shoot day sessions, and then the "getCapture" function gets the shot information for each shot. I am trying to use this getCapture function to read the shots' information(ie name, size, subjects) along with ['Node Information']['NAME'] in order to just get the shot names only. I want to do this with all sessions of the day and all their shots. I've figured out how to do this and return the list of names in the example below, using "console.log(captureInfo._data['Node Information']['NAME'])"..However I want to be able to use these results in an array outside of this scope. when I try to push these results like in the second example, it prints out several copies of the names over and over again. I want to be able to add the shoot names to an array so the array can be used outside of this whole scope, just not sure how to do so.
Example that works to log and list all the shots just once, but does not put the results into a variable/array:
var sessionURL = ("X:\\Private_LowPerf\\TESTING_DATA\\Capture day 1.file");
async function main (){
var captures = [];
var captureInfo = [];
var captureNames = [];
if (sessionURL){
var sessions = await _db.listSessions(sessionURL);
sessions.forEach(async (item) => {
if(sessions){
var captures = await _db.listCaptures(item);
}
captures.forEach(async (item) => {
if(sessions){
var captureInfo = await _db.getCapture(item.path);
console.log(captureInfo._data['Node Information']['NAME']);
}
})
})
}
}
main();
Example that isn't working because it prints out too many copies of the names due to a scoping issue I assume:
async function main (){
var captures = [];
var captureInfo = [];
var captureNames = [];
if (sessionURL){
var sessions = await _db.listSessions(sessionURL);
sessions.forEach(async (item) => {
if(sessions){
var captures = await _db.listCaptures(item);
}
captures.forEach(async (item) => {
if(sessions){
var captureInfo = await _db.getCapture(item.path);
}
captureNames.push(captureInfo._data['Node Information']['NAME']);
console.log(captureNames);
})
})
}
}
main();

You need to put the console log outside the loops, otherwise, everytime the loops execute you will print the current items, plus the previous ones.
Array.prototype.forEachAsync = async function(cb){
for(let x of this){
await cb(x);
}
}
async function main() {
var captures = [];
var captureInfo = [];
var captureNames = [];
if (sessionURL) {
var sessions = await _db.listSessions(sessionURL);
await sessions.forEachAsync(async (item) => {
if (sessions) {
var captures = await _db.listCaptures(item);
}
await captures.forEachAsync(async (item) => {
if (sessions) {
var captureInfo = await _db.getCapture(item.path);
captureNames.push(captureInfo._data['Node Information']['NAME']);
}
})
})
console.log(captureNames);
}
}
main();
EDIT: as you are dealing with async functions you need to be sure that your loops were executed before calling console.log. This is why we need the forEachAsync

How to properly use Parse / Promise?

I am writing some JavaScript codes using Parse.com.
To be honest, I have been reading how to use Promise and done lots of research but cannot still figure out how to use it properly..
Here is a scenario:
I have two tables (objects) called Client and InvoiceHeader
Client can have multiple InvoiceHeaders.
InvoiceHeader has a column called "Amount" and I want a total amount of each client's InvoiceHeaders.
For example, if Client A has two InvoiceHeaders with amount 30 and 20 and Client B has got nothing, the result I want to see in tempArray is '50, 0'.
However, with the following codes, it looks like it's random. I mean sometimes the tempArray got '50, 50' or "50, 0". I suspect it is due to the wrong usage of Promise.
Please help me. I have been looking into the codes and stuck for a few days.
$(document).ready(function() {
var client = Parse.Object.extend("Client");
var query = new Parse.Query(client);
var tempArray = [];
query.find().then(function(objects) {
return objects;
}).then(function (objects) {
var promises = [];
var totalForHeader = 0;
objects.forEach(function(object) {
totalForHeader = 0;
var invoiceHeader = Parse.Object.extend('InvoiceHeader');
var queryForInvoiceHeader = new Parse.Query(invoiceHeader);
queryForInvoiceHeader.equalTo('headerClient', object);
var prom = queryForInvoiceHeader.find().then(function(headers) {
headers.forEach(function(header) {
totalForHeader += totalForHeader +
parseFloat(header.get('headerOutstandingAmount'));
});
tempArray.push(totalForHeader);
});
promises.push(prom);
});
return Parse.Promise.when.apply(Parse.Promise, promises);
}).then(function () {
// after all of above jobs are done, do something here...
});
} );

Assuming Parse.com's Promise class follows the A+ spec, and I understood which bits you wanted to end up where, this ought to work:
$(document).ready(function() {
var clientClass = Parse.Object.extend("Client");
var clientQuery = new Parse.Query(clientClass);
clientQuery.find().then(function(clients) {
var totalPromises = [];
clients.forEach(function(client) {
var invoiceHeaderClass = Parse.Object.extend('InvoiceHeader');
var invoiceHeaderQuery = new Parse.Query(invoiceHeaderClass);
invoiceHeaderQuery.equalTo('headerClient', client);
var totalPromise = invoiceHeaderQuery.find().then(function(invoiceHeaders) {
var totalForHeader = 0;
invoiceHeaders.forEach(function(invoiceHeader) {
totalForHeader += parseFloat(invoiceHeader.get('headerOutstandingAmount'));
});
return totalForHeader;
});
totalPromises.push(totalPromise);
});
return Parse.Promise.when(totalPromises);
}).then(function(totals) {
// here you can use the `totals` array.
});
});

Asynchronous for loop: how do I make the for loop wait for function to finish?

I'm racking my brain trying to figure out how to sequence this / place callbacks to get what I need.
I have a loop that checks for the existence of files. I'd like it to callback when it's done, but the for loop and the callback finish before the "fs.open" finishes... typical asynchronous problem.
I am using node v0.10.29, express v3.14.0, and am looking at using the "async" library, but again, just can't figure out the logic that I need...
Here is what I have:
Input
function checkForAllFiles(callback)
{
var requiredFiles = [];
requiredFiles[requiredFiles.length] = "../Client/database/one.js";
requiredFiles[requiredFiles.length] = "../Client/database/two.dat";
requiredFiles[requiredFiles.length] = "../Client/database/three.xml";
requiredFiles[requiredFiles.length] = "../Client/database/four.dat";
requiredFiles[requiredFiles.length] = "../Client/database/five.dat";
requiredFiles[requiredFiles.length] = "../Client/database/six.xml";
var missingFiles = [];
for(var r=0; r<requiredFiles.length; r++)
{
fs.open(requiredFiles[r], 'r', function(err, fd){
if(err)
{
missingFiles[missingFiles.length] = err.path;
console.log("found missing file = ", err.path);
}
});
console.log("r = ", r);
}
console.log("sending callback: ", missingFiles);
callback(missingFiles);
}
Output
0
1
2
3
4
5
sending callback: []
found missing file: ../Client/database/three.xml
Desired Output
0
1
found missing file: ../Client/database/three.xml
2
3
4
5
sending callback: ["../Client/database/three.xml"]

I would use the reject method in the async module (which I see you've already found). What it will do is return an array in its callback that contains any elements that don't match a specified check function. For the check function, I'd recommend just using fs.exists instead of watching for an error on fs.open.
Using those functions you can actually reduce the whole check to one line. Something like this:
function checkForAllFiles(callback)
{
var requiredFiles = [];
requiredFiles[requiredFiles.length] = "../Client/database/one.js";
requiredFiles[requiredFiles.length] = "../Client/database/two.dat";
requiredFiles[requiredFiles.length] = "../Client/database/three.xml";
requiredFiles[requiredFiles.length] = "../Client/database/four.dat";
requiredFiles[requiredFiles.length] = "../Client/database/five.dat";
requiredFiles[requiredFiles.length] = "../Client/database/six.xml";
async.reject(requiredFiles, fs.exists, callback);
}
callback will get called with an array that contains just the files that don't exist.

Use the async library and the eachSeries method. Example:
async.eachSeries(array,
function(element, next) {
// do something with element
next();
}
);
It will sequentially go through the array and process each element. Calling next goes to the next element. Series makes sure it does it in the order of the array, otherwise the order of going through the array is not guaranteed. If you have other async functions called within it, just pass the next function around and call it when done with all the needed functions and the next array element will be processed.
Maybe something like this:
var missingFiles = []
async.eachSeries(requiredFiles, function(file, nextFile){
fs.open(file, 'r', function(err, fd){
if(err)
{
missingFiles[missingFiles.length] = err.path;
console.log("found missing file = ", err.path);
}
nextFile();
});
console.log("r = ", file);
});
console.log("sending callback: ", missingFiles);
callback(missingFiles);

Grabbing and sending elements from MongoDB in Node.js server response

In my Express route, I am trying to return a list of elements that I am grabbing from MongoDB using Mongoose. I'm basically iterating through an array of items, and making MongoDB calls to get the parameter objects that each item has. However, I'm having trouble making sure that I get all the parameters before I send the response. I've tried using promises, other async library functions, etc, but none of them have seemed to work.
The current iteration of the code looks like this (I have tried a lot of different things):
exports.findAll = function(req, res){
Flow.find({}, function(err, items) {
console.log(items);
var payload = {}
var params = [];
for (var i=0; i < items.length; i++) {
var count2 = 0;
async.whilst(
function() {
return ((items[i]) && (count2 < items[i].params.length));
},
function(callback) {
Parameter.findById(items[i].params[count2], function(err, out) {
params.push(out);
count2++;
callback();
});
},
function(err) {
console.log(params);
var payload = {
"flows": items,
"params": params
};
res.send(payload);
console.log('success: flows found');
}
);
}
This code sends a payload with params not being completely full.
What would be a good way to deal with this? Honestly I just want these database calls to be synchronous, but I just can't figure out how to make this work.

This doesn't really seem necessary as you can actually use the $in operator with all the results from your first query:
Flow.find({},function(err,items) {
var ids = [];
// blocking? yes, but should be minor - do better if there are problems
for ( var i=0; i < items.length; i++ ) {
for ( var n=0; n < items[i].params.length; n++ ) {
ids.push( items[i].params[n] );
}
}
Parameter.find({ "_id": { "$in": ids } },function(err,params) {
res.send({ "flows": items, "params": params });
});
});
So there should be no reason to execute multiple queries inside an async loop, or loops as your code seems to be missing as the direct cause of the problem there.