Push inside forEach with query not working properly - javascript

I'm working with mongodb stitch/realm and I'm trying to modify objects inside an array with a foreach and also pushing ids into a new array.
For each object that i'm modifying, I'm also doing a query first, after the document is found I start modifying the object and then pushing the id into another array so I can use both arrays later.
The code is something like this:
exports = function(orgLoc_id, data){
var HttpStatus = require('http-status-codes');
// Access DB
const db_name = context.values.get("database").name;
const db = context.services.get("mongodb-atlas").db(db_name);
const orgLocPickupPointCollection = db.collection("organizations.pickup_points");
const orgLocStreamsCollection = db.collection("organizations.streams");
const streamsCollection = db.collection("streams");
let stream_ids = [];
data.forEach(function(stream) {
return streamsCollection.findOne({_id: stream.stream_id}, {type: 1, sizes: 1}).then(res => { //if I comment this query it will push without any problem
if(res) {
let newId = new BSON.ObjectId();
stream._id = newId;
stream.location_id = orgLoc_id;
stream.stream_type = res.type;
stream.unit_price = res.sizes[0].unit_price_dropoff;
stream._created = new Date();
stream._modified = new Date();
stream._active = true;
stream_ids.push(newId);
}
})
})
console.log('stream ids: ' + stream_ids);
//TODO
};
But when I try to log 'stream_ids' it's empty and nothing is shown. Properties stream_type and unit_price are not assigned.
I've tried promises but I haven't had success

It's an asynchronous issue. You're populating the value of the array inside a callback. But because of the nature of the event loop, it's impossible that any of the callbacks will have been called by the time the console.log is executed.
You mentioned a solution involving promises, and that's probably the right tack. For example something like the following:
exports = function(orgLoc_id, data) {
// ...
let stream_ids = [];
const promises = data.map(function(stream) {
return streamsCollection.findOne({ _id: stream.stream_id }, { type: 1, sizes: 1 })
.then(res => { //if I comment this query it will push without any problem
if (res) {
let newId = new BSON.ObjectId();
// ...
stream_ids.push(newId);
}
})
})
Promise.all(promises).then(function() {
console.log('stream ids: ' + stream_ids);
//TODO
// any code that needs access to stream_ids should be in here...
});
};
Note the change of forEach to map...that way you're getting an array of all the Promises (I'm assuming your findOne is returning a promise because of the .then).
Then you use a Promise.all to wait for all the promises to resolve, and then you should have your array.
Side note: A more elegant solution would involve returning newId inside your .then. In that case Promise.all will actually resolve with an array of the results of all the promises, which would be the values of newId.

Related

How to correctly use 'async, await and promises' in nodejs, while allocating values to a variable returned from a time-consuming function?

Problem Statement:
Our aim is to allocate values in the array ytQueryAppJs, which are returned from a time consuming function httpsYtGetFunc().
The values in ytQueryAppJs needs to be used many times in further part of the code, hence it needs to be done 'filled', before the code proceeds further.
There are many other arrays like ytQueryAppJs, namely one of them is ytCoverAppJs, that needs to be allocated the value, the same way as ytQueryAppJs.
The values in ytCoverAppJs further require the use of values from ytQueryAppJs. So a solution with clean code would be highly appreciated.
(I am an absolute beginner. I have never used async, await or promises and I'm unaware of the correct way to use it. Please guide.)
Flow (to focus on):
The user submits a queryValue in index.html.
An array ytQueryAppJs is logged in console, based on the query.
Expected Log in Console (similar to):
Current Log in Console:
Flow (originally required by the project):
User submits query in index.html.
The values of arrays, ytQueryAppJs, ytCoverAppJs, ytCoverUniqueAppJs, ytLiveAppJs, ytLiveUniqueAppJs gets logged in the console, based on the query.
Code to focus on, from 'app.js':
// https://stackoverflow.com/a/14930567/14597561
function compareAndRemove(removeFromThis, compareToThis) {
return (removeFromThis = removeFromThis.filter(val => !compareToThis.includes(val)));
}
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
let promise = new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = []
response.on('data', (d) => {
chunks.push(d)
})
// Parsing the chunks
response.on('end', () => {
resultOfYtGetFunc = JSON.parse((Buffer.concat(chunks).toString()))
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < (resultOfYtGetFunc.items).length; i++) {
extractedResultOfYtGetFunc[i] = resultOfYtGetFunc.items[i].id.videoId;
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
})
})
})
let result = await promise;
return result;
}
app.post("/", function(req, res) {
// Accessing the queryValue, user submitted in index.html. We're using body-parser package here.
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Complete app.post method from app.js:
(For better understanding of the problem.)
app.post("/", function(req, res) {
// Accessing the queryValue user submitted in index.html.
query = req.body.queryValue;
// Fetcing top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
// Fetching 'cover' songs related to user's query and putting them in the array.
if (query.includes("cover") == true) {
ytCoverAppJs = httpsYtGetFunc(query);
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else {
ytCoverAppJs = httpsYtGetFunc(query + " cover");
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
}
// Fetching 'live performances' related to user's query and putting them in the array.
if (query.includes("live") == true) {
ytLiveAppJs = httpsYtGetFunc(query);
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else {
ytLiveAppJs = httpsYtGetFunc(query + " live");
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
}
// Emptying all the arrays.
ytQueryAppJs.length = 0;
ytCoverAppJs.length = 0;
ytCoverUniqueAppJs.length = 0;
ytLiveAppJs.length = 0;
ytLiveUniqueAppJs.length = 0;
});
Unfortunately you can use the async/await on http module when making requests. You can install and use axios module . In your case it will be something like this
const axios = require('axios');
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
const promise = axios.get(urlOfYtGetFunc).then(data => {
//do your data manipulations here
})
.catch(err => {
//decide what happens on error
})
Or async await
const data = await axios.get(urlOfYtGetFunc);
//Your data variable will become what the api has returned
If you still want to catch errors on async await you can use try catch
try{
const data = await axios.get(urlOfYtGetFunc);
}catch(err){
//In case of error do something
}
I have just looked at the code I think the issue is how you are handling the async code in the request handler. You are not awaiting the result of the function call to httpsYtGetFunc in the body so when it returns before the promise is finished which is why you get the Promise {Pending}.
Another issue is that the array is not extractedResultOfYtGetFunc is not initialised and you may access indexes that don't exist. The method to add an item to the array is push.
To fix this you need to restructure your code slightly. A possible solution is something like this,
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI";
urlOfYtGetFunc =
"https://www.googleapis.com/youtube/v3/search?key=" +
apiKey +
"&part=snippet&q=" +
queryOfYtGetFunc +
"&maxResults=4&order=relevance&type=video";
return new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = [];
response.on("data", (d) => {
chunks.push(d);
});
// Parsing the chunks
response.on("end", () => {
// Initialising the array
extractedResultOfYtGetFunc = []
resultOfYtGetFunc = JSON.parse(Buffer.concat(chunks).toString());
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < resultOfYtGetFunc.items.length; i++) {
// Adding the element to the array
extractedResultOfYtGetFunc.push(resultOfYtGetFunc.items[i].id.videoId);
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
});
});
});
}
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = await httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Another option would be to use axios,
The code for this would just be,
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
try{
ytQueryAppJs = await axios.get(url); // replace with your URL
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
} catch(e) {
console.log(e);
}
});
Using Axios would be a quicker way as you don't need to write promise wrappers around everything, which is required as the node HTTP(S) libraries don't support promises out of the box.

How to make multiple requests and save the data objects into a new array

I have two methods, which make a GET request. The first method saves user objects into the Benutzer array and in the benutzerIDs array only the user ids are saved. In the second method I'd like to loop through the benutzerIDs array and use these IDs for another GET Request and save the Objects into the array test. I have a problem with executing this request because of the multiple IDs. Currently I can't save the data into test. How can I loop throught the benutzerIDs and make a request based on this IDs?
getBenutzerDesVorgesetzten(id) {
axios
.get(server.baseURL + "/benutzer/vorgesetzter/" + id)
.then(response => {
this.benutzerIDs = [];
this.Benutzer = response.data;
this.Benutzer.forEach((item) => {
this.benutzerIDs.push(item.BenutzerID);
});
});
this.getAnträgeByBenutzerID();
},
getAnträgeByBenutzerID(){
this.test = [];
this.benutzerIDs.forEach((id) =>
this.test = axios.all([axios.get(server.baseURL + "/urlaubsantrag/benutzer/" + id)]))
}
Try this:
getAnträgeByBenutzerID(){
this.test = [];
const promises = [];
this.benutzerIDs.forEach(id => {
promises.push(axios.get(server.baseURL + "/urlaubsantrag/benutzer/" + id))
})
axios
.all(promises)
.then(res =>
//res is an array of raw data
this.test = res.map(item => item.data);
)
}
You'll need to use Promise.all and Array.prototype.map:
getAnträgeByBenutzerID(){
return Promise.all(this.benutzerIDs.map((id) =>
axios.get(server.baseURL + "/urlaubsantrag/benutzer/" + id)])));
}
The map callback creates an array of promises which are then collectively returned once they are all resolved.

Adding more elements to an object in javascript

Context: I'm fetching 'car' data below (see the code that starts with "for") from a GET request and am pushing it to the 'array' array. And for each car.ID that I get, I need to run another GET request in sequence (the GET uses car.ID as a parameter and I have no problems in doing this).
Problem: after I fetch the results from the second GET, how to push the data to the same object of the array (i.e. I want to "complement" the object above that ended on car.BrandID by adding a few more key: value pairs to the same "line")?
THANK YOU IN ADVANCE.
for (let car of carsJustObtained) {
for (i=0; i<=2; i++){
array.push(
{
timestamp: epoch,
ID : car.ID,
BrandID : car.BrandID
})
//code continues but don't worry
FULL CODE BELOW:
function gotCars(carsJustObtained) {
for (let car of carsJustObtained) {
for (i=0; i<=2; i++){
array.push(
{
timestamp: epoch,
ID : car.ID,
BrandID : car.BrandID,
ModelID : car.ModelID,
}
);
//given car.ID the second GET will be triggered because the path depends on this variable!
let path_get_all_prices = `xxx=${car.ID}?securityToken=xxx&vehiclePriceTypeID=xxx`;
let get = https.get(
{
hostname: 'xxx.com',
path: path_get_all_prices
},
(getRes) => {
console.log(`getting prices for car ${car.ID}...`);
var reply = "";
getRes.on("data", (chunk) => (reply += chunk));
const obj = JSON.parse(reply);
gotPrices(obj.Response);
}
);
function gotPrices(pricesJustObtained) {
for (let price of pricesJustObtained){
array.push(
//how to add results to the same array of the 1st GET? There are three 'prices' for each car.ID
)};
};
};
You have to find the index of your object in your array, then you can add everything you can to this object :
array[index].name = 'Hello';
There are many ways to do this. I recommend you read about array.map()
This function lets you iterate your array and in each iteration perform the get request and extend the current element.
The key is to recognise that you are not pushing, the second time
What you are doing is reading each element of the array, and adding some information to that element.
for (let car of carsJustObtained) {
const newInfo=getFurtherInformationAboutCar(car) // this is your second getting
car.newInfo1 = newInfo.param1;
car.newInfo2 = newInfo.param2;
car.newInfo3 = newInfo.param3;
}
To answer your specific question about "merging" information
If you have one set of properties already defined for the car, and you want to merge in multiple new properties, a simple way to do it is as follows:
car = { ...car, ...objectContainingNewProperties};
If your original car was {a:2, b:3, c:4} and objectContainingNewProperties was {c: 10, d:20, e:30}, the result would be:
{ a:2,
b:3,
c:10,
d:20,
e:30 }
Any same-named properties in the second object will overwrite those in the original object.
Your second request is of course asynchronous, so by the time you get its response, you have already populated your array with all information from the first request.
I would suggest to use a promise-enabled alternative to http.get, as promises are a native feature in JavaScript that makes working with asynchronous events less messy. I will show here how it can work with node-fetch.
As fetch is natively supported in browser agents, you can run the snippet below to see the result. As a demo I have used https://jsonplaceholder.typicode.com/ as a server resource: it returns JSON for several sample datasets, including todos and users. A todo has some properties (like a title) and has a user id. A user has an email and a username. So we could make the todos-request the first request, and the users-request the second one (based on the user id received in the first). So the principle is the same as with your cars and prices.
This relies heavily on promises:
// For demo, we use these two URls:
// They both need a number following it
let url1 = "https://jsonplaceholder.typicode.com/todos/";
let url2 = "https://jsonplaceholder.typicode.com/users/";
let promises = [];
// Let's say we build an array with 5 objects:
for (let i = 1; i <= 5; i++) {
promises.push(
// Make the request
fetch(url1 + i*30)
// Parse the response as JSON
.then(resp => resp.json())
// Process this data
.then(data => {
// Create our own object from this data
let obj = {
user: data.userId,
todo: data.title
};
// Make second request, to get user's email, joining it with obj
return Promise.all([obj, fetch(url2 + obj.user)])
})
.then(([obj, resp2]) => Promise.all([obj, resp2.json()]))
// Merge the new data with the old
.then(([obj, data2]) => Object.assign(obj, {
email: data2.email,
logon: data2.username
}))
);
}
// Wait for all requests to finish...
Promise.all(promises).then(results => {
console.log(results); // The result!
});
With async/await
The above can be made even more readable, if you use the async/await syntax:
let url1 = "https://jsonplaceholder.typicode.com/todos/";
let url2 = "https://jsonplaceholder.typicode.com/users/";
async function getOne(i) {
let resp = await fetch(url1 + i*30);
let data = await resp.json();
// Create object from first request
let obj = {
user: data.userId,
todo: data.title
};
// Make second request, joining it with obj
let resp2 = await fetch(url2 + obj.user);
let data2 = await resp2.json();
return Object.assign(obj, {
email: data2.email,
logon: data2.username
});
}
let promises = [];
for (let i = 1; i <= 5; i++) {
promises.push(getOne(i));
}
Promise.all(promises).then(results => {
console.log(results);
});

Nested Firebase Firestore forEach promise queries

I am using Firebase Cloud Firestore, however, I think this may be more of a JavaScript promise issue.
I have a collection called "students" which I am querying. For each found student I want to issue another query to find "parents" related by id.
For this, I have to nest a promise / foreach query and result inside another promise / foreach query and result.
Its currently executing the entire "students" promise/loop, returning from the function, then executing each of the "parents" promise/loops afterwards.
I want it to step through one found student, then execute all parents for that student, then go onto the next student, then return from the function.
getStudents(grade) {
let grade_part = this.getGrade(grade);
var dbRef = db.collection("students");
var dbQuery = dbRef.where('bb_current_grade', '==', grade_part);
var dbPromise = dbQuery.get();
var allStudents = [];
return dbPromise.then(function(querySnapshot) {
querySnapshot.forEach(doc => {
console.log("student");
var studentPlusParents = doc.data();
studentPlusParents.parents = [];
var dbRef = db.collection("parents");
var dbQuery = dbRef.where('student_id', '==', doc.data().id);
var dbPromise = dbQuery.get();
dbPromise.then(function(querySnapshot) {
querySnapshot.forEach(parentDoc => {
console.log("Add parent");
studentPlusParents.parents.push(parentDoc.data());
});
});
//console.log(studentPlusParents);
allStudents.push(studentPlusParents)
});
//console.log(allStudents);
return Promise.all(allStudents);
})
.catch(function(error) {
console.log("Error getting documents: ", error);
});
}

Unhandled Promise rejection of a property undefined

setInterval(function(){
for(i=1;i<=len;i++)
{
plug[i].getSysInfo().then(data => {
var json = JSON.stringify(data,undefined,2);
var obj = JSON.parse(json);
var _macadd = obj.mac;
plug[i].getConsumption().then(data => {
var json = JSON.stringify(data,undefined,2);
var obj = JSON.parse(json);
var _current = obj.current;
var _voltage = obj.voltage;
var _power = (obj.power).toPrecision(5);
var _total = obj.total;
var _err_code = obj.err_code;
console.log(_power);
//console.log(_macadd);
//console.log(voltage);
if(_power > 1800)
{
_power = 0.00;
}
// var fs = require('fs');
// fs.writeFile("emeter.json", json);
var sql1 = "INSERT INTO emeter SET ? ";
var post1 = { macaddress: _macadd,current: _current,voltage: _voltage,power: _power, total: _total,err_code: _err_code};
var tmp = JSON.stringify(post1,undefined,2);
console.log(tmp);
con.query(sql1, post1, function (err, result) {
if (err) throw err;
console.log("inserted");
});
});
});
}
},30000);
While I run this, I am getting below error
Unhandled promise rejection Type error:
Cannot read property of 'getconsumption' of undefined.
The above code worked well when I replace the index with one value.To retrieve just one value it is working. When it is in For loop then the error message is returned. Please help me in solving this.
You're trying to use a for loop index value inside an asynchronous callback. The problem is that the .then() handlers are called AFTER the for loop has already finished and thus the for loop index is at its terminal value (beyond the end of the array) when you try to use it.
There are many ways to fix this. One simple way would be to change this:
for(i=1;i<=len;i++)
to this:
for(let i=1;i<=len;i++)
Using ES6 let creates a new unique variable for each execution of the for loop so that each one maintains its own copy of the index, even for the duration of the asynchronous calls within that for loop block.
In ES5 (where you don't have let), you could change your loop to use .forEach():
plug.forEach(function(val, index) {
// use val and index here rather than plug[i]
});
This also creates a new val and index variable for each invocation of the loop.

Categories