I have the following code that I can´t figure out how to make this entire process finish before sending a response back to the client. It has a for loop and make the requests to 2 external functions.
The problem that I can see in the console is the response which goes way faster than the process takes to complete the processing. It makes the client believe everything is fine, but, actually some error might has happened. I´ve tried to await, I´ve tried reading other posts, I´ve tried making this to return a promise and return a resolve, I´ve tried an index to check the array length... If someone could help me out, I would appreciate that. Thanks in advance.
var updateProdCharTree = async (req, res, next) => {
return new Promise((resolve, reject) => {
var data = req.body.raw.data;
var productLine = req.body.raw.productLine;
var bu = req.body.raw.bu;
let contErros = null;
let contAcertos = 0;
var maxId = 0;
let queryMaxId = `SELECT max(ProductCharFatherId) as maxId FROM maxiplastmatriz.productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database.query(queryMaxId)
.then(resultqueryMaxId => {
if(resultqueryMaxId.length){
maxId = resultqueryMaxId[0]['maxId'];
}else{
maxId = 0;
}
let queryAllNodes = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database.query(queryAllNodes)
.then( async resultqueryAllNodes => {
for (let index = 0; index < data.children.length; index++) {
const element = data.children[index];
if (data.children[index].dbId != undefined) {
let query = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine} and Id=${data.children[index].dbId} and ProductCharFatherId=${data.children[index].id}`;
database.query(query)
.then( async result => {
if (result.length) { // Update char in productchar1
maxId++;
var params = {element: element, productLine: productLine, bu: bu, index: index};
waitResUp = await updateProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
contAcertos++;
} else { // Add char in productchar1 shouldn´t get in here
console.log("Erro em: updateProdCharTree > addProductProductChar1");
}
})
.catch(err => {
console.log("Erro em query");
contErros = 1;
})
}else{ // Add new char in productchar1
maxId++;
var params = {element: element, productLine: productLine, bu: bu, index: index, maxId: maxId};
waitRes = await addProductProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
console.log("waitRes", waitRes.insertId);
contAcertos++;
}
}
})
.catch(err => {
console.log("Erro em queryAllNodes", err);
contErros = 1;
})
})
.catch(err => {
console.log("Erro em queryMaxId");
contErros = 1;
});
if (contAcertos == data.children.length) {
resolve("result"); // ***RES ATTEMPT***
}
})
}
Beginner here. Please be pacient. You were once either.
For starters, you can refactor your code to something like below so that response is returned after all async functions within loop are executed:
var updateProdCharTree = async (req, res, next) => {
return new Promise((resolve, reject) => {
var data = req.body.raw.data;
var productLine = req.body.raw.productLine;
var bu = req.body.raw.bu;
let contErros = null;
let contAcertos = 0;
var maxId = 0;
let queryMaxId = `SELECT max(ProductCharFatherId) as maxId FROM maxiplastmatriz.productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database
.query(queryMaxId)
.then((resultqueryMaxId) => {
if (resultqueryMaxId.length) {
maxId = resultqueryMaxId[0]["maxId"];
} else {
maxId = 0;
}
let queryAllNodes = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database
.query(queryAllNodes)
.then(async (resultqueryAllNodes) => {
return Promise.all(
data.chilren.map(async (element) => {
if (data.children[index].dbId != undefined) {
let query = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine} and Id=${data.children[index].dbId} and ProductCharFatherId=${data.children[index].id}`;
let result = await database.query(query);
if (result.length) {
// Update char in productchar1
maxId++;
var params = {
element: element,
productLine: productLine,
bu: bu,
index: index,
};
waitResUp = await updateProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
contAcertos++;
} else {
// Add char in productchar1 shouldn´t get in here
console.log(
"Erro em: updateProdCharTree > addProductProductChar1"
);
}
} else {
// Add new char in productchar1
maxId++;
var params = {
element: element,
productLine: productLine,
bu: bu,
index: index,
maxId: maxId,
};
waitRes = await addProductProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
console.log("waitRes", waitRes.insertId);
contAcertos++;
}
})
);
})
.then(() => {
// This will be called after all queries are executed
resolve("result");
})
.catch((err) => {
console.log("Erro em queryAllNodes", err);
contErros = 1;
// NOTE: Ideally we should be rejecting promise in case of error
});
})
.catch((err) => {
console.log("Erro em queryMaxId");
contErros = 1;
// NOTE: Ideally we should be rejecting promise in case of error
});
});
};
As #Jeremy Thille suggested this code can be improvised further to only use async-await instead of Promises with then callback so that it is more cleaner and easy to understand.
Related
await mongodb.connect(process.env.MongoUrl , async (err, c)=>
{
if(err){console.log(err)}
var GetData = c.db("Data").collection("Servers")
const Server = await GetData.findOne({IdServer:id}).then(r => {return r})
if(Server != null)
{
return callback(Server.prefix)
}
else
{
GetData.insertOne({IdServer:id,Sersial:false,JoinServerHistory:new Date().toLocaleDateString(),History:"-",prefix:"-",caunt:50})
return callback("")
}
})
}
const Prfix = GetPrfix("9273490839208409" , (r) => {console.log(r)})
/////// ^<<<<<<<< take "r" and and Put in a variable <<<<<<<< ^
The best way is to declare it first, then assign it the value:
var newVar = null;
await mongodb.connect(process.env.MongoUrl , async (err, c)=>
{
if(err){console.log(err)}
var GetData = c.db("Data").collection("Servers")
const Server = await GetData.findOne({IdServer:id}).then(r => {newVar = r;}) // Right here is the magic
// ... SOME MORE CODE
})
}
// You can use `newVar` here as `r`.
I have tried using async-await, .then and now promise. I am quite new in javascript development.
The code
indexRouter.get('/dashboard', checkSignIn, async(request, response) => {
snapshot = await db.doc('users/accounts').get()
sites = snapshot.data()['sites']
const userId = request.session.userId
snapshot = await db.doc(`users/${userId}`).get()
var linkedSites = snapshot.data()['linked sites']
let getDs = new Promise((resolve,reject)=>{
console.log("1")
linkedSites.forEach((site) =>{
console.log("2")
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot)=>{
console.log("3")
snapshot.forEach((doc) => {
console.log("4")
console.log(doc.id)
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS.....
details.push({site:{'email': emailId, 'password': password, 'keys': keys}})
})
})
})
console.log("5")
console.log(details)
resolve(details)
}
);
getDs.then((details)=>{
console.log("rendering")
response.render('dashboard', {'details':details, 'linkedSites': linkedSites, 'sites': sites})
})
}
I am getting the response
1
2
2
5
[]
rendering
error: ...details not found in ejs
3
4
rsp121#gmail.com
3
4
test#gmail.com
According to the output, it seems like db.doc line after console.log(2) is getting executed after a certain time and resolve(details) is sent before.
Found a solution to the problem:
indexRouter.get('/dashboard', checkSignIn, async(request, response) => {
snapshot = await db.doc('users/accounts').get()
sites = snapshot.data()['sites']
const userId = request.session.userId
snapshot = await db.doc(`users/${userId}`).get()
var linkedSites = snapshot.data()['linked sites']
if(linkedSites){
const getSnapshot = (site) => {
return new Promise(resolve => {
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot) =>{
snapshot.forEach((doc) =>{
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS
details[site] = {'email': emailId, 'password': password, 'keys': keys}
resolve(true)
})
})
})
}
Promise.all(linkedSites.map(getSnapshot)).then(()=>{
console.log(linkedSites)
response.set('Cache-Control', 'no-store, no-cache, must-revalidate, private')
response.render('dashboard', {'details':details, 'linkedSites': linkedSites, 'sites': sites})
})
}
The problem is your promise resolved before the db.doc is resolved, and as your db.doc promise is inside a loop. So, you should be using the promise.all
The below code should work for you.
indexRouter.get("/dashboard", checkSignIn, async (request, response) => {
snapshot = await db.doc("users/accounts").get();
sites = snapshot.data()["sites"];
const userId = request.session.userId;
snapshot = await db.doc(`users/${userId}`).get();
var linkedSites = snapshot.data()["linked sites"];
let getDs = new Promise((resolve, reject) => {
const promises = [];
console.log("1");
linkedSites.forEach((site) => {
console.log("2");
promises.push(
new Promise((internalResolve) => {
db.doc(`users/${userId}`)
.collection(site)
.get()
.then((snapshot) => {
console.log("3");
snapshot.forEach((doc) => {
console.log("4");
console.log(doc.id);
emailId = doc.id;
keys = doc.data()["keys"];
var passwordEncrypt = doc.data()["password"];
password = cryptoJS;
details.push({
site: {
email: emailId,
password: password,
keys: keys,
},
});
internalResolve();
});
});
})
);
});
Promise.all(promises).then(() => {
console.log("5");
console.log(details);
resolve(details);
});
});
getDs.then((details) => {
console.log("rendering");
return response.render("dashboard", {
details: details,
linkedSites: linkedSites,
sites: sites,
});
});
});
More cleaner with async/await.
indexRouter.get("/dashboard", checkSignIn, async (request, response) => {
snapshot = await db.doc("users/accounts").get();
sites = snapshot.data()["sites"];
const userId = request.session.userId;
snapshot = await db.doc(`users/${userId}`).get();
var linkedSites = snapshot.data()["linked sites"];
console.log("1");
linkedSites.forEach(async (site) => {
console.log("2");
const snapshot = await db.doc(`users/${userId}`).collection(site).get();
console.log("3");
snapshot.forEach((doc) => {
console.log("4");
console.log(doc.id);
emailId = doc.id;
keys = doc.data()["keys"];
var passwordEncrypt = doc.data()["password"];
password = cryptoJS;
details.push({
site: {
email: emailId,
password: password,
keys: keys,
},
});
});
});
console.log("rendering");
return response.render("dashboard", {
details: details,
linkedSites: linkedSites,
sites: sites,
});
});
This is your code corrected, optimized and using last especifications of ECMA Script, the error is as the other comment said you are no waiting for the result of promises inside your "new Promise.." declaration. Dont foget to use try/catch inside async functions if you have not a top error handler.
The optimization is in firsts snapshot variables, this way you are getting data in parallel instead secuantially.
indexRouter.get('/dashboard', checkSignIn, async (request, response) => {
try {
let details = [];
const userId = request.session.userId
let [snapshot1, snapshot2] = await Promise.all([db.doc('users/accounts').get(), await db.doc(`users/${userId}`).get()])
let sites = snapshot1.data()['sites']
var linkedSites = snapshot2.data()['linked sites'];
await Promise.all(
linkedSites.map((site) =>
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot) => {
snapshot.forEach((doc) => {
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS
details.push({ site: { 'email': emailId, 'password': password, 'keys': keys } })
})
})
)
)
response.render('dashboard', { 'details': details, 'linkedSites': linkedSites, 'sites': sites })
} catch (e) {
//Render error you want
}})
linkedSites.map.... return an array of Promises that in the end are wrapped inside Promise.all and Promise.all wait until all promises are fullfilled or one of them is rejected in this last case your code goes to catch without reach the line response.render inside the try. You can avoid this catching locally the error of each promise in the map using
.then((snapshot) => {
...
}).catch(e=> { /*Do something with the rror*/})
I have an http trigger in cloud functions that is working, however I am getting some logs come back in a weird order. I am pretty sure I have not executed the promises correctly.
here is the code:
exports.createGame = functions.https.onRequest((req, res) => {
return cors(req, res, () => {
// Check for POST request
if (req.method !== "POST") {
res.status(400).send('Please send a POST request');
return;
}
const createGame = admin.firestore().collection('games')
generatePin()
function generatePin() {
...pin generating code goes here...
addGame(newGidToString)
}
function addGame(newGidToString) {
var getGame = admin.firestore().collection('games')
getGame = getGame.where('gid', '==', newGidToString)
getGame.get().then(snapshot => {
if (snapshot.empty) {
console.log('BODY ', req.body)
const promises = [];
const gameTitle = req.body.gametitle
const targetGID = req.body.target
const numPlayers = req.body.playercount.toString()
const newGID = newGidToString
const collections = ['games', 'clues', 'detours', 'messages', 'questions', 'tagstate']
collections.forEach(function (element) {
console.log('start loop', element)
let elementsRef = admin.firestore().collection(element)
elementsRef = elementsRef.where('gid', '==', targetGID)
elementsRef.get().then(snapshot => {
var batch = admin.firestore().batch()
snapshot.forEach(function (doc) {
// change the gid to the new game gid
let data = doc.data()
data.gid = newGID
if(data.template){
data.template = false
}
if(!data.parent) {
data.parent = targetGID
}
if (!data.playercount) {
data.playercount = numPlayers
}
if (data.gametitle) {
data.gametitle = gameTitle
}
let elementRef = admin.firestore().collection(element).doc()
batch.set(elementRef, data)
})
batch.commit().then(data => {
console.log('complete batch ', element)
})
})
})
res.send({ status: 200, message: 'Game: added.', pin: newGID})
console.log('completed');
} else {
console.log('found a match ', snapshot)
// re-run the generator for a new pin
generatePin()
}
})
}
})
})
Here is a screen shot of the console logs.
Notice the order of the logs. It would seem logical that completed would come at the end fo the loops.
Any help is great appreciated.
UPDATE:
function addGame(newGidToString) {
var getGame = admin.firestore().collection('games')
getGame = getGame.where('gid', '==', newGidToString)
getGame.get().then(snapshot => {
if (snapshot.empty) {
console.log('BODY ', req.body)
const promises = [];
const gameTitle = req.body.gametitle
const targetGID = req.body.target
const numPlayers = req.body.playercount.toString()
const newGID = newGidToString
const collections = ['games', 'clues', 'detours', 'messages', 'questions', 'tagstate']
collections.forEach(function (element) {
console.log('start loop', element)
let elementsRef = admin.firestore().collection(element)
elementsRef = elementsRef.where('gid', '==', targetGID)
// elementsRef.get().then(snapshot => {
const promGet = elementsRef.get(); //lets get our promise
promises.push(promGet); //place into an array
promGet.then(snapshot => { //we can now continue as before
var batch = admin.firestore().batch()
snapshot.forEach(function (doc) {
// change the gid to the new game gid
let data = doc.data()
data.gid = newGID
if(data.template){
data.template = false
}
if(!data.parent) {
data.parent = targetGID
}
if (!data.playercount) {
data.playercount = numPlayers
}
if (data.gametitle) {
data.gametitle = gameTitle
}
let elementRef = admin.firestore().collection(element).doc()
batch.set(elementRef, data)
})
batch.commit().then(data => {
console.log('complete batch ', element)
})
})
})
Promise.all(promises).then(() => {
res.send({ status: 200, message: 'Game: added.', pin: newGID })
console.log('completed');
});
// res.send({ status: 200, message: 'Game: added.', pin: newGID})
// console.log('completed');
} else {
console.log('found a match ', snapshot)
// re-run the generator for a new pin
generatePin()
}
})
}
SECOND UPDATE: (based off Keith's answer)
function addGame(newGidToString) {
var getGame = admin.firestore().collection('games')
getGame = getGame.where('gid', '==', newGidToString)
getGame.get().then(snapshot => {
if (snapshot.empty) {
console.log('BODY ', req.body)
const gameTitle = req.body.gametitle
const targetGID = req.body.target
const numPlayers = req.body.playercount.toString()
const newGID = newGidToString
const promises = [];
const collections = ['games', 'clues', 'detours', 'messages', 'questions', 'tagstate']
collections.forEach(function (element) {
console.log('start loop', element)
let elementsRef = admin.firestore().collection(element)
elementsRef = elementsRef.where('gid', '==', targetGID)
const getPromise = elementsRef.get(); //lets get our promise
promises.push(getPromise); //place into an array
getPromise.then(snapshot => { //we can now continue as before
var batch = admin.firestore().batch()
snapshot.forEach(function (doc) {
// change the gid to the new game gid
let data = doc.data()
data.gid = newGID
if(data.template){
data.template = false
}
if(!data.parent) {
data.parent = targetGID
}
if (!data.playercount) {
data.playercount = numPlayers
}
if (data.gametitle) {
data.gametitle = gameTitle
}
let elementRef = admin.firestore().collection(element).doc()
batch.set(elementRef, data)
})
batch.commit()
})
})
Promise.all(promises).then(() => {
res.send({ status: 200, message: 'Game: added.', pin: newGID })
console.log('completed from promise');
});
} else {
console.log('found a match ', snapshot)
// re-run the generator for a new pin
generatePin()
}
})
}
When looping with promises, one thing to remember is if you do a forEach, it's not going to wait.
It appears that what the OP wants to do is before he hits the complete, and does a res send, wants to make sure all promises have been completed.
Promise.all https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all , is exactly what this does.
So every time you create a promise if you push this onto an array, we can later use Promise.all to make sure all the promises are complete.
So if we replace
elementsRef.get().then(snapshot => {
with
const promGet = elementsRef.get(); //lets get our promise
promises.push(promGet); //place into an array
promGet.then(snapshot => { //we can now continue as before
Your promise is now stored for later so that Promise.all can check to make sure everything has complete. You could do this in one line, but I feel it's easier to see what's happening this way. eg->
promises.push(promGet.then(snapshot => {
Finally now we have got our promise array, before we do our res.send, lets make sure they have all completed.
Promise.all(promises).then(() => {
res.send({ status: 200, message: 'Game: added.', pin: newGID})
console.log('completed');
});
One thing to keep in mind when using Promises this way, is that you have a fair few things going on at the same time, sometimes this can be an issue, eg. connected to some service might put restrictions on the number of concurrent connections etc. So in these cases it might be an idea doing things in series or a mix of both. There are even libs that can do concurrecy, eg. Do only X amounts of promises at a time.
And finally, if you can use async / await, I'd advice using them, as it makes Promises so much nicer to work with.
The firebase function I'm currently using retrieves data from a certain branch in the database where the value may or may not have percent encoding. The value is a user's username and it's encoded if there's a '.' in the name. When the user gets a notification, it has their name in the body of it, and I'm trying to figure out how to removePercentEncoding if necessary. My cloud function:
exports.newPost = functions.database.ref('/{school}/posts').onWrite((change, context) => {
const school = context.params.school
const postUsername = admin.database().ref('/{school}/lastPost/lastPostUser').once('value')
var db = admin.database();
var val1, val2;
db.ref(`/Herrick Middle School/lastPost/lastPostUser`).once('value').then(snap => {
val1 = snap.val();
console.log(snap.val());
return val1
}).then(() => {
return db.ref("test2/val").once('value');
}).catch(err => {
console.log(err);
});
return loadUsers().then(users => {
let tokens = [];
for (let user of users) {
tokens.push(user.pushToken);
console.log(`pushToken: ${user.pushToken}`);
}
let payload = {
notification: {
title: school,
body: `${val1} just posted something.`,
sound: 'Apex',
badge: '1'
}
};
return admin.messaging().sendToDevice(tokens, payload);
});
});
function loadUsers() {
let dbRef = admin.database().ref('/Herrick Middle School/regisTokens');
let defer = new Promise((resolve, reject) => {
dbRef.once('value', (snap) => {
let data = snap.val();
let users = [];
for (var property in data) {
users.push(data[property]);
console.log(`data: ${property}`);
}
resolve(users);
}, (err) => {
reject(err);
});
});
return defer;
}
More specifically, I was hoping someone could shed some light on how to remove encoding from
val
Thanks in advance.
not sure i understand but either native JS decodeURI() or regex like this
var encoded = "john%doe%doe%bird";
console.log(encoded.replace(/%/g, "."));
As shown bellow, I am pushing the object link_to_json returns into an array allShirts declared in html_to_json.
However, the console.dir on the third last line and the return value of html_to_json logs an array of undefined references. Which I presume is because console.dir and return is executed before link_to_json functions finished.
How do I ensure the return value of html_to_json is a filled up allShirts array?
//Go to individual links and scrape relevant info
const link_to_json = (link) => {
request(link, (err, res, body) => {
if (!error_handler(err, res, link)) {
const $ = cheerio.load(body);
const shirt_detail = $('.shirt-details').find('h1').text();
const Title = shirt_detail.substr(shirt_detail.indexOf(' ') + 1);
const Price = shirt_detail.substr(0, shirt_detail.indexOf(' '));
const ImageURL = $('.shirt-picture').find('img').attr('src');
const URL = link;
return new Shirt(Title, Price, ImageURL, URL);
} else return {};
});
}
//Crawl through all individual links listed in Root
const html_to_json = body => {
const allShirts = [];
const $ = cheerio.load(body);
$('.products').find('a').each((index, val) => {
allShirts.push(link_to_json(rootURL + $(val).attr('href')));
});
console.dir(allShirts); // <--- HERE
return allShirts;
}
There's a few ways to go after this, but I like the Async library for this sort of thing.
How I'd handle your problem is to actually get all the URLs first, so changing your body scrape to something like this instead:
const shirtLinks = [];
$('.products').find('a').each((index, val) => {
shirtLinks.push(rootURL + $(val).attr('href'));
});
You need your conversion function to be asynchronous as well:
const linkToJSON = (link, cb) => {
request(link, (err, res, body) => {
if (!error_handler(err, res, link)) {
const $ = cheerio.load(body);
const shirt_detail = $('.shirt-details').find('h1').text();
const Title = shirt_detail.substr(shirt_detail.indexOf(' ') + 1);
const Price = shirt_detail.substr(0, shirt_detail.indexOf(' '));
const ImageURL = $('.shirt-picture').find('img').attr('src');
const URL = link;
return cb(null, new Shirt(Title, Price, ImageURL, URL));
}
return cb();
});
}
Then use async to map them across the async function that fetches the data:
async.map(shirtLinks, linkToJSON, (err, results) => {
console.dir(results);
});
This is how I would do it. I find it easier to debug this way.
let getShirtDetailsBody = (link) => {
return new Promise((resolve, reject) => {
request(link, (err, res, body) => {
if (err) {
reject(err)
} else {
resolve(body)
}
})
})
}
let getShirt = (body) => {
const $ = cheerio.load(body);
const shirt_detail = $('.shirt-details').find('h1').text();
const Title = shirt_detail.substr(shirt_detail.indexOf(' ') + 1)
const Price = shirt_detail.substr(0, shirt_detail.indexOf(' '))
const ImageURL = $('.shirt-picture').find('img').attr('src')
const URL = link
return new Shirt(Title, Price, ImageURL, URL)
}
let getAllProductsShirtsBody = (body) => {
const $ = cheerio.load(body)
return Promise.all($('.products').find('a').map((index, val) => {
return getShirtDetailsBody(`rootURL${$(val).attr('href')}`)
}))
}
getAllProductsShirtsBody(yourbody).then(allShirtsBody => {
const allShirts = allShirtsBody.map(shirtBody => { return getShirt(shirtBody) })
console.log(allShirts)
}).catch(err => { console.log(err) })