We Keep Coding

Sending res to client before finishing async processes in Node.js

I have the following code that I can´t figure out how to make this entire process finish before sending a response back to the client. It has a for loop and make the requests to 2 external functions.
The problem that I can see in the console is the response which goes way faster than the process takes to complete the processing. It makes the client believe everything is fine, but, actually some error might has happened. I´ve tried to await, I´ve tried reading other posts, I´ve tried making this to return a promise and return a resolve, I´ve tried an index to check the array length... If someone could help me out, I would appreciate that. Thanks in advance.
var updateProdCharTree = async (req, res, next) => {
return new Promise((resolve, reject) => {
var data = req.body.raw.data;
var productLine = req.body.raw.productLine;
var bu = req.body.raw.bu;
let contErros = null;
let contAcertos = 0;
var maxId = 0;
let queryMaxId = `SELECT max(ProductCharFatherId) as maxId FROM maxiplastmatriz.productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database.query(queryMaxId)
.then(resultqueryMaxId => {
if(resultqueryMaxId.length){
maxId = resultqueryMaxId[0]['maxId'];
}else{
maxId = 0;
}
let queryAllNodes = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database.query(queryAllNodes)
.then( async resultqueryAllNodes => {
for (let index = 0; index < data.children.length; index++) {
const element = data.children[index];
if (data.children[index].dbId != undefined) {
let query = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine} and Id=${data.children[index].dbId} and ProductCharFatherId=${data.children[index].id}`;
database.query(query)
.then( async result => {
if (result.length) { // Update char in productchar1
maxId++;
var params = {element: element, productLine: productLine, bu: bu, index: index};
waitResUp = await updateProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
contAcertos++;
} else { // Add char in productchar1 shouldn´t get in here
console.log("Erro em: updateProdCharTree > addProductProductChar1");
}
})
.catch(err => {
console.log("Erro em query");
contErros = 1;
})
}else{ // Add new char in productchar1
maxId++;
var params = {element: element, productLine: productLine, bu: bu, index: index, maxId: maxId};
waitRes = await addProductProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
console.log("waitRes", waitRes.insertId);
contAcertos++;
}
}
})
.catch(err => {
console.log("Erro em queryAllNodes", err);
contErros = 1;
})
})
.catch(err => {
console.log("Erro em queryMaxId");
contErros = 1;
});
if (contAcertos == data.children.length) {
resolve("result"); // ***RES ATTEMPT***
}
})
}
Beginner here. Please be pacient. You were once either.

For starters, you can refactor your code to something like below so that response is returned after all async functions within loop are executed:
var updateProdCharTree = async (req, res, next) => {
return new Promise((resolve, reject) => {
var data = req.body.raw.data;
var productLine = req.body.raw.productLine;
var bu = req.body.raw.bu;
let contErros = null;
let contAcertos = 0;
var maxId = 0;
let queryMaxId = `SELECT max(ProductCharFatherId) as maxId FROM maxiplastmatriz.productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database
.query(queryMaxId)
.then((resultqueryMaxId) => {
if (resultqueryMaxId.length) {
maxId = resultqueryMaxId[0]["maxId"];
} else {
maxId = 0;
}
let queryAllNodes = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database
.query(queryAllNodes)
.then(async (resultqueryAllNodes) => {
return Promise.all(
data.chilren.map(async (element) => {
if (data.children[index].dbId != undefined) {
let query = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine} and Id=${data.children[index].dbId} and ProductCharFatherId=${data.children[index].id}`;
let result = await database.query(query);
if (result.length) {
// Update char in productchar1
maxId++;
var params = {
element: element,
productLine: productLine,
bu: bu,
index: index,
};
waitResUp = await updateProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
contAcertos++;
} else {
// Add char in productchar1 shouldn´t get in here
console.log(
"Erro em: updateProdCharTree > addProductProductChar1"
);
}
} else {
// Add new char in productchar1
maxId++;
var params = {
element: element,
productLine: productLine,
bu: bu,
index: index,
maxId: maxId,
};
waitRes = await addProductProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
console.log("waitRes", waitRes.insertId);
contAcertos++;
}
})
);
})
.then(() => {
// This will be called after all queries are executed
resolve("result");
})
.catch((err) => {
console.log("Erro em queryAllNodes", err);
contErros = 1;
// NOTE: Ideally we should be rejecting promise in case of error
});
})
.catch((err) => {
console.log("Erro em queryMaxId");
contErros = 1;
// NOTE: Ideally we should be rejecting promise in case of error
});
});
};
As #Jeremy Thille suggested this code can be improvised further to only use async-await instead of Promises with then callback so that it is more cleaner and easy to understand.

Stripe functions returning calls after function finished

Relatively new to Javascript however, i'm trying to work with Stripe and my way around a user submitting another payment method and then paying an invoice with that method. if the payment fails again - it should remove the subscription alltogether. I'm using Firebase Realtime Database with GCF & Node.js 8.
Here is what i have so far
exports.onSecondPaymentAttempt = functions.database.ref("users/{userId}/something/somethingHistory/{subDbId}/newPayment").onCreate((snapshot, context)=>{
var s = snapshot.val();
var fields = s.split(",");
const cardToken = fields[0];
const cus_id = fields[1];
const conn_id = fields[2];
const subDbId = context.params.subDbId;
const userId = context.params.userId;
return stripe.customers.createSource(
cus_id,
{source: cardToken},{
stripeAccount: `${conn_id}`,
},
(err, card)=> {
console.log(err);
if(err){
return console.log("error attaching card "+ err)
}else{
const invoiceNo = admin.database().ref(`users/${userId}/something/somethingHistory/${subDbId}`)
return invoiceNo.once('value').then(snapshot=>{
const invoiceNumber = snapshot.child("invoiceId").val();
const subId = snapshot.child("subscriptionId").val();
return stripe.invoices.pay(
invoiceNumber,
{
expand: ['payment_intent','charge','subscription'],
},{
stripeAccount: `${conn_id}`,
},
(err, invoice)=>{
if(err){
return console.log("error paying invoice "+ err)
}else{
if(invoice.payment_intent.status==="succeeded"){
//DO SOME CODE
return console.log("New Payment succeeded for "+invoiceNumber)
}else{
//DO SOME OTHER CODE
//CANCEL
return stripe.subscriptions.del(
subId,{
stripeAccount: `${conn_id}`,
},
(err, confirmation)=> {
if(err){
return console.log("Subscription error")
}else{
return console.log("Subscription cancelled")
}
});
}
}
});
})
}
});
To me it looks like an incredibly inefficient / ugly way of achieving the effect and overall the user is sitting waiting for a response for approx 15 seconds although the function finishes its execution after 1862ms - I still get responses up to 5 - 10 seconds after.
What's the most efficient way of achieving the same desired effect of registering a new payment source, paying subscription and then handling the result of that payment?

You should use the Promises returned by the Stripe asynchronous methods, as follows (untested, it probably needs some fine tuning, in particular with the objects passed to the Stripe methods):
exports.onSecondPaymentAttempt = functions.database.ref("users/{userId}/something/somethingHistory/{subDbId}/newPayment").onCreate((snapshot, context) => {
var s = snapshot.val();
var fields = s.split(",");
const cardToken = fields[0];
const cus_id = fields[1];
const conn_id = fields[2];
const subDbId = context.params.subDbId;
const userId = context.params.userId;
return stripe.customers.createSource(
//Format of this object to be confirmed....
cus_id,
{ source: cardToken },
{ stripeAccount: `${conn_id}` }
)
.then(card => {
const invoiceNo = admin.database().ref(`users/${userId}/something/somethingHistory/${subDbId}`)
return invoiceNo.once('value')
})
.then(snapshot => {
const invoiceNumber = snapshot.child("invoiceId").val();
const subId = snapshot.child("subscriptionId").val();
return stripe.invoices.pay(
invoiceNumber,
{ expand: ['payment_intent', 'charge', 'subscription'] },
{ stripeAccount: `${conn_id}` }
)
})
.then(invoice => {
if (invoice.payment_intent.status === "succeeded") {
//DO SOME CODE
console.log("New Payment succeeded for " + invoiceNumber)
return null;
} else {
//DO SOME OTHER CODE
//CANCEL
return stripe.subscriptions.del(
subId, {
stripeAccount: `${conn_id}`,
});
}
})
.catch(err => {
//....
return null;
})
});
I would suggest you watch the three videos about "JavaScript Promises" from the official Firebase video series, which explain why it is key to correctly chain and return the promises returned by the asynchronous methods.

Webpage is rendering before data is collected from firebase - NodeJS and EJS

I have tried using async-await, .then and now promise. I am quite new in javascript development.
The code
indexRouter.get('/dashboard', checkSignIn, async(request, response) => {
snapshot = await db.doc('users/accounts').get()
sites = snapshot.data()['sites']
const userId = request.session.userId
snapshot = await db.doc(`users/${userId}`).get()
var linkedSites = snapshot.data()['linked sites']
let getDs = new Promise((resolve,reject)=>{
console.log("1")
linkedSites.forEach((site) =>{
console.log("2")
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot)=>{
console.log("3")
snapshot.forEach((doc) => {
console.log("4")
console.log(doc.id)
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS.....
details.push({site:{'email': emailId, 'password': password, 'keys': keys}})
})
})
})
console.log("5")
console.log(details)
resolve(details)
}
);
getDs.then((details)=>{
console.log("rendering")
response.render('dashboard', {'details':details, 'linkedSites': linkedSites, 'sites': sites})
})
}
I am getting the response
1
2
2
5
[]
rendering
error: ...details not found in ejs
3
4
rsp121#gmail.com
3
4
test#gmail.com
According to the output, it seems like db.doc line after console.log(2) is getting executed after a certain time and resolve(details) is sent before.
Found a solution to the problem:
indexRouter.get('/dashboard', checkSignIn, async(request, response) => {
snapshot = await db.doc('users/accounts').get()
sites = snapshot.data()['sites']
const userId = request.session.userId
snapshot = await db.doc(`users/${userId}`).get()
var linkedSites = snapshot.data()['linked sites']
if(linkedSites){
const getSnapshot = (site) => {
return new Promise(resolve => {
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot) =>{
snapshot.forEach((doc) =>{
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS
details[site] = {'email': emailId, 'password': password, 'keys': keys}
resolve(true)
})
})
})
}
Promise.all(linkedSites.map(getSnapshot)).then(()=>{
console.log(linkedSites)
response.set('Cache-Control', 'no-store, no-cache, must-revalidate, private')
response.render('dashboard', {'details':details, 'linkedSites': linkedSites, 'sites': sites})
})
}

The problem is your promise resolved before the db.doc is resolved, and as your db.doc promise is inside a loop. So, you should be using the promise.all
The below code should work for you.
indexRouter.get("/dashboard", checkSignIn, async (request, response) => {
snapshot = await db.doc("users/accounts").get();
sites = snapshot.data()["sites"];
const userId = request.session.userId;
snapshot = await db.doc(`users/${userId}`).get();
var linkedSites = snapshot.data()["linked sites"];
let getDs = new Promise((resolve, reject) => {
const promises = [];
console.log("1");
linkedSites.forEach((site) => {
console.log("2");
promises.push(
new Promise((internalResolve) => {
db.doc(`users/${userId}`)
.collection(site)
.get()
.then((snapshot) => {
console.log("3");
snapshot.forEach((doc) => {
console.log("4");
console.log(doc.id);
emailId = doc.id;
keys = doc.data()["keys"];
var passwordEncrypt = doc.data()["password"];
password = cryptoJS;
details.push({
site: {
email: emailId,
password: password,
keys: keys,
},
});
internalResolve();
});
});
})
);
});
Promise.all(promises).then(() => {
console.log("5");
console.log(details);
resolve(details);
});
});
getDs.then((details) => {
console.log("rendering");
return response.render("dashboard", {
details: details,
linkedSites: linkedSites,
sites: sites,
});
});
});
More cleaner with async/await.
indexRouter.get("/dashboard", checkSignIn, async (request, response) => {
snapshot = await db.doc("users/accounts").get();
sites = snapshot.data()["sites"];
const userId = request.session.userId;
snapshot = await db.doc(`users/${userId}`).get();
var linkedSites = snapshot.data()["linked sites"];
console.log("1");
linkedSites.forEach(async (site) => {
console.log("2");
const snapshot = await db.doc(`users/${userId}`).collection(site).get();
console.log("3");
snapshot.forEach((doc) => {
console.log("4");
console.log(doc.id);
emailId = doc.id;
keys = doc.data()["keys"];
var passwordEncrypt = doc.data()["password"];
password = cryptoJS;
details.push({
site: {
email: emailId,
password: password,
keys: keys,
},
});
});
});
console.log("rendering");
return response.render("dashboard", {
details: details,
linkedSites: linkedSites,
sites: sites,
});
});

This is your code corrected, optimized and using last especifications of ECMA Script, the error is as the other comment said you are no waiting for the result of promises inside your "new Promise.." declaration. Dont foget to use try/catch inside async functions if you have not a top error handler.
The optimization is in firsts snapshot variables, this way you are getting data in parallel instead secuantially.
indexRouter.get('/dashboard', checkSignIn, async (request, response) => {
try {
let details = [];
const userId = request.session.userId
let [snapshot1, snapshot2] = await Promise.all([db.doc('users/accounts').get(), await db.doc(`users/${userId}`).get()])
let sites = snapshot1.data()['sites']
var linkedSites = snapshot2.data()['linked sites'];
await Promise.all(
linkedSites.map((site) =>
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot) => {
snapshot.forEach((doc) => {
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS
details.push({ site: { 'email': emailId, 'password': password, 'keys': keys } })
})
})
)
)
response.render('dashboard', { 'details': details, 'linkedSites': linkedSites, 'sites': sites })
} catch (e) {
//Render error you want
}})
linkedSites.map.... return an array of Promises that in the end are wrapped inside Promise.all and Promise.all wait until all promises are fullfilled or one of them is rejected in this last case your code goes to catch without reach the line response.render inside the try. You can avoid this catching locally the error of each promise in the map using
.then((snapshot) => {
...
}).catch(e=> { /*Do something with the rror*/})

Removing Percent Encoding from Firebase Cloud Functions

The firebase function I'm currently using retrieves data from a certain branch in the database where the value may or may not have percent encoding. The value is a user's username and it's encoded if there's a '.' in the name. When the user gets a notification, it has their name in the body of it, and I'm trying to figure out how to removePercentEncoding if necessary. My cloud function:
exports.newPost = functions.database.ref('/{school}/posts').onWrite((change, context) => {
const school = context.params.school
const postUsername = admin.database().ref('/{school}/lastPost/lastPostUser').once('value')
var db = admin.database();
var val1, val2;
db.ref(`/Herrick Middle School/lastPost/lastPostUser`).once('value').then(snap => {
val1 = snap.val();
console.log(snap.val());
return val1
}).then(() => {
return db.ref("test2/val").once('value');
}).catch(err => {
console.log(err);
});
return loadUsers().then(users => {
let tokens = [];
for (let user of users) {
tokens.push(user.pushToken);
console.log(`pushToken: ${user.pushToken}`);
}
let payload = {
notification: {
title: school,
body: `${val1} just posted something.`,
sound: 'Apex',
badge: '1'
}
};
return admin.messaging().sendToDevice(tokens, payload);
});
});
function loadUsers() {
let dbRef = admin.database().ref('/Herrick Middle School/regisTokens');
let defer = new Promise((resolve, reject) => {
dbRef.once('value', (snap) => {
let data = snap.val();
let users = [];
for (var property in data) {
users.push(data[property]);
console.log(`data: ${property}`);
}
resolve(users);
}, (err) => {
reject(err);
});
});
return defer;
}
More specifically, I was hoping someone could shed some light on how to remove encoding from
val
Thanks in advance.

not sure i understand but either native JS decodeURI() or regex like this
var encoded = "john%doe%doe%bird";
console.log(encoded.replace(/%/g, "."));

The right solution for promises

I have the app on node.js with connecting to firebase. I need to update the data correctly.
How to call the function getOrSetUserTrackDay(day) in a promise to get a good value, but not undefined?
let userData = [];
let userID = req.params.userID;
let today = req.params.today;
let yesterday = req.params.yesterday;
db.collection('users').doc(userID).get()
.then((userDataFromDB) => {
if (!userDataFromDB.exists) {
res.status(404).send('User not found');
}
else {
function getOrSetUserTrackDay(day) {
let userTrackRef = db.collection('history').doc('daily').collection(day).doc(userID);
userTrackRef.get()
.then(userTrackData => {
if (userTrackData.exists) {
return userTrackData.data(); // good
}
else {
let userNewData = {
username: userDataFromDB.data().username,
photoUrl: userDataFromDB.data().photoUrl
};
userTrackRef.update(userNewData);
return userNewData; // good
}
})
}
userData = {
user: userDataFromDB.data(),
today: getOrSetUserTrackDay(today), // undefined
yesterday: getOrSetUserTrackDay(yesterday) // undefined
};
res.json(userData);
}
})
.catch((err) => {
console.log(err);
res.status(404).send(err);
});

well getOrSetUserTrackDay has no return statement, hence it returns undefined - but, since it contains asynchronous code, you'll never be able to use it synchronously
So, you can do the following
let userData = [];
let userID = req.params.userID;
let today = req.params.today;
let yesterday = req.params.yesterday;
db.collection('users').doc(userID).get()
.then((userDataFromDB) => {
if (!userDataFromDB.exists) {
res.status(404).send('User not found');
}
else {
let getOrSetUserTrackDay = day => {
let userTrackRef = db.collection('history').doc('daily').collection(day).doc(userID);
return userTrackRef.get()
.then(userTrackData => {
if (userTrackData.exists) {
return userTrackData.data(); // good
} else {
let userNewData = {
username: userDataFromDB.data().username,
photoUrl: userDataFromDB.data().photoUrl
};
userTrackRef.update(userNewData);
return userNewData; // good
}
});
};
Promise.all([getOrSetUserTrackDay(today), getOrSetUserTrackDay(yesterday)])
.then(([today, yesterday]) => res.json({
user: userDataFromDB.data(),
today,
yesterday
}));
}
}).catch((err) => {
console.log(err);
res.status(404).send(err);
});
Note: changed getOrSetUserTrackDay from a function declaration to a function expression (in this case, an arrow function for no particular reason) - because Function declarations should not be placed in blocks. Use a function expression or move the statement to the top of the outer function.