I'm using two chained firebase requests in my code. the first one is fetching all the reviews, then I iterate trough all the results, firing a second firebase request for each element in loop.
After the loop, I've updated the state with the new data using setState. And that update is making my GUI transparent, like so:
the bottom part is randomly transparent each time, sometimes the render is visible a bit, sometimes not at all. When I remove the setState block, everything is fine.
The code in question:
getReviews() {
let reviewData = {}
let vendorId = this.props.item.vendor_id
let product = this.props.item.key
firebase.database().ref('u_products/' + vendorId + '/' + product + '/reviews/').once('value', (data) => {
reviewData = data.val()
if (!reviewData) {
this.setState({
loadedReviews: true
})
return
} else {
for (var key in reviewData) {
firebase.database().ref('users/' + reviewData[key].poster_uid + '/').once('value', (data) => {
let userData = data.val()
if (userData) {
this.getUserImageUrl()
...
}
})
}
this.state.reviewData = reviewData
this.setState({
dataSource: this.state.dataSource.cloneWithRows(reviewData),
loadedReviews: true,
})
}
})
}
Intended behaviour is first firebase request -> second one iterating for all the results from the first request ->setState.
Does anyone else have this issue?
firebase's .once() returns a Promise, so what you need to do is create an array of Promises, then call Promise.all(array).then((arrayOfData) => { ... });
You can now process the resulting array, then call .setState()
Here's a mockup:
/* firebase mockup */
function Result(url) {
this.url = url;
this.once = function(param, callback) {
return new Promise((resolve, reject) => {
setTimeout(() => {
var data = {
val: () => {
return this.url + " query result";
}
};
resolve(data);
}, Math.random() * 500 + 1000);
});
};
}
var firebase = {
database: function() {
return firebase;
},
ref: function(url) {
return new Result(url);
}
};
var reviewData = {
"test": {
poster_uid: "testData"
},
"2nd": {
poster_uid: "2ndData"
}
};
// ACTUAL IMPORTANT CODE BELOW #################
var arrayOfPromises = [];
for (var key in reviewData) {
arrayOfPromises.push(firebase.database().ref('users/' + reviewData[key].poster_uid + '/').once('value'));
}
Promise.all(arrayOfPromises).then(function(arrayOfResults) {
arrayOfResults.map((data, i) => {
let userData = data.val();
if (userData) {
console.log(i, userData);
}
});
console.log("setting state now");
});
Related
I am tring to download around 20.000 images from a website by scraping it.
(I am authorized by the owner)
The path of the image is composed like this:
VolumeId/ChapterId/PageId.jpg
There are around 100 volumes, and every volume has x chapters, and every chapter has y pages.
In the database I have stored for every volume the number of chapters, but i don't have
the number of pages, so I have to navigate to the image url and check if it exists.
I know for sure that there are less than 30 pages per chapter so I did something like this:
let exists = true;
for (let i = 0; i < 30 && exists; i++) {
fetch(`imgUrl/${i}.jpg`)
.then(data => {
if (data.ok)
return data.arrayBuffer();
else
exists = false;
.then(arrayBuffer => {
if (exists) {
let buffer = Buffer.from(arrayBuffer );
if (!fs.existsSync(path.join(__dirname, imgPath))) {
fs.mkdirSync(path.join(__dirname, imgPath), {
recursive: true,
});
}
fs.writeFile(
path.join(__dirname, imgPath + "/" + img + ".jpg"),
buffer,
(err) => {
if (err) throw err;
}
);
}
});
}
The problem:
The problem is that the loop does not wait for the image to be fetched and saved locally.
I have tried with async/await and promises (I think I have implemented them wrong)
Is there a better way to download a large quantity of data? Maybe with streams?
It can be a little bit complicated to implement your code with just async/await and at the same time assure the "exists" condition between iterations, I suggest you use a class that implements an async iterator, refer to the official documentation for more details, the following code achieve what you are looking for (note: the code snippet you provided didn't show where "imgPath" is coming from so just fix my code accordingly) :
class FetchImages {
constructor(urls) {
this.urls = urls;
this.index = 0;
}
[Symbol.asyncIterator]() {
const urlsIterator = this.urls[Symbol.iterator]();
return {
async next() {
if (++index == 30) {
return {
done: true
};
}
const iteratorResult = urlsIterator.next();
if (iteratorResult.done) {
return { done: true };
}
const url = iteratorResult.value;
try {
let response = await fetch(url);
let data;
if (response.status == 200) {
data = await response.arrayBuffer();
} else {
// equivalent to exists == false, exit condition of the iterator
return {
done: true
};
};
let buffer = Buffer.from(data);
if (!fs.existsSync(path.join(__dirname, imgPath))) {
fs.mkdirSync(path.join(__dirname, imgPath), {
recursive: true,
});
}
fs.writeFileSync(
path.join(__dirname, imgPath),
buffer,
);
return {
done: false,
value: imgPath
};
} catch (err) {
return {
done: false,
value: err.message
};
}
}
}
}
}
(async function () {
const fetchImages = new FetchImages(urls);
for await (const fetchImage of fetchImages) {
// check status of each fetch
console.log(fetchImage);
}
})();
I am trying to fetch data from different collections in my cloud Firestore database in advance before I process them and apply them to batch, I created two async functions, one to capture the data and another to execute certain code only after all data is collected, I didn't want the code executing and creating errors before the data is fetched when i try to access the matchesObject after the async function to collect data is finished, it keeps saying "it cannot access a property matchStatus of undefined", matchStatus is a field in my object, after a while, i see the data showing all the documents saved to matchObjects and the document i want is there, I logged the data being saved to the matches object, it retrieves all the data and I can confirm that the document I am looking for is fetched from firestore, it's like the code doesn't wait for the object to be finished before it runs, or something, i thought took care of that with async and await? could anyone shed some light as to why it is undefined one moment
axios.request(options).then(function(response) {
console.log('Total matches count :' + response.data.matches.length);
const data = response.data;
var matchesSnapshot;
var marketsSnapshot;
var tradesSnapshot;
var betsSnapshot;
matchesObject = {};
marketsObject = {};
tradesObject = {};
betsObject = {};
start();
async function checkDatabase() {
matchesSnapshot = await db.collection('matches').get();
matchesSnapshot.forEach(doc => {
matchesObject[doc.id] = doc.data();
console.log('matches object: ' + doc.id.toString())
});
marketsSnapshot = await db.collection('markets').get();
marketsSnapshot.forEach(doc2 => {
marketsObject[doc2.id] = doc2.data();
console.log('markets object: ' + doc2.id.toString())
});
tradesSnapshot = await db.collection('trades').get();
tradesSnapshot.forEach(doc3 => {
tradesObject[doc3.id] = doc3.data();
console.log('trades object: ' + doc3.id.toString())
});
betsSnapshot = await db.collection('bets').get();
betsSnapshot.forEach(doc4 => {
betsObject[doc4.id] = doc4.data();
console.log('bets object: ' + doc4.id.toString())
});
}
async function start() {
await checkDatabase();
// this is the part which is undefined, it keeps saying it cant access property matchStatus of undefined
console.log('here is matches object ' + matchesObject['302283']['matchStatus']);
if (Object.keys(matchesObject).length != 0) {
for (let bets of Object.keys(betsObject)) {
if (matchesObject[betsObject[bets]['tradeMatchId']]['matchStatus'] == 'IN_PLAY' && betsObject[bets]['matched'] == false) {
var sfRef = db.collection('users').doc(betsObject[bets]['user']);
batch11.set(sfRef, {
accountBalance: admin.firestore.FieldValue + parseFloat(betsObject[bets]['stake']),
}, {
merge: true
});
var sfRef = db.collection('bets').doc(bets);
batch12.set(sfRef, {
tradeCancelled: true,
}, {
merge: true
});
}
}
}
});
I think you missed on how firebase works. For fetch data use something like this
const getWithKey= async({
collection, value, orderBy = 'asc', where = 'id', operationString = '=='
}) => {
if (idControl(value)) {
const data = await db.collection(collection)
.where(where, operationString, value).get();
if (data?.docs[0]) {
return [data.docs[0].data()];
}
}
const documents = [];
await db
.collection(collection)
.orderBy(where, orderBy)
.get()
.then((item) => {
item.forEach((doc) => {
documents.push({ id: doc.id, ...doc.data() });
});
});
return documents || [];
}
Relatively new to Javascript however, i'm trying to work with Stripe and my way around a user submitting another payment method and then paying an invoice with that method. if the payment fails again - it should remove the subscription alltogether. I'm using Firebase Realtime Database with GCF & Node.js 8.
Here is what i have so far
exports.onSecondPaymentAttempt = functions.database.ref("users/{userId}/something/somethingHistory/{subDbId}/newPayment").onCreate((snapshot, context)=>{
var s = snapshot.val();
var fields = s.split(",");
const cardToken = fields[0];
const cus_id = fields[1];
const conn_id = fields[2];
const subDbId = context.params.subDbId;
const userId = context.params.userId;
return stripe.customers.createSource(
cus_id,
{source: cardToken},{
stripeAccount: `${conn_id}`,
},
(err, card)=> {
console.log(err);
if(err){
return console.log("error attaching card "+ err)
}else{
const invoiceNo = admin.database().ref(`users/${userId}/something/somethingHistory/${subDbId}`)
return invoiceNo.once('value').then(snapshot=>{
const invoiceNumber = snapshot.child("invoiceId").val();
const subId = snapshot.child("subscriptionId").val();
return stripe.invoices.pay(
invoiceNumber,
{
expand: ['payment_intent','charge','subscription'],
},{
stripeAccount: `${conn_id}`,
},
(err, invoice)=>{
if(err){
return console.log("error paying invoice "+ err)
}else{
if(invoice.payment_intent.status==="succeeded"){
//DO SOME CODE
return console.log("New Payment succeeded for "+invoiceNumber)
}else{
//DO SOME OTHER CODE
//CANCEL
return stripe.subscriptions.del(
subId,{
stripeAccount: `${conn_id}`,
},
(err, confirmation)=> {
if(err){
return console.log("Subscription error")
}else{
return console.log("Subscription cancelled")
}
});
}
}
});
})
}
});
To me it looks like an incredibly inefficient / ugly way of achieving the effect and overall the user is sitting waiting for a response for approx 15 seconds although the function finishes its execution after 1862ms - I still get responses up to 5 - 10 seconds after.
What's the most efficient way of achieving the same desired effect of registering a new payment source, paying subscription and then handling the result of that payment?
You should use the Promises returned by the Stripe asynchronous methods, as follows (untested, it probably needs some fine tuning, in particular with the objects passed to the Stripe methods):
exports.onSecondPaymentAttempt = functions.database.ref("users/{userId}/something/somethingHistory/{subDbId}/newPayment").onCreate((snapshot, context) => {
var s = snapshot.val();
var fields = s.split(",");
const cardToken = fields[0];
const cus_id = fields[1];
const conn_id = fields[2];
const subDbId = context.params.subDbId;
const userId = context.params.userId;
return stripe.customers.createSource(
//Format of this object to be confirmed....
cus_id,
{ source: cardToken },
{ stripeAccount: `${conn_id}` }
)
.then(card => {
const invoiceNo = admin.database().ref(`users/${userId}/something/somethingHistory/${subDbId}`)
return invoiceNo.once('value')
})
.then(snapshot => {
const invoiceNumber = snapshot.child("invoiceId").val();
const subId = snapshot.child("subscriptionId").val();
return stripe.invoices.pay(
invoiceNumber,
{ expand: ['payment_intent', 'charge', 'subscription'] },
{ stripeAccount: `${conn_id}` }
)
})
.then(invoice => {
if (invoice.payment_intent.status === "succeeded") {
//DO SOME CODE
console.log("New Payment succeeded for " + invoiceNumber)
return null;
} else {
//DO SOME OTHER CODE
//CANCEL
return stripe.subscriptions.del(
subId, {
stripeAccount: `${conn_id}`,
});
}
})
.catch(err => {
//....
return null;
})
});
I would suggest you watch the three videos about "JavaScript Promises" from the official Firebase video series, which explain why it is key to correctly chain and return the promises returned by the asynchronous methods.
I'm using an HTTP-triggered Firebase cloud function to make an HTTP request. I get back an array of results (events from Meetup.com), and I push each result to the Firebase realtime database. But for each result, I also need to make another HTTP request for one additional piece of information (the category of the group hosting the event) to fold into the data I'm pushing to the database for that event. Those nested requests cause the cloud function to crash with an error that I can't make sense of.
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require('request');
exports.foo = functions.https.onRequest(
(req, res) => {
var ref = admin.database().ref("/foo");
var options = {
url: "https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****",
json: true
};
return request(
options,
(error, response, body) => {
if (error) {
console.log(JSON.stringify(error));
return res.status(500).end();
}
if ("results" in body) {
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if ("name" in result &&
"description" in result &&
"group" in result &&
"urlname" in result.group
) {
var groupOptions = {
url: "https://api.meetup.com/" + result.group.urlname + "?sign=true&photo-host=public&key=****",
json: true
};
var categoryResult = request(
groupOptions,
(groupError, groupResponse, groupBody) => {
if (groupError) {
console.log(JSON.stringify(error));
return null;
}
if ("category" in groupBody &&
"name" in groupBody.category
) {
return groupBody.category.name;
}
return null;
}
);
if (categoryResult) {
var event = {
name: result.name,
description: result.description,
category: categoryResult
};
ref.push(event);
}
}
}
return res.status(200).send("processed events");
} else {
return res.status(500).end();
}
}
);
}
);
The function crashes, log says:
Error: Reference.push failed: first argument contains a function in property 'foo.category.domain._events.error' with contents = function (err) {
if (functionExecutionFinished) {
logDebug('Ignoring exception from a finished function');
} else {
functionExecutionFinished = true;
logAndSendError(err, res);
}
}
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1436:15)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
If I leave out the bit for getting the group category, the rest of the code works fine (just writing the name and description for each event to the database, no nested requests). So what's the right way to do this?
I suspect this issue is due to the callbacks. When you use firebase functions, the exported function should wait on everything to execute or return a promise that resolves once everything completes executing. In this case, the exported function will return before the rest of the execution completes.
Here's a start of something more promise based -
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require("request-promise-native");
exports.foo = functions.https.onRequest(async (req, res) => {
const ref = admin.database().ref("/foo");
try {
const reqEventOptions = {
url:
"https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=xxxxxx",
json: true
};
const bodyEventRequest = await request(reqEventOptions);
if (!bodyEventRequest.results) {
return res.status(200).end();
}
await Promise.all(
bodyEventRequest.results.map(async result => {
if (
result.name &&
result.description &&
result.group &&
result.group.urlname
) {
const event = {
name: result.name,
description: result.description
};
// get group information
const groupOptions = {
url:
"https://api.meetup.com/" +
result.group.urlname +
"?sign=true&photo-host=public&key=xxxxxx",
json: true
};
const categoryResultResponse = await request(groupOptions);
if (
categoryResultResponse.category &&
categoryResultResponse.category.name
) {
event.category = categoryResultResponse.category.name;
}
// save to the databse
return ref.push(event);
}
})
);
return res.status(200).send("processed events");
} catch (error) {
console.error(error.message);
}
});
A quick overview of the changes -
Use await and async calls to wait for things to complete vs. being triggered in a callback (async and await are generally much easier to read than promises with .then functions as the execution order is the order of the code)
Used request-promise-native which supports promises / await (i.e. the await means wait until the promise returns so we need something that returns a promise)
Used const and let vs. var for variables; this improves the scope of variables
Instead of doing checks like if(is good) { do good things } use a if(isbad) { return some error} do good thin. This makes the code easier to read and prevents lots of nested ifs where you don't know where they end
Use a Promise.all() so retrieving the categories for each event is done in parallel
There are two main changes you should implement in your code:
Since request does not return a promise you need to use an interface wrapper for request, like request-promise in order to correctly chain the different asynchronous events (See Doug's comment to your question)
Since you will then call several times (in parallel) the different endpoints with request-promise you need to use Promise.all() in order to wait all the promises resolve before sending back the response. This is also the case for the different calls to the Firebase push() method.
Therefore, modifying your code along the following lines should work.
I let you modifying it in such a way you get the values of name and description used to construct the event object. The order of the items in the results array is exactly the same than the one of the promises one. So you should be able, knowing that, to get the values of name and description within results.forEach(groupBody => {}) e.g. by saving these values in a global array.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
var rp = require('request-promise');
exports.foo = functions.https.onRequest((req, res) => {
var ref = admin.database().ref('/foo');
var options = {
url:
'https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****',
json: true
};
rp(options)
.then(body => {
if ('results' in body) {
const promises = [];
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if (
'name' in result &&
'description' in result &&
'group' in result &&
'urlname' in result.group
) {
var groupOptions = {
url:
'https://api.meetup.com/' +
result.group.urlname +
'?sign=true&photo-host=public&key=****',
json: true
};
promises.push(rp(groupOptions));
}
}
return Promise.all(promises);
} else {
throw new Error('err xxxx');
}
})
.then(results => {
const promises = [];
results.forEach(groupBody => {
if ('category' in groupBody && 'name' in groupBody.category) {
var event = {
name: '....',
description: '...',
category: groupBody.category.name
};
promises.push(ref.push(event));
} else {
throw new Error('err xxxx');
}
});
return Promise.all(promises);
})
.then(() => {
res.send('processed events');
})
.catch(error => {
res.status(500).send(error);
});
});
I made some changes and got it working with Node 8. I added this to my package.json:
"engines": {
"node": "8"
}
And this is what the code looks like now, based on R. Wright's answer and some Firebase cloud function sample code.
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require("request-promise-native");
exports.foo = functions.https.onRequest(
async (req, res) => {
var ref = admin.database().ref("/foo");
var options = {
url: "https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****",
json: true
};
await request(
options,
async (error, response, body) => {
if (error) {
console.error(JSON.stringify(error));
res.status(500).end();
} else if ("results" in body) {
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if ("name" in result &&
"description" in result &&
"group" in result &&
"urlname" in result.group
) {
var groupOptions = {
url: "https://api.meetup.com/" + result.group.urlname + "?sign=true&photo-host=public&key=****",
json: true
};
var groupBody = await request(groupOptions);
if ("category" in groupBody && "name" in groupBody.category) {
var event = {
name: result.name,
description: result.description,
category: groupBody.category.name
};
await ref.push(event);
}
}
}
res.status(200).send("processed events");
}
}
);
}
);
I need to change the value of the array from 'User' to 'Admin' if the function is clicked on and I have to code it in cloud code.
but there is a problem the array does not change
the following code is working but only the part with nameRoleQuery is not working and that's the part which I need to change.
promoteToAdmin: function promoteToAdmin(request, response) {
if (!request.params.companyUser) {
response.error('Request did not have an authenticated user attached with it');
} else {
var companyUser;
var companyUserQuery = new Parse.Query('CompanyUser');
companyUserQuery.include('user');
companyUserQuery.include('company');
companyUserQuery.get(request.params.companyUser, {
useMasterKey: true
}).then((giveRolename) => {
var nameRoleQuery = new Parse.Query(Parse.Role);
request.nameRoleQuery.set('user', ['Admin']);
return nameRoleQuery.save(null, {
useMasterKey: true
});
}).then((companyUserObject) => {
companyUser = companyUserObject;
var userRoleQuery = new Parse.Query(Parse.Role);
userRoleQuery.equalTo('name', 'Company-User-' + companyUser.get('company').id);
return userRoleQuery.first({
useMasterKey: true
});
}).then((userRole) => {
var usersInUserRole = userRole.relation('users');
usersInUserRole.remove(companyUser.get('user'));
return userRole.save(null, {
useMasterKey: true
});
}).then((userRoleSaveResult) => {
var adminRoleQuery = new Parse.Query(Parse.Role);
adminRoleQuery.equalTo('name', 'Company-Admin-' + companyUser.get('company').id);
return adminRoleQuery.first({
useMasterKey: true
});
}).then((adminRole) => {
var usersInAdminRole = adminRole.relation('users');
usersInAdminRole.add(companyUser.get('user'));
return adminRole.save(null,{
useMasterKey: true
});
}).then((saveResult) => {
console.log('after');
response.success('fissa is aan');
}, (error) => {
console.log(error);
});
console.log();
}
}
the role array needs to change.
Still unsure exactly what you're trying to do, but this is what I imagine you want to happen.
CompanyUser has a pointer called 'user' of type Parse.User. You want to update this Parse.User, based on your image, when you run this query. The following should work:
//AS ABOVE
.then((giveRolename) => {
var user = giveRoleName.get("user");
user.remove('role','user');
user.add('role','admin');
return user.save(null, {useMasterKey:true});
}).then((companyUserObject) => {
//companyUserObject is Parse.User object. If CompanyUser object is needed, store in variable beforehand.
I've put below what I would suggest as an improvement to your current code. It's less sequential and provides greater fallback in an error happens in the middle of the process.
It's without arrow functions, so you'll have to change accordingly.
function promoteToAdmin(request, response) {
var params = request.params;
var companyUserId = params["companyUser"];
var companyUser;
return Parse.Promise.as().then(
function()
{
if(!companyUser)
{
return Parse.Promise.error("Request did not have an user attached with it.");
}
var CompanyUser = Parse.Object.extend("CompanyUser");
var companyUserQuery = new Parse.Query(CompanyUser);
companyUserQuery.include("user");
companyUserQuery.include("company");
return companyUserQuery.get(companyUserId,{useMasterKey:true});
}
).then(
function(fetchedCompanyUser)
{
companyUser = fetchedCompanyUser;
var company = companyUser.get("company");
var userRoleQuery = new Parse.Query(Parse.Role);
userRoleQuery.equalTo('name', "Company-User-" + company.id )
var adminRoleQuery = new Parse.Query(Parse.Role);
adminRoleQuery.equalTo('name', "Company-Admin-" + company.id);
return Parse.Promise.when(
userRoleQuery.first({useMasterKey:true}),
adminRoleQuery.first({useMasterKey:true})
)
}
).then(
function(userRole,adminRole)
{
if(!userRole || !adminRole)
{
return Parse.Promise.error("No role found")
}
var user = companyUser.get("user");
user.remove("role","user");
user.add("role","admin");
userRole.getUsers().remove(user);
adminRole.getUsers().add(user);
return Parse.Promise.when(
user.save(null,{useMasterKey:true}),
userRole.save(null,{useMasterKey:true}),
adminRole.save(null,{useMasterKey:true})
)
}
).then(
function(user,userRole,adminRole)
{
response.success("success");
},
function(error)
{
response.error(error);
}
)
}