I have this firebase function which is properly deployed for testing on Firebase :
exports.testDataPoint = functions.database.ref('/testDataPoint/{uid}/{id}/')
.onCreate(event => {
if (event.data.exists()) {
return admin.database().ref("/test/"+event.params.uid+"/accumulate")
.transaction(current => {
return (current || 0) + event.data.val();
})
}
else{
return Promise.resolve(true)
}
});
When I try to write a data of 10000 entries at once the function is not triggered at all. But if the number of entries is around 1000, function triggers perfectly.
Here is the script I'm using to write data:
function testGroupWrites() {
let users = {};
for (let i = 0; i < 1000; i++) {
let id = shortId.generate();
let jobs = {};
for (let j = 0; j < 10; j++) {
let uid = shortId.generate();
console.log(uid);
jobs[uid] = 1;
}
users[id] = jobs;
}
db.ref("testDataPoint")
.set(users)
.then(() => {
console.log("written");
})
}
Is there any limit to how much data can be written at a point above which function will not gets triggered? I'm testing this scenario, because in my project firebase function will be doing the same thing to calculate values.
I'm using Admin SDK in my script to write dummy data.
Yes there is a limit, please check the below:
So the maximum is 1000 entries, more info here:
https://firebase.google.com/docs/database/usage/limits
Related
I wrote the following code in Node.js:
const axios = require("axios");
const prompt = require("prompt-sync")();
let numRepo = prompt("Welcome, How many repositories to search? ");
numRepo = parseInt(numRepo);
while (!Number.isInteger(numRepo))
numRepo = parseInt(prompt("Please insert integer: "));
let n;
if (numRepo < 100) n = 1;
else n = numRepo % 100;
axios
.get(
`https://api.github.com/search/repositories?q=language:js&sort=stars&order=desc&per_page=${numRepo}&page=${n}`
)
.then((response) => {
let repo = [];
response.data.items.forEach((e) => repo.push(e));
console.log(repo);
})
.catch((error) => {
console.log(error);
});
I used the GitHub Search APi, and my goal is to write a function whose purpose is to check for each repository how much unused packaged it has.
How can this be done?
I've been coding a Poll Command for my Discord Bot. It is in Discord.JS but when I am going to run the command, it does this error:
I've been trying to fix this issue for a while and it still does this issue. I've changed some lines of code, particularly line 65 and line 70-80.
Code:
const options = [
'🇦',
'🇧',
'🇨',
'🇩',
'🇪',
'🇫',
'🇬',
'🇭',
'🇮',
'🇯',
'🇰',
'🇱',
'🇲',
'🇳',
'🇴',
'🇵',
'🇶',
'🇷',
'🇸',
'🇹',
'🇺',
'🇻',
'🇼',
'🇽',
'🇾',
'🇿',
];
const pollLog = {};
function canSendPoll(user_id) {
if (pollLog[user_id]) {
const timeSince = Date.now() - pollLog[user_id].lastPoll;
if (timeSince < 1) {
return false;
}
}
return true;
}
exports.run = async (client, message, args, level, Discord) => {
if (args) {
if (!canSendPoll(message.author.id)) {
return message
.channel
.send(`${message.author} please wait before sending another poll.`);
} else if (args.length === 1) { // yes no unsure question
const question = args[0];
pollLog[message.author.id] = {
lastPoll: Date.now()
};
return message
.channel
.send(`${message.author} asks: ${question}`)
.then(async (pollMessage) => {
await pollMessage.react('👍');
await pollMessage.react('👎');
await pollMessage.react(message.guild.emojis.get('475747395754393622'));
});
} else { // multiple choice
args = args.map(a => a.replace(/"/g, ''));
const question = args[0];
const questionOptions = message.content.match(/"(.+?)"/g);
if (questionOptions.length > 20) {
return message.channel.send(`${message.author} Polls are limited to 20 options.`);
} else {
pollLog[message.author.id] = {
lastPoll: Date.now()
};
return message
.channel
.send(`${message.author} asks: ${question}
${questionOptions
.map((option, i) => `${options[i]} - ${option}`).join('\n')}
`)
.then(async (pollMessage) => {
for (let i = 0; i < questionOptions.length; i++) {
await pollMessage.react(options[i]);
}
});
}
}
} else {
return message.channel.send(`**Poll |** ${message.author} invalid Poll! Question and options should be wrapped in double quotes.`);
}
}
The reason some of the question is listed as choices is because you define question as args[0], which is simply the first word given. You can solve this by looping through the arguments and adding those that don't appear to be a choice into the question. See the sample code below.
const args = message.content.trim().split(/ +/g);
// Defining the question...
let question = [];
for (let i = 1; i < args.length; i++) {
if (args[i].startsWith('"')) break;
else question.push(args[i]);
}
question = question.join(' ');
// Defining the choices...
const choices = [];
const regex = /(["'])((?:\\\1|\1\1|(?!\1).)*)\1/g;
let match;
while (match = regex.exec(args.join(' '))) choices.push(match[2]);
// Creating and sending embed...
let content = [];
for (let i = 0; i < choices.length; i++) content.push(`${options[i]} ${choices[i]}`);
content = content.join('\n');
var embed = new Discord.RichEmbed()
.setColor('#8CD7FF')
.setTitle(`**${question}**`)
.setDescription(content);
message.channel.send(`:bar_chart: ${message.author} started a poll.`, embed)
.then(async m => {
for (let i = 0; i < choices.length; i++) await m.react(options[i]);
});
The Regex used is from this answer (explanation included). It removes the surrounding quotation marks, allows escaped quotes, and more, but requires a solution like this to access the desired capturing group.
Note that you'll still have to check whether there's a question and if choices exist, and display any errors as you wish.
I need to read a grid and take that data and call a $getJSON url. The grid could have over 100 lines of data. The getJSON returns a list of comma separated values that I add to an array. Once the loop is finished I take the array and process it for the duplicates. I need to use the duplicates in another process. I know that I can't determine the order of the data that is coming back but I need to know that all of the calls have been make.
for (let i = 0; i < rowscount; i++){
$.getJSON(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" +
terms,
function (data) {
var pmids = data.esearchresult.idlist;
var pmidlist = pmids.join();
pmid_List.push(pmidlist);
if (i == rowscount - 1) {
// call the related function
}
});
}
I can't figure out how to be sure that the process has finished. The call to the related function has been done early at times.
Well if we keep track of how many have completed we can fire off the code when the last one is done.
let complete = 0;
for (let i = 0; i < rowscount; i++){
$.getJSON(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" +
terms,
function (data) {
var pmids = data.esearchresult.idlist;
var pmidlist = pmids.join();
pmid_List.push(pmidlist);
complete += 1;
if (complete == rowscount) {
// call the related function
}
});
}
I'd use fetch and Promise.all
const link = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=";
Promise.all(Array.from({
length: 3
}, () => fetch(link + 'foo').then(e => e.json()))).then(e => {
//called when all requests are done
console.log(e);
})
Try this
function getJson(url, i) {
return $.getJSON(url, function (data) {
//var pmids = data.esearchresult.idlist;
//var pmidlist = pmids.join();
//pmid_List.push(pmidlist);
console.log('completed', i)
return data;
});
}
function run() {
let promises = []
for (let i = 0; i < rowscount; i++) {
const terms = 'foot';
const url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" + terms;
promises.push(getJson(url, i));
}
return promises;
}
Promise.all(run()).then(() => console.log('All are completed'));
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
Im using bluebirds Promise.map() method to run 100,000 firebase queries as shown below and the function takes about 10 seconds to run.. If I set the concurrency higher than 1000 then I receive the error
Maximum call stack size exceeded
Any ideas on how to fix this and also how to speed this up. It seems to me that perhaps Promise.map() may not be the right function to use or maybe I am mismanaging the memory some how. Any ideas thank you.
exports.postMadeByFriend = functions.https.onCall(async (data, context) => {
const mainUserID = "hJwyTHpoxuMmcJvyR6ULbiVkqzH3";
const follwerID = "Rr3ePJc41CTytOB18puGl4LRN1R2"
const otherUserID = "q2f7RFwZFoMRjsvxx8k5ryNY3Pk2"
var refs = [];
for (var x = 0; x < 100000; x += 1) {
if (x === 999) {
const ref = admin.database().ref(`Followers`).child(mainUserID).child(follwerID)
refs.push(ref);
continue;
}
const ref = admin.database().ref(`Followers`).child(mainUserID).child(otherUserID);
refs.push(ref);
}
await Promise.map(refs, (ref) => {
return ref.once('value')
}, {
concurrency: 10000
}).then((val) => {
console.log("Something happened: " + JSON.stringify(val));
return val;
}).catch((error) => {
console.log("an error occured: " + error);
return error;
})
Edits
const runtimeOpts = {
timeoutSeconds: 300,
memory: '2GB'
}
exports.postMadeByFriend = functions.runWith(runtimeOpts).https.onCall(async (data, context) => {
const mainUserID = "hJwyTHpoxuMmcJvyR6ULbiVkqzH3";
const follwerID = "Rr3ePJc41CTytOB18puGl4LRN1R2"
const otherUserID = "q2f7RFwZFoMRjsvxx8k5ryNY3Pk2"
var refs = [];
for (var x = 0; x < 100000; x += 1) {
if (x === 999) {
const ref = admin.database().ref(`Followers`).child(mainUserID).child(follwerID)
refs.push(ref);
continue;
}
const ref = admin.database().ref(`Followers`).child(mainUserID).child(otherUserID);
refs.push(ref);
}
await Promise.map(refs, (ref) => {
return ref.once('value')
}, {
concurrency: 10000
}).then((val) => {
console.log("Something happened: " + JSON.stringify(val));
return val;
}).catch((error) => {
console.log("an error occured: " + error);
return error;
})
Update:
If the goal is to have a number of friends posts the better way to do it would be to have a cloud function that increments a counter on every new post saved to the DB. That way you can get the number of posts with no calculation needed.
Here is the similar answer, and a code sample with the like counter
Original answer:
You could try to increase the memory allocated to your Cloud Funciton:
In the Google Cloud Platform Console, select Cloud Functions from the left menu.
Select a function by clicking on its name in the functions list.
Click the Edit icon in the top menu.
Select a memory allocation from the drop-down menu labeled Memory allocated.
Click Save to update the function.
As described in the Manage functions deployment page
First I'm not sure that there is a real problem but I guess I'll share my reasoning.
I use Firebase as a database / backend for the archiving of all the data from various sensors at home and an UI with cool graphs in hosting. So every 10 minutes I push various data (temperature, humidity, CO2 level, illumination, ...) coming from various rooms. I have almost 3 years of data available (so my base has a lots of nodes)
So my database structure is like that :
root
readings
room_id
GUID
time
temp
hum
lum
For a few years I had a PHP script hosted at home that checked if the latest item inside each readings/room_id has a time value that is not too old (no more than 11 minutes old). I translated it to Firebase cloud function some days ago and I got something like this :
exports.monitor = functions.https.onRequest((req, res) => {
const tstamp = Math.floor(Date.now() / 1000);
var sensors = ["r01", "r02", "r03", "r04", "r05"];
var promiseArray = [];
var result = {};
for (var i = 0; i < sensors.length; i++) {
console.log('Adding promise for ' + sensors[i]);
promiseArray.push(admin.database().ref('/readings/' + sensors[i]).limitToLast(1).once("child_added"));
}
Promise.all(promiseArray).then(snapshots => {
console.log('All promises done : ' + snapshots.length);
res.set('Cache-Control', 'private, max-age=300');
for (var i = 0; i < snapshots.length; i++) {
differenceInMinutes = (tstamp - snapshots[i].val().time) / 60;
result[sensors[i]] = {current: tstamp,
sensor: snapshots[i].val().time,
diff: Math.round(differenceInMinutes * 10) / 10};
if (differenceInMinutes < 11) {
result[sensors[i]]['status'] = "OK";
} else {
result[sensors[i]]['status'] = "KO";
}
}
return res.status(200).json(result);
}).catch(error => {
console.error('Error while getting sensors details', error.message);
res.sendStatus(500);
});
});
The code works well. So my question is : if I add another room ID in the sensors array that does not exists inside "readings" in my database, I thought I'll get an error (failed promise) instead I only got a huge timeout error, I don't want that kind of timeout on Firebase Cloud Functions (to avoid any unwanted cost).
Is that normal ? Is my code wrong ? Do I have to start by getting a shallow snapshot of "readings/room_id" check that it exists and check if has children ?
Thanks a lot for your help.
EDIT : With the help of Frank I fixed my code, here is the revised version :
exports.monitor = functions.https.onRequest((req, res) => {
const tstamp = Math.floor(Date.now() / 1000);
var sensors = ["r01", "r02", "r03", "r04", "r05"];
var promiseArray = [];
var result = {};
for (var i = 0; i < sensors.length; i++) {
console.log('Adding promise for ' + sensors[i]);
promiseArray.push(admin.database().ref('/readings/' + sensors[i]).limitToLast(1).once("value"));
}
Promise.all(promiseArray).then(queryResults => {
console.log('All promises done : ' + queryResults.length);
res.set('Cache-Control', 'private, max-age=300');
queryResults.forEach((snapshots, i) => {
snapshots.forEach((snapshot) => {
var currentData = snapshot.val();
differenceInMinutes = (tstamp - currentData.time) / 60;
result[sensors[i]] = {current: tstamp,
sensor: currentData.time,
diff: Math.round(differenceInMinutes * 10) / 10};
if (differenceInMinutes < 11) {
result[sensors[i]]['status'] = "OK";
} else {
result[sensors[i]]['status'] = "KO";
}
});
});
return res.status(200).json(result);
}).catch(error => {
console.error('Error while getting sensors details', error.message);
res.sendStatus(500);
});
});
a child_added event only fires when there is a child node. If there are not child nodes under the location (or matching the query) it will not fire.
To ensure you also get notified in the condition there are no children, you should listen to the value event:
for (var i = 0; i < sensors.length; i++) {
console.log('Adding promise for ' + sensors[i]);
var query = admin.database().ref('/readings/' + sensors[i]).limitToLast(1).once("value")
promiseArray.push(query);
}
Since a value event may match multiple children in a single snapshot (despite your query only requesting a single child), you will need to loop over the children of the resulting snapshot:
Promise.all(promiseArray).then((queryResults) => {
console.log('All promises done : ' + queryResults.length);
res.set('Cache-Control', 'private, max-age=300');
queryResults.forEach((snapshots) => {
snapshots.forEach((snapshot) => {
differenceInMinutes = (tstamp - snapshot.val().time) / 60;
...