Im using bluebirds Promise.map() method to run 100,000 firebase queries as shown below and the function takes about 10 seconds to run.. If I set the concurrency higher than 1000 then I receive the error
Maximum call stack size exceeded
Any ideas on how to fix this and also how to speed this up. It seems to me that perhaps Promise.map() may not be the right function to use or maybe I am mismanaging the memory some how. Any ideas thank you.
exports.postMadeByFriend = functions.https.onCall(async (data, context) => {
const mainUserID = "hJwyTHpoxuMmcJvyR6ULbiVkqzH3";
const follwerID = "Rr3ePJc41CTytOB18puGl4LRN1R2"
const otherUserID = "q2f7RFwZFoMRjsvxx8k5ryNY3Pk2"
var refs = [];
for (var x = 0; x < 100000; x += 1) {
if (x === 999) {
const ref = admin.database().ref(`Followers`).child(mainUserID).child(follwerID)
refs.push(ref);
continue;
}
const ref = admin.database().ref(`Followers`).child(mainUserID).child(otherUserID);
refs.push(ref);
}
await Promise.map(refs, (ref) => {
return ref.once('value')
}, {
concurrency: 10000
}).then((val) => {
console.log("Something happened: " + JSON.stringify(val));
return val;
}).catch((error) => {
console.log("an error occured: " + error);
return error;
})
Edits
const runtimeOpts = {
timeoutSeconds: 300,
memory: '2GB'
}
exports.postMadeByFriend = functions.runWith(runtimeOpts).https.onCall(async (data, context) => {
const mainUserID = "hJwyTHpoxuMmcJvyR6ULbiVkqzH3";
const follwerID = "Rr3ePJc41CTytOB18puGl4LRN1R2"
const otherUserID = "q2f7RFwZFoMRjsvxx8k5ryNY3Pk2"
var refs = [];
for (var x = 0; x < 100000; x += 1) {
if (x === 999) {
const ref = admin.database().ref(`Followers`).child(mainUserID).child(follwerID)
refs.push(ref);
continue;
}
const ref = admin.database().ref(`Followers`).child(mainUserID).child(otherUserID);
refs.push(ref);
}
await Promise.map(refs, (ref) => {
return ref.once('value')
}, {
concurrency: 10000
}).then((val) => {
console.log("Something happened: " + JSON.stringify(val));
return val;
}).catch((error) => {
console.log("an error occured: " + error);
return error;
})
Update:
If the goal is to have a number of friends posts the better way to do it would be to have a cloud function that increments a counter on every new post saved to the DB. That way you can get the number of posts with no calculation needed.
Here is the similar answer, and a code sample with the like counter
Original answer:
You could try to increase the memory allocated to your Cloud Funciton:
In the Google Cloud Platform Console, select Cloud Functions from the left menu.
Select a function by clicking on its name in the functions list.
Click the Edit icon in the top menu.
Select a memory allocation from the drop-down menu labeled Memory allocated.
Click Save to update the function.
As described in the Manage functions deployment page
Stock Overflow -
I'm trying to process an image collection (~2000 images) with NodeJS. I'm able to extract the information needed, but I'm having a hard time getting the timing right so that I can save the outcome to a JSON file.
Towards the end you'll see
console.log(palette);
// Push single image data to output array.
output.push(palette);
The console.log works fine, but the the push method is appears to be executed after the empty output array has been written to data.json. Tried having a nested promise where I wouldn't write the file until all images have been processed.
The callback function in getPixels gets executed asynchronously.
The order of the output array is arbitrary.
Any and all help greatly appreciated! Thank you!
// Extract color information from all images in imageDirectory
var convert = require('color-convert'),
fs = require('fs'),
getPixels = require("get-pixels"),
startTime = Date.now();
var processedImages = new Promise((resolve, reject) => {
var imageDirectory = 'input',
images = fs.readdirSync(imageDirectory),
output = [];
console.log('Found ' + images.length + ' images.');
for (var image in images) {
var imageLoaded = new Promise((resolve, reject) => {
getPixels(imageDirectory + '/' + images[image], function(error, pixels) {
if(error) {
return 'Bad image path';
}
resolve(pixels);
});
});
imageLoaded.then((pixels) => {
var palette = {
coloredPixels : 0,
hues : [],
image : images[image],
classification : false,
pixelCount : null
};
palette.pixelCount = pixels.shape[0] *
pixels.shape[1] *
pixels.shape[2];
for (var i = 0; i < 256; i++) {
palette.hues[i] = 0;
}
for (var i = 0; i < palette.pixelCount; i += 4) {
var rgb = [pixels.data[i ],
pixels.data[i + 1],
pixels.data[i + 2]],
hsl = convert.rgb.hsl(rgb),
hue = hsl[0],
saturation = hsl[1];
if (saturation) {
palette.hues[hue]++;
palette.coloredPixels++;
}
}
console.log(palette);
// Push single image data to output array.
output.push(palette);
});
}
resolve(output);
});
processedImages.then((output) => {
// write output array to data.json
var json = JSON.stringify(output, null, 2);
fs.writeFileSync('data.json', json);
// Calculate time spent
var endTime = Date.now();
console.log('Finished in ' + (endTime - startTime) / 1000 + ' seconds.');
});
What you want to do is transform an array of "images" to an array of promises and wait for all promises to resolve, and then perform more transformations. Think of it as a series of transformations, because that's what you're doing here. In a nutshell:
const imagePromises = images.map(image => new Promise(resolve, reject) {
getPixels(imageDirectory + '/' + image, (error, pixels) => {
if(error) {
reject('Bad image path');
return;
}
resolve(pixels);
});
const output = Promise.all(imagePromises).then(results =>
results.map(pixels => {
return {
// do your crazy palette stuff (build a palette object)
};
});
First I'm not sure that there is a real problem but I guess I'll share my reasoning.
I use Firebase as a database / backend for the archiving of all the data from various sensors at home and an UI with cool graphs in hosting. So every 10 minutes I push various data (temperature, humidity, CO2 level, illumination, ...) coming from various rooms. I have almost 3 years of data available (so my base has a lots of nodes)
So my database structure is like that :
root
readings
room_id
GUID
time
temp
hum
lum
For a few years I had a PHP script hosted at home that checked if the latest item inside each readings/room_id has a time value that is not too old (no more than 11 minutes old). I translated it to Firebase cloud function some days ago and I got something like this :
exports.monitor = functions.https.onRequest((req, res) => {
const tstamp = Math.floor(Date.now() / 1000);
var sensors = ["r01", "r02", "r03", "r04", "r05"];
var promiseArray = [];
var result = {};
for (var i = 0; i < sensors.length; i++) {
console.log('Adding promise for ' + sensors[i]);
promiseArray.push(admin.database().ref('/readings/' + sensors[i]).limitToLast(1).once("child_added"));
}
Promise.all(promiseArray).then(snapshots => {
console.log('All promises done : ' + snapshots.length);
res.set('Cache-Control', 'private, max-age=300');
for (var i = 0; i < snapshots.length; i++) {
differenceInMinutes = (tstamp - snapshots[i].val().time) / 60;
result[sensors[i]] = {current: tstamp,
sensor: snapshots[i].val().time,
diff: Math.round(differenceInMinutes * 10) / 10};
if (differenceInMinutes < 11) {
result[sensors[i]]['status'] = "OK";
} else {
result[sensors[i]]['status'] = "KO";
}
}
return res.status(200).json(result);
}).catch(error => {
console.error('Error while getting sensors details', error.message);
res.sendStatus(500);
});
});
The code works well. So my question is : if I add another room ID in the sensors array that does not exists inside "readings" in my database, I thought I'll get an error (failed promise) instead I only got a huge timeout error, I don't want that kind of timeout on Firebase Cloud Functions (to avoid any unwanted cost).
Is that normal ? Is my code wrong ? Do I have to start by getting a shallow snapshot of "readings/room_id" check that it exists and check if has children ?
Thanks a lot for your help.
EDIT : With the help of Frank I fixed my code, here is the revised version :
exports.monitor = functions.https.onRequest((req, res) => {
const tstamp = Math.floor(Date.now() / 1000);
var sensors = ["r01", "r02", "r03", "r04", "r05"];
var promiseArray = [];
var result = {};
for (var i = 0; i < sensors.length; i++) {
console.log('Adding promise for ' + sensors[i]);
promiseArray.push(admin.database().ref('/readings/' + sensors[i]).limitToLast(1).once("value"));
}
Promise.all(promiseArray).then(queryResults => {
console.log('All promises done : ' + queryResults.length);
res.set('Cache-Control', 'private, max-age=300');
queryResults.forEach((snapshots, i) => {
snapshots.forEach((snapshot) => {
var currentData = snapshot.val();
differenceInMinutes = (tstamp - currentData.time) / 60;
result[sensors[i]] = {current: tstamp,
sensor: currentData.time,
diff: Math.round(differenceInMinutes * 10) / 10};
if (differenceInMinutes < 11) {
result[sensors[i]]['status'] = "OK";
} else {
result[sensors[i]]['status'] = "KO";
}
});
});
return res.status(200).json(result);
}).catch(error => {
console.error('Error while getting sensors details', error.message);
res.sendStatus(500);
});
});
a child_added event only fires when there is a child node. If there are not child nodes under the location (or matching the query) it will not fire.
To ensure you also get notified in the condition there are no children, you should listen to the value event:
for (var i = 0; i < sensors.length; i++) {
console.log('Adding promise for ' + sensors[i]);
var query = admin.database().ref('/readings/' + sensors[i]).limitToLast(1).once("value")
promiseArray.push(query);
}
Since a value event may match multiple children in a single snapshot (despite your query only requesting a single child), you will need to loop over the children of the resulting snapshot:
Promise.all(promiseArray).then((queryResults) => {
console.log('All promises done : ' + queryResults.length);
res.set('Cache-Control', 'private, max-age=300');
queryResults.forEach((snapshots) => {
snapshots.forEach((snapshot) => {
differenceInMinutes = (tstamp - snapshot.val().time) / 60;
...
ScreenShot: http://i.imgur.com/Shyihhb.png
I'm trying to make a code to return two arrays, but when the function calls the final part of code, the console gets flooded with dots (screenshot) and stop working, like a crash.
I'm using MongoJS driver for database connection.
Here's the part of my code:
lastcollect.find({from_id: usuario}, {limit: 10, sort: {time: -1}}, function(err, echoData) {
if (err || !echoData) {
console.log("No messages found");
} else {
var users = [];
var n = 0;
var max = echoData.length;
echoData.forEach(function(returnData) {
usercollect.findOne({idu: returnData.to_id}, {}, function(err, echoData) {
if (err || !echoData) {
console.log("Req.Info error", err);
n = n + 1;
} else {
n = n + 1;
var r = [];
r['name'] = echoData.name;
r['avatar'] = echoData.avatar;
r['phone'] = echoData.phone;
r['idu'] = echoData.idu;
users[echoData.idu] = r;
console.log('DB exec ' + n);
if (n === max) {
console.log('DB exec END');
var data = [];
data.data = echoData;
data.users = users;
console.log('users', users);
console.log('data', data);
io.sockets.emit('return lastusers ' + usuario, data);
}
}
});
});
}
});
I'm stuck at this for 4 hours, so if anyone can help i would appreciate.
Thanks!
The problem is your line
users[echoData.idu] = r;
You are setting the value at index echoData.idu to r. The problem is your array begins un-initialized, and I'm guessing echoData.idu is not a simple count. The ","s you are seeing are the empty spots in the array! Try this to see what I mean:
var testArray = [];
testArray[9] = "a";
console.log(testArray);
I would recommend saying instead
users.push(r);
or
users[n] = r; //because n is your counter.
If you don't care about the order, you could also speed up this call by using the excellent async npm libary, here: https://github.com/caolan/async
I have a piece of JavaScript code that I am executing using the node.js interpreter.
for(var i = 1; i < LIMIT; i++) {
var user = {
id: i,
name: "MongoUser [" + i + "]"
};
db.users.save(user, function(err, saved) {
if(err || !saved) {
console.log("Error");
} else {
console.log("Saved");
}
});
}
How can I measure the time taken by these database insert operations? I could compute the difference of date values after and before this piece of code but that would be incorrect because of the asynchronous nature of the code.
Use the Node.js console.time() and console.timeEnd():
var i;
console.time("dbsave");
for(i = 1; i < LIMIT; i++){
db.users.save({id : i, name : "MongoUser [" + i + "]"}, end);
}
end = function(err, saved) {
console.log(( err || !saved )?"Error":"Saved");
if(--i === 1){
console.timeEnd("dbsave");
}
};
There is a method that is designed for this. Check out process.hrtime(); .
So, I basically put this at the top of my app.
var start = process.hrtime();
var elapsed_time = function(note){
var precision = 3; // 3 decimal places
var elapsed = process.hrtime(start)[1] / 1000000; // divide by a million to get nano to milli
console.log(process.hrtime(start)[0] + " s, " + elapsed.toFixed(precision) + " ms - " + note); // print message + time
start = process.hrtime(); // reset the timer
}
Then I use it to see how long functions take. Here's a basic example that prints the contents of a text file called "output.txt":
var debug = true;
http.createServer(function(request, response) {
if(debug) console.log("----------------------------------");
if(debug) elapsed_time("recieved request");
var send_html = function(err, contents) {
if(debug) elapsed_time("start send_html()");
response.writeHead(200, {'Content-Type': 'text/html' } );
response.end(contents);
if(debug) elapsed_time("end send_html()");
}
if(debug) elapsed_time("start readFile()");
fs.readFile('output.txt', send_html);
if(debug) elapsed_time("end readFile()");
}).listen(8080);
Here's a quick test you can run in a terminal (BASH shell):
for i in {1..100}; do echo $i; curl http://localhost:8080/; done
Invoking console.time('label') will record the current time in milliseconds, then later calling console.timeEnd('label') will display the duration from that point.
The time in milliseconds will be automatically printed alongside the label, so you don't have to make a separate call to console.log to print a label:
console.time('test');
//some code
console.timeEnd('test'); //Prints something like that-> test: 11374.004ms
For more information, see Mozilla's developer docs on console.time.
Surprised no one had mentioned yet the new built in libraries:
Available in Node >= 8.5, and should be in Modern Browers
https://developer.mozilla.org/en-US/docs/Web/API/Performance
https://nodejs.org/docs/latest-v8.x/api/perf_hooks.html#
Node 8.5 ~ 9.x (Firefox, Chrome)
// const { performance } = require('perf_hooks'); // enable for node
const delay = time => new Promise(res=>setTimeout(res,time))
async function doSomeLongRunningProcess(){
await delay(1000);
}
performance.mark('A');
(async ()=>{
await doSomeLongRunningProcess();
performance.mark('B');
performance.measure('A to B', 'A', 'B');
const measure = performance.getEntriesByName('A to B')[0];
// firefox appears to only show second precision.
console.log(measure.duration);
// apparently you should clean up...
performance.clearMarks();
performance.clearMeasures();
// Prints the number of milliseconds between Mark 'A' and Mark 'B'
})();
https://repl.it/#CodyGeisler/NodeJsPerformanceHooks
Node 12.x
https://nodejs.org/docs/latest-v12.x/api/perf_hooks.html
const { PerformanceObserver, performance } = require('perf_hooks');
const delay = time => new Promise(res => setTimeout(res, time))
async function doSomeLongRunningProcess() {
await delay(1000);
}
const obs = new PerformanceObserver((items) => {
console.log('PerformanceObserver A to B',items.getEntries()[0].duration);
// apparently you should clean up...
performance.clearMarks();
// performance.clearMeasures(); // Not a function in Node.js 12
});
obs.observe({ entryTypes: ['measure'] });
performance.mark('A');
(async function main(){
try{
await performance.timerify(doSomeLongRunningProcess)();
performance.mark('B');
performance.measure('A to B', 'A', 'B');
}catch(e){
console.log('main() error',e);
}
})();
For anyone want to get time elapsed value instead of console output :
use process.hrtime() as #D.Deriso suggestion, below is my simpler approach :
function functionToBeMeasured() {
var startTime = process.hrtime();
// do some task...
// ......
var elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime));
console.log('It takes ' + elapsedSeconds + 'seconds');
}
function parseHrtimeToSeconds(hrtime) {
var seconds = (hrtime[0] + (hrtime[1] / 1e9)).toFixed(3);
return seconds;
}
var start = +new Date();
var counter = 0;
for(var i = 1; i < LIMIT; i++){
++counter;
db.users.save({id : i, name : "MongoUser [" + i + "]"}, function(err, saved) {
if( err || !saved ) console.log("Error");
else console.log("Saved");
if (--counter === 0)
{
var end = +new Date();
console.log("all users saved in " + (end-start) + " milliseconds");
}
});
}
Old question but for a simple API and light-weight solution; you can use perfy which uses high-resolution real time (process.hrtime) internally.
var perfy = require('perfy');
function end(label) {
return function (err, saved) {
console.log(err ? 'Error' : 'Saved');
console.log( perfy.end(label).time ); // <——— result: seconds.milliseconds
};
}
for (var i = 1; i < LIMIT; i++) {
var label = 'db-save-' + i;
perfy.start(label); // <——— start and mark time
db.users.save({ id: i, name: 'MongoUser [' + i + ']' }, end(label));
}
Note that each time perfy.end(label) is called, that instance is auto-destroyed.
Disclosure: Wrote this module, inspired by D.Deriso's answer. Docs here.
You could also try exectimer. It gives you feedback like:
var t = require("exectimer");
var myFunction() {
var tick = new t.tick("myFunction");
tick.start();
// do some processing and end this tick
tick.stop();
}
// Display the results
console.log(t.timers.myFunction.duration()); // total duration of all ticks
console.log(t.timers.myFunction.min()); // minimal tick duration
console.log(t.timers.myFunction.max()); // maximal tick duration
console.log(t.timers.myFunction.mean()); // mean tick duration
console.log(t.timers.myFunction.median()); // median tick duration
[edit] There is an even simpler way now to use exectime. Your code could be wrapped like this:
var t = require('exectimer'),
Tick = t.Tick;
for(var i = 1; i < LIMIT; i++){
Tick.wrap(function saveUsers(done) {
db.users.save({id : i, name : "MongoUser [" + i + "]"}, function(err, saved) {
if( err || !saved ) console.log("Error");
else console.log("Saved");
done();
});
});
}
// Display the results
console.log(t.timers.myFunction.duration()); // total duration of all ticks
console.log(t.timers.saveUsers.min()); // minimal tick duration
console.log(t.timers.saveUsers.max()); // maximal tick duration
console.log(t.timers.saveUsers.mean()); // mean tick duration
console.log(t.timers.saveUsers.median()); // median tick duration
You can use a wrapper function to easily report the execution time of any existing function.
A wrapper is a used to extend an existing function to do something before and after the existing function's execution - and is a convenient way to compose logic.
Here is an example of using the withDurationReporting wrapper:
// without duration reporting
const doSomethingThatMayTakeAWhile = async (someArg: string, anotherArg: number) => {
/** your logic goes here */
}
// with duration reporting
const doSomethingThatMayTakeAWhileWithReporting = withDurationReporting(
'doSomethingThatMayTakeAWhile',
doSomethingThatMayTakeAWhile
);
// note: you can define the function with duration reporting directly, too
const doSomethingThatMayTakeAWhile = withDurationReporting(
'doSomethingThatMayTakeAWhile',
async (someArg: string, anotherArg: number) => {
/** your logic goes here */
}
)
And here is the wrapper itself:
import { hrtime } from 'process';
const roundToHundredths = (num: number) => Math.round(num * 100) / 100; // https://stackoverflow.com/a/14968691/3068233
/**
* a wrapper which reports how long it took to execute a function, after the function completes
*/
export const withDurationReporting = <R extends any, T extends (...args: any[]) => Promise<R>>(
title: string,
logic: T,
options: {
reportingThresholdSeconds: number;
logMethod: (message: string, metadata?: Record<string, any>) => void;
} = {
reportingThresholdSeconds: 1, // report on anything that takes more than 1 second, by default
logMethod: console.log, // log with `console.log` by default
},
) => {
return (async (...args: Parameters<T>): Promise<R> => {
const startTimeInNanoseconds = hrtime.bigint();
const result = await logic(...args);
const endTimeInNanoseconds = hrtime.bigint();
const durationInNanoseconds = endTimeInNanoseconds - startTimeInNanoseconds;
const durationInSeconds = roundToHundredths(Number(durationInNanoseconds) / 1e9); // https://stackoverflow.com/a/53970656/3068233
if (durationInSeconds >= options.reportingThresholdSeconds)
options.logMethod(`${title} took ${durationInSeconds} seconds to execute`, { title, durationInSeconds });
return result;
}) as T;
};
I designed a simple method for this, using console.time() & console.timeEnd():
measure function definition
function measureRunningTime(func,...args){
const varToString = varObj => Object.keys(varObj)[0]
const displayName = func.name || varToString({ func })
console.time(displayName)
func(...args)
console.timeEnd(displayName)
}
To use it, pass a function without arguments, with arguments binded, or with arguments as the following parameters.
Examples:
let's say I want to check the running time of the simplest searching algorithm - SimpleSearch:
measured function definition (your code here)
const simpleSearch = (array = [1,2,3] ,item = 3) => {
for(let i = 0; i< array.length; i++){
if (array[i] === item) return i;
}
return -1
}
implementation without arguments
measureRunningTime(simpleSearch)
//Prints something like that-> simpleSearch: 0.04ms
implementation with arguments using .bind()
const array = [1,2,3]
const item = 3
measureRunningTime(simpleSearch.bind(null, array, item))
//Prints something like that-> bound simpleSearch: 0.04ms
implementation with arguments without using .bind()
const array = [1,2,3]
const item = 3
measureRunningTime(simpleSearch, array, item)
//Prints something like that-> simpleSearch: 0.04ms
-> Take notice!! this implementation is far from perfect - for example there is no error handling - but it can be used to check the running times of simple algorithms,
Moreover , I'm not an experienced programmer so take everything with a grain of salt 🧂 👌
I had same issue while moving from AWS to Azure
For express & aws, you can already use, existing time() and timeEnd()
For Azure, use this:
https://github.com/manoharreddyporeddy/my-nodejs-notes/blob/master/performance_timers_helper_nodejs_azure_aws.js
These time() and timeEnd() use the existing hrtime() function, which give high-resolution real time.
Hope this helps.
I need this to be cumulative, and to measure different stuff.
Built these functions:
function startMeasuring(key) {
measureTimers[key] = process.hrtime();
}
function stopMeasuring(key) {
if (!measures[key]) {
measures[key] = 0;
}
let hrtime = process.hrtime(measureTimers[key]);
measures[key] += hrtime[0] + hrtime[1] / 1e9;
measureTimers[key] = null;
}
Usage:
startMeasuring("first Promise");
startMeasuring("first and second Promises");
await new Promise((resolve) => {
setTimeout(resolve, 1400);
});
stopMeasuring("first Promise");
stopMeasuring("first and second Promises");
startMeasuring("first and second Promises");
await new Promise((resolve) => {
setTimeout(resolve, 600);
});
stopMeasuring("first and second Promises");
console.log("Measure Results", measures);
/*
Measusre Results {
setting: 0.00002375,
'first Promise': 1.409392916,
'first and second Promise': 2.015160376
}
*/