We Keep Coding

Jest Mocking Nested Singleton Dependency

I am fairly new to Jest and am struggling to understand correct way of testing code that uses nested dependencies like MongoDb.
Here is my file hierarchy
src/getOrder/index.js <- code I want to test
src/singletons/index.js <- a singleton that will be created and used by getOrder/index.js
My getOrder/index.js looks something like this
const { getSuper, catchError } = require('../singletons');
module.exports = async function (context, req)
{
let response = {}
if (req.body.guid)
{
try
{
response = await getOrder(context, req);
}
catch (err)
{
response = catchError(context, err)
}
}
else
{
response.data = 'Missing Payload'
response.status = 400;
}
context.res =
{
status: response.status,
headers: { 'Content-Type': 'application/json' },
body: response
}
}
async function getOrder(context, req)
{
//get API singleton
let sd = await getSuper()
//get order
let res = await sd.get(`/orders/${req.body.guid}`);
//return
return { 'status': res.status, 'data': res.data };
}
And then my singletons/index.js looks like this
const axios = require('axios');
const https = require('https');
const MongoClient = require('mongodb').MongoClient;
const DateTime = require('luxon').DateTime
const dbOptions =
{
useUnifiedTopology: false,
useNewUrlParser: true
};
//singleton variables
//mongo db connection
let db;
//super connection
let sd = {};
async function getDb()
{
//establish db connection if one isn't present
if (!db)
{
const client = new MongoClient(process.env.dbUri, dbOptions);
db = await (await client.connect()).db(process.env.dbName)
}
return db;
}
async function getSuper()
{
if (sd.exp && sd.exp > DateTime.local().toString().substr(0, 19))
{
return sd.instance;
}
else
{
//get db connection
let db = await getDb()
//get token
let token = await db.collection('secrets').findOne({ 'name': process.env.SDAccessToken })
//set exp time
sd.exp = token.exp;
if (!sd.instance)
{
//creat axios instance
sd.instance = axios.create({
baseURL: process.env.SDApiUrl,
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { 'Content-Type': 'application/json' }
})
//set token
sd.instance.defaults.headers.common['Authorization'] = `Bearer ${token.value}`
}
else
{
//update token
sd.instance.defaults.headers.common['Authorization'] = `Bearer ${token.value}`
}
return sd.instance;
}
}
function catchError(context, err)
{
let response = {}
if (err.response && err.response.data)
{
response.status = err.response.status
response.data = err.response.data
context.log(response)
}
else
{
context.log(err)
response.status = 500
response.data = err
}
return response;
}
module.exports =
{
getDb,
getSuper,
catchError
}
Notice how when the sd singleton is initialized it also uses the db singleton (it calls getDb)
So I am unsure how to mock either of these from my test file. I AM trying to use #shelf/jest-mongodb to mock my database, however I am quite unsure how to turn into a singleton under a mock getDb function that will live somewhere outside of the test (so it can be reused in other tests, etc)

Essentially I figured out that as long as you have the mocking package I was using (#shelf/jest-mongodb) configured in the jest.config.js file and the jest-mongodb-config.js file then it will mock the db functionality where it is initialized. Whether it's directly in your tests, or 5 nest submodules deep in your project folder

Connection resets after 60 seconds in node.js upload application

I've written an application in node.js consisting of a server and a client for storing/uploading files.
For reproduction purposes, here's a proof of concept using a null write stream in the server and a random read stream in the client.
Using node.js 12.19.0 on Ubuntu 18.04. The client depends on node-fetch v2.6.1.
The issue I have is after 60 seconds the connection is reset and haven't found a way to make this work.
Any ideas are appreciated.
Thank you.
testServer.js
// -- DevNull Start --
var util = require('util')
, stream = require('stream')
, Writable = stream.Writable
, setImmediate = setImmediate || function (fn) { setTimeout(fn, 0) }
;
util.inherits(DevNull, Writable);
function DevNull (opts) {
if (!(this instanceof DevNull)) return new DevNull(opts);
opts = opts || {};
Writable.call(this, opts);
}
DevNull.prototype._write = function (chunk, encoding, cb) {
setImmediate(cb);
}
// -- DevNull End --
const http = require('http');
const server = http.createServer();
server.on('request', async (req, res) => {
try {
req.socket.on('end', function() {
console.log('SOCKET END: other end of the socket sends a FIN packet');
});
req.socket.on('timeout', function() {
console.log('SOCKET TIMEOUT');
});
req.socket.on('error', function(error) {
console.log('SOCKET ERROR: ' + JSON.stringify(error));
});
req.socket.on('close', function(had_error) {
console.log('SOCKET CLOSED. IT WAS ERROR: ' + had_error);
});
const writeStream = DevNull();
const promise = new Promise((resolve, reject) => {
req.on('end', resolve);
req.on('error', reject);
});
req.pipe(writeStream);
await promise;
res.writeHead(200);
res.end('OK');
} catch (err) {
res.writeHead(500);
res.end(err.message);
}
});
server.listen(8081)
.on('listening', () => { console.log('Listening on port', server.address().port); });
testClient.js
// -- RandomStream Start --
var crypto = require('crypto');
var stream = require('stream');
var util = require('util');
var Readable = stream.Readable;
function RandomStream(length, options) {
// allow calling with or without new
if (!(this instanceof RandomStream)) {
return new RandomStream(length, options);
}
// init Readable
Readable.call(this, options);
// save the length to generate
this.lenToGenerate = length;
}
util.inherits(RandomStream, Readable);
RandomStream.prototype._read = function (size) {
if (!size) size = 1024; // default size
var ready = true;
while (ready) { // only cont while push returns true
if (size > this.lenToGenerate) { // only this left
size = this.lenToGenerate;
}
if (size) {
ready = this.push(crypto.randomBytes(size));
this.lenToGenerate -= size;
}
// when done, push null and exit loop
if (!this.lenToGenerate) {
this.push(null);
ready = false;
}
}
};
// -- RandomStream End --
const fetch = require('node-fetch');
const runSuccess = async () => { // Runs in ~35 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(256e6) // new RandomStream(1024e6)
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
const runFail = async () => { // Fails after 60 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(1024e6)
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
// runSuccess().then(() => process.exit(0));
runFail().then(() => process.exit(0));

I tried (unsuccessfully) to reproduce what you are seeing based on your code example. Neither the success call is completing in ~35 seconds nor is the error being thrown in 60 seconds.
However, that being said, I think what is happening here is that your client is terminating the request.
You can increase the timeout by adding a httpAgent to the fetch PUT call. You can then set a timeout in the httpAgent.
const http = require('http');
...
const runFail = async () => { // Fails after 60 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(1024e6),
agent: new http.Agent({ keepAlive: true, timeout: 300000 })
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
See the fetch docs for adding a custom http(s) agent here
See options for creating http(s) agent here

This turned out to be a bug in node.js
Discussion here: https://github.com/nodejs/node/issues/35661

Always throws registration failed error while subscribing push notifications

I am working on solutions using which i can send desktop push notification to subscribed clients.
I have created basic solution in where whenever user click on button i ask user for whether they want to allow notifications for my app or not!
I am getting an error of "Registration failed - permission denied" whenever i click on button for first time.
So that i am not able to get required endpoints to save at backend
Here is my code
index.html
<html>
<head>
<title>PUSH NOT</title>
<script src="index.js"></script>
</head>
<body>
<button onclick="main()">Ask Permission</button>
</body>
</html>
index.js
const check = () => {
if (!("serviceWorker" in navigator)) {
throw new Error("No Service Worker support!");
} else {
console.log("service worker supported")
}
if (!("PushManager" in window)) {
throw new Error("No Push API Support!");
} else {
console.log("PushManager worker supported")
}
};
const registerServiceWorker = async () => {
const swRegistration = await navigator.serviceWorker.register("/service.js?"+Math.random());
return swRegistration;
};
const requestNotificationPermission = async () => {
const permission = await window.Notification.requestPermission();
// value of permission can be 'granted', 'default', 'denied'
// granted: user has accepted the request
// default: user has dismissed the notification permission popup by clicking on x
// denied: user has denied the request.
if (permission !== "granted") {
throw new Error("Permission not granted for Notification");
}
};
const main = async () => {
check();
const swRegistration = await registerServiceWorker();
const permission = await requestNotificationPermission();
};
// main(); we will not call main in the beginning.
service.js
// urlB64ToUint8Array is a magic function that will encode the base64 public key
// to Array buffer which is needed by the subscription option
const urlB64ToUint8Array = base64String => {
const padding = "=".repeat((4 - (base64String.length % 4)) % 4);
const base64 = (base64String + padding)
.replace(/\-/g, "+")
.replace(/_/g, "/");
const rawData = atob(base64);
const outputArray = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
outputArray[i] = rawData.charCodeAt(i);
}
return outputArray;
};
const saveSubscription = async subscription => {
console.log("Save Sub")
const SERVER_URL = "http://localhost:4000/save-subscription";
const response = await fetch(SERVER_URL, {
method: "post",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(subscription)
});
return response.json();
};
self.addEventListener("activate", async () => {
try {
const applicationServerKey = urlB64ToUint8Array(
"BFPtpIVOcn2y25il322-bHQIqXXm-OACBtFLdo0EnzGfs-jIGXgAzjY6vNapPb4MM1Z1WuTBUo0wcIpQznLhVGM"
);
const options = { applicationServerKey, userVisibleOnly: true };
const subscription = await self.registration.pushManager.subscribe(options);
console.log(JSON.stringify(subscription))
const response = await saveSubscription(subscription);
} catch (err) {
console.log(err.code)
console.log(err.message)
console.log(err.name)
console.log('Error', err)
}
});
self.addEventListener("push", function(event) {
if (event.data) {
console.log("Push event!! ", event.data.text());
} else {
console.log("Push event but no data");
}
});
Also i have created a bit of backend as well
const express = require("express");
const cors = require("cors");
const bodyParser = require("body-parser");
const webpush = require('web-push')
const app = express();
app.use(cors());
app.use(bodyParser.json());
const port = 4000;
app.get("/", (req, res) => res.send("Hello World!"));
const dummyDb = { subscription: null }; //dummy in memory store
const saveToDatabase = async subscription => {
// Since this is a demo app, I am going to save this in a dummy in memory store. Do not do this in your apps.
// Here you should be writing your db logic to save it.
dummyDb.subscription = subscription;
};
// The new /save-subscription endpoint
app.post("/save-subscription", async (req, res) => {
const subscription = req.body;
await saveToDatabase(subscription); //Method to save the subscription to Database
res.json({ message: "success" });
});
const vapidKeys = {
publicKey:
'BFPtpIVOcn2y25il322-bHQIqXXm-OACBtFLdo0EnzGfs-jIGXgAzjY6vNapPb4MM1Z1WuTBUo0wcIpQznLhVGM',
privateKey: 'mHSKS-uwqAiaiOgt4NMbzYUb7bseXydmKObi4v4bN6U',
}
webpush.setVapidDetails(
'mailto:janakprajapati90#email.com',
vapidKeys.publicKey,
vapidKeys.privateKey
)
const sendNotification = (subscription, dataToSend='') => {
webpush.sendNotification(subscription, dataToSend)
}
app.get('/send-notification', (req, res) => {
const subscription = {endpoint:"https://fcm.googleapis.com/fcm/send/dLjyDYvI8yo:APA91bErM4sn_wRIW6xCievhRZeJcIxTiH4r_oa58JG9PHUaHwX7hQlhMqp32xEKUrMFJpBTi14DeOlECrTsYduvHTTnb8lHVUv3DkS1FOT41hMK6zwMvlRvgWU_QDDS_GBYIMRbzjhg",expirationTime:null,keys:{"p256dh":"BE6kUQ4WTx6v8H-wtChgKAxh3hTiZhpfi4DqACBgNRoJHt44XymOWFkQTvRPnS_S9kmcOoDSgOVD4Wo8qDQzsS0",auth:"CfO4rOsisyA6axdxeFgI_g"}} //get subscription from your databse here.
const message = 'Hello World'
sendNotification(subscription, message)
res.json({ message: 'message sent' })
})
app.listen(port, () => console.log(`Example app listening on port ${port}!`));
Please help me

Try the following code:
index.js
const check = () => {
if (!("serviceWorker" in navigator)) {
throw new Error("No Service Worker support!");
} else {
console.log("service worker supported")
}
if (!("PushManager" in window)) {
throw new Error("No Push API Support!");
} else {
console.log("PushManager worker supported")
}
};
const saveSubscription = async subscription => {
console.log("Save Sub")
const SERVER_URL = "http://localhost:4000/save-subscription";
const response = await fetch(SERVER_URL, {
method: "post",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(subscription)
});
return response.json();
};
const urlB64ToUint8Array = base64String => {
const padding = "=".repeat((4 - (base64String.length % 4)) % 4);
const base64 = (base64String + padding)
.replace(/\-/g, "+")
.replace(/_/g, "/");
const rawData = atob(base64);
const outputArray = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
outputArray[i] = rawData.charCodeAt(i);
}
return outputArray;
};
const registerServiceWorker = async () => {
return navigator.serviceWorker.register("service.js?"+Math.random()).then((swRegistration) => {
console.log(swRegistration);
return swRegistration;
});
};
const requestNotificationPermission = async (swRegistration) => {
return window.Notification.requestPermission().then(() => {
const applicationServerKey = urlB64ToUint8Array(
"BFPtpIVOcn2y25il322-bHQIqXXm-OACBtFLdo0EnzGfs-jIGXgAzjY6vNapPb4MM1Z1WuTBUo0wcIpQznLhVGM"
);
const options = { applicationServerKey, userVisibleOnly: true };
return swRegistration.pushManager.subscribe(options).then((pushSubscription) => {
console.log(pushSubscription);
return pushSubscription;
});
});
};
const main = async () => {
check();
const swRegistration = await registerServiceWorker();
const subscription = await requestNotificationPermission(swRegistration);
// saveSubscription(subscription);
};
service.js
self.addEventListener("push", function(event) {
if (event.data) {
console.log("Push event!! ", event.data.text());
} else {
console.log("Push event but no data");
}
});

I can think of three reasons that the permission is denied
1) your site is not on https (including localhost that is not on https), the default behaviour from chrome as far as i know is to block notifications on http sites. If that's the case, click on the info icon near the url, then click on site settings, then change notifications to ask
2) if you are on Safari, then safari is using the deprecated interface of the Request permission, that is to say the value is not returned through the promise but through a callback so instead of
Notification.requestPermission().then(res => console.log(res))
it is
Notification.requestPermission(res => console.log(res))
3) Your browser settings are blocking the notifications request globally, to ensure that this is not your problem run the following code in the console (on a secured https site)
Notification.requestPermission().then(res => console.log(res))
if you receive the alert box then the problem is something else, if you don't then make sure that the browser is not blocking notifications requests

How to use nock to intercept requests regardless of body

I'm trying to use nock in my tests to intercept the request calls i'm making from the native https module in Node.js. I'm using Promise.all to make two requests to the external server. I want my tests to intercept the calls, and check some of the form fields to make sure they're filled in as i want.
I have my class setup below (kept the most relevant parts of code in):
const archiver = require('archiver');
const { generateKeyPairSync } = require('crypto');
const FormData = require('form-data');
const fs = require('fs');
const https = require('https');
class Platform {
constructor() {
this.FILESTORE_USERNAME = process.env.FILESTORE_USERNAME;
this.FILESTORE_PASSWORD = process.env.FILESTORE_PASSWORD;
}
store(serviceName) {
const { publicKey, privateKey } = this._generateKeys();
return Promise.all([this._postKey(publicKey), this._postKey(privateKey)])
.then(() => {
return this._zipKeys(publicKey, privateKey, serviceName);
})
.catch((err) => {
throw err;
});
}
_postKey(key) {
const options = this._getOptions();
const keyName = (key.search(/(PUBLIC)/) !== -1) ? 'publicKey' : 'privateKey';
const form = new FormData();
form.append('file', key);
form.append('Name', keyName);
form.append('MimeMajor', 'application');
form.append('MimeMinor', 'x-pem-file');
form.append('Extension', (keyName == 'publicKey') ? 'pub' : '');
form.append('FileClass', 'MFS::File');
options.headers = form.getHeaders();
options.headers.Authorization = 'Basic ' + Buffer.from(this.FILESTORE_USERNAME + ':' + this.FILESTORE_PASSWORD).toString('base64');
return new Promise((resolve, reject) => {
let post = https.request(options, (res) => {
let data = '';
if (res.statusCode < 200 || res.statusCode > 299) {
reject(new Error('File Storage API returned a status code outside of acceptable range: ' + res.statusCode));
} else {
res.setEncoding('utf8');
res.on('data', (chunk) => {
data += chunk;
});
res.on('error', (err) => {
reject(err);
});
res.on('end', () => {
if (data) {
resolve(JSON.parse(data));
} else {
resolve();
}
});
}
});
post.on('error', (err) => {
reject(err);
});
form.pipe(post);
post.end();
});
}
_getOptions() {
return {
hostname: 'api.example.com',
path: '/media/files/',
method: 'POST',
};
}
}
module.exports = Platform;
And then, my testing code looks like the below. I'm using mocha, sinon, chai, sinon-chai and nock.
const Platform = require('/usr/src/app/api/Services/Platform');
const crypto = require('crypto');
const fs = require('fs');
const nock = require('nock');
const yauzl = require('yauzl');
describe('Platform', function() {
let platform;
beforeEach(() => {
platform = new Platform();
});
afterEach(() => {
const list = fs.readdirSync('/usr/src/app/api/Services/data/');
list.forEach((file) => {
fs.unlink('/usr/src/app/api/Services/data/' + file, (err) => {
if (err) throw err;
});
});
nock.cleanAll();
});
after(() => {
nock.restore();
});
describe('store', function() {
it('should post each generated key to an external storage place', async function() {
this.timeout(5000);
// const stub = sinon.stub(platform, '_postKey').resolves();
const scope = nock('https://api.example.com')
.persist()
.post('/media/files/', (body) => {
// console.log(body);
})
.reply(200);
let serviceName = 'test';
let actual = await platform.store(serviceName)
.catch((err) => {
(() => { throw err; }).should.not.throw();
});
console.log(scope);
// expect(stub.callCount).to.equal(2);
expect(actual).to.be.a('string');
expect(actual).to.include(serviceName + '.zip');
// stub.reset();
});
});
});
The problem I am coming across is this error that is thrown when running my tests:
AssertionError: expected [Function] to not throw an error but 'Error:
Nock: No match for request {\n "method": "POST",\n "url":
"https://api.example.com/media/files/",\n "headers": {\n
"content-type": "multipart/form-data;
boundary=--------------------------363749230271182821931703",\n
"authorization": "Basic abcdef1224u38454857483hfjdhjgtuyrwyt="\n },\n
"body":
"----------------------------363749230271182821931703\r\nContent-Disposition: form-data; name=\"file\"\r\n\r\n-----BEGIN PUBLIC
KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq+QnVOYVjbrHIlAEsEoF\nZ4sTvqiB3sJGwecNhmgrUp9U8oqgoB50aW6VMsL71ATRyq9b3vMQKpjbU3R2RcOF\na6mlaBtBjxDGu2nEpGX++mtPCdD9HV7idvWgJ3XS0vGaCM//8ukY+VLBc1IB8CHC\nVj+8YOD5Y9TbdpwXR+0zCaiHwwd8MHIo1kBmQulIL7Avtjh55OmQZZtjO525lbqa\nWUZ24quDp38he2GjLDeTzHm9z1RjYJG6hS+Ui0s2xRUs6VAr7KFtiJmmjxPS9/vZ\nwQyFcz/R7AJKoEH8p7NE7nn/onbybJy+SWRxjXVH8afHkVoC65BiNoMiEzk1rIsx\ns92woHnq227JzYwFYcLD0W+TYjtGCB8+ks+QRIiV0pFJ3ja5VFIxjn9MxLntWcf2\nhsiYrmfJlqmpW1DMfZrtt41cJUFQwt7CpN72aix7btmd/q0syh6VVlQEHq/0nDky\nItv7dqyqZc9NNOMqK9/kXWhbq5cwS21mm+kTGas5KSdeIR0LH7uVtivB+LKum14e\nRDGascZcXZIVTbOeCxA6BD7LyaJPzXmlMy4spXlhGoDYyVRhpvv2K03Mg7ybiB4X\nEL1oJtiCFkRX5LtRJv0PCRJjaa3UvfyIuz8bHK4ANxIZqcwZwER+g02gw8iqNfMa\nDWXpfMJUU8TQuLGWZQaGJc8CAwEAAQ==\n-----END
PUBLIC
KEY-----\n\r\n----------------------------363749230271182821931703\r\nContent-Disposition: form-data;
name=\"Name\"\r\n\r\npublicKey\r\n----------------------------363749230271182821931703\r\nContent-Disposition: form-data;
name=\"MimeMajor\"\r\n\r\napplication\r\n----------------------------363749230271182821931703\r\nContent-Disposition: form-data;
name=\"MimeMinor\"\r\n\r\nx-pem-file\r\n----------------------------363749230271182821931703\r\nContent-Disposition: form-data;
name=\"Extension\"\r\n\r\npub\r\n----------------------------363749230271182821931703\r\nContent-Disposition: form-data;
name=\"FileClass\"\r\n\r\nMFS::File\r\n----------------------------363749230271182821931703--\r\n"\n}'
was thrown
I take it it's because nock expects me to fake out the body for the request to get a correct match? Is there a way of just looking for requests made to that address, regardless of the body, so I can do my own tests or whatever.

When the post method of a Nock Scope is passed a second argument, it is used to match against the body of the request.
Docs for specifying the request body
In your test, you're passing a function as the second argument, but not returning true so Nock is not considering it a match.
From the docs:
Function: nock will evaluate the function providing the request body
object as first argument. Return true if it should be considered a
match
Since your goal is to assert form fields on the request, your best approach would be to leave the function there, do your assertions where the // console.log(body); line is, but add return true; to the end of the function.
You could also return true or false depending on if your form fields match your assertions, but in my experience it makes the error output from the test convoluted. My preference is to use standard chai expect() calls and let the assertions bubble errors before Nock continues with request matching.

Nested HTTP requests in Firebase cloud function

I'm using an HTTP-triggered Firebase cloud function to make an HTTP request. I get back an array of results (events from Meetup.com), and I push each result to the Firebase realtime database. But for each result, I also need to make another HTTP request for one additional piece of information (the category of the group hosting the event) to fold into the data I'm pushing to the database for that event. Those nested requests cause the cloud function to crash with an error that I can't make sense of.
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require('request');
exports.foo = functions.https.onRequest(
(req, res) => {
var ref = admin.database().ref("/foo");
var options = {
url: "https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****",
json: true
};
return request(
options,
(error, response, body) => {
if (error) {
console.log(JSON.stringify(error));
return res.status(500).end();
}
if ("results" in body) {
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if ("name" in result &&
"description" in result &&
"group" in result &&
"urlname" in result.group
) {
var groupOptions = {
url: "https://api.meetup.com/" + result.group.urlname + "?sign=true&photo-host=public&key=****",
json: true
};
var categoryResult = request(
groupOptions,
(groupError, groupResponse, groupBody) => {
if (groupError) {
console.log(JSON.stringify(error));
return null;
}
if ("category" in groupBody &&
"name" in groupBody.category
) {
return groupBody.category.name;
}
return null;
}
);
if (categoryResult) {
var event = {
name: result.name,
description: result.description,
category: categoryResult
};
ref.push(event);
}
}
}
return res.status(200).send("processed events");
} else {
return res.status(500).end();
}
}
);
}
);
The function crashes, log says:
Error: Reference.push failed: first argument contains a function in property 'foo.category.domain._events.error' with contents = function (err) {
if (functionExecutionFinished) {
logDebug('Ignoring exception from a finished function');
} else {
functionExecutionFinished = true;
logAndSendError(err, res);
}
}
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1436:15)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
If I leave out the bit for getting the group category, the rest of the code works fine (just writing the name and description for each event to the database, no nested requests). So what's the right way to do this?

I suspect this issue is due to the callbacks. When you use firebase functions, the exported function should wait on everything to execute or return a promise that resolves once everything completes executing. In this case, the exported function will return before the rest of the execution completes.
Here's a start of something more promise based -
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require("request-promise-native");
exports.foo = functions.https.onRequest(async (req, res) => {
const ref = admin.database().ref("/foo");
try {
const reqEventOptions = {
url:
"https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=xxxxxx",
json: true
};
const bodyEventRequest = await request(reqEventOptions);
if (!bodyEventRequest.results) {
return res.status(200).end();
}
await Promise.all(
bodyEventRequest.results.map(async result => {
if (
result.name &&
result.description &&
result.group &&
result.group.urlname
) {
const event = {
name: result.name,
description: result.description
};
// get group information
const groupOptions = {
url:
"https://api.meetup.com/" +
result.group.urlname +
"?sign=true&photo-host=public&key=xxxxxx",
json: true
};
const categoryResultResponse = await request(groupOptions);
if (
categoryResultResponse.category &&
categoryResultResponse.category.name
) {
event.category = categoryResultResponse.category.name;
}
// save to the databse
return ref.push(event);
}
})
);
return res.status(200).send("processed events");
} catch (error) {
console.error(error.message);
}
});
A quick overview of the changes -
Use await and async calls to wait for things to complete vs. being triggered in a callback (async and await are generally much easier to read than promises with .then functions as the execution order is the order of the code)
Used request-promise-native which supports promises / await (i.e. the await means wait until the promise returns so we need something that returns a promise)
Used const and let vs. var for variables; this improves the scope of variables
Instead of doing checks like if(is good) { do good things } use a if(isbad) { return some error} do good thin. This makes the code easier to read and prevents lots of nested ifs where you don't know where they end
Use a Promise.all() so retrieving the categories for each event is done in parallel

There are two main changes you should implement in your code:
Since request does not return a promise you need to use an interface wrapper for request, like request-promise in order to correctly chain the different asynchronous events (See Doug's comment to your question)
Since you will then call several times (in parallel) the different endpoints with request-promise you need to use Promise.all() in order to wait all the promises resolve before sending back the response. This is also the case for the different calls to the Firebase push() method.
Therefore, modifying your code along the following lines should work.
I let you modifying it in such a way you get the values of name and description used to construct the event object. The order of the items in the results array is exactly the same than the one of the promises one. So you should be able, knowing that, to get the values of name and description within results.forEach(groupBody => {}) e.g. by saving these values in a global array.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
var rp = require('request-promise');
exports.foo = functions.https.onRequest((req, res) => {
var ref = admin.database().ref('/foo');
var options = {
url:
'https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****',
json: true
};
rp(options)
.then(body => {
if ('results' in body) {
const promises = [];
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if (
'name' in result &&
'description' in result &&
'group' in result &&
'urlname' in result.group
) {
var groupOptions = {
url:
'https://api.meetup.com/' +
result.group.urlname +
'?sign=true&photo-host=public&key=****',
json: true
};
promises.push(rp(groupOptions));
}
}
return Promise.all(promises);
} else {
throw new Error('err xxxx');
}
})
.then(results => {
const promises = [];
results.forEach(groupBody => {
if ('category' in groupBody && 'name' in groupBody.category) {
var event = {
name: '....',
description: '...',
category: groupBody.category.name
};
promises.push(ref.push(event));
} else {
throw new Error('err xxxx');
}
});
return Promise.all(promises);
})
.then(() => {
res.send('processed events');
})
.catch(error => {
res.status(500).send(error);
});
});

I made some changes and got it working with Node 8. I added this to my package.json:
"engines": {
"node": "8"
}
And this is what the code looks like now, based on R. Wright's answer and some Firebase cloud function sample code.
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require("request-promise-native");
exports.foo = functions.https.onRequest(
async (req, res) => {
var ref = admin.database().ref("/foo");
var options = {
url: "https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****",
json: true
};
await request(
options,
async (error, response, body) => {
if (error) {
console.error(JSON.stringify(error));
res.status(500).end();
} else if ("results" in body) {
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if ("name" in result &&
"description" in result &&
"group" in result &&
"urlname" in result.group
) {
var groupOptions = {
url: "https://api.meetup.com/" + result.group.urlname + "?sign=true&photo-host=public&key=****",
json: true
};
var groupBody = await request(groupOptions);
if ("category" in groupBody && "name" in groupBody.category) {
var event = {
name: result.name,
description: result.description,
category: groupBody.category.name
};
await ref.push(event);
}
}
}
res.status(200).send("processed events");
}
}
);
}
);