I am trying to use Twitter's API with node.js using async/await (which I admit I am new to) but I am struggling to get to the next cursor value.
Why does my getFollowers function bellow always returns before the await block?
require('dotenv').config();
const Twitter = require('twitter');
const client = new Twitter({
consumer_key: process.env.API_KEY,
consumer_secret: process.env.API_KEY_SECRET,
access_token_key: process.env.ACCESS_TOKEN,
access_token_secret: process.env.ACCESS_TOKEN_SECRET
});
const getFollowers = async (screen_name, count, cursor) => {
console.log("Cursor: " + cursor);
const params = {
screen_name: screen_name,
count: count,
cursor: cursor
};
const promise = await client.get('followers/list', params)
.then(data => {
console.log("This promise is never executed...");
return data.next_cursor;
})
.catch(err => console.error(err));
return promise;
}
const main = async () => {
let cursor = -1;
while (cursor != 0) {
getFollowers(process.env.SCREEN_NAME, 200, cursor)
.then(next_cursor => {
cursor = next_cursor;
console.log("This promise is never executed either... " + cursor);
});
}
}
main();
With your .then statement in main(), you weren't awaiting for client.get() to resolve, but for data.next_cursor(). Therefore, promise of client.get() remained pending.
Instead, return the promise of client.get() as a in getFollowers(). This will make sure that when you call getFollowers().then() in main(), you are referring to client.get.
Edit:
Following the line of thought in the answer in this question, I have modified getFollowers(). It now includes a promise that is resolved when cursor hits the value of 0. Every other value, a request will be made.
I have a concern though with the rate limit of requests, which is set to 15 per 15 minutes. Since a new request is made for every non 0 next_cursor value, you'll reach this limit quite soon for accounts with many followers.
Also note that the data retrieved will be stored in an array. I am not sure what your use case exactly is.
const Twitter = require('twitter');
const client = new Twitter({
consumer_key: '',
consumer_secret: '',
bearer_token: ''
});
let output = [];
const getFollowers = (screen_name, count) => {
let cursor = -1;
const params = {
screen_name: screen_name,
count: count,
cursor: cursor
};
return new Promise((resolve, reject) => {
client.get('followers/list', params, function getData(err, data, response) {
if (err) reject(response.body);
output.push(data);
cursor = data.next_cursor;
if (cursor > 0) {
client.get('followers/list', params, getData);
}
if (cursor = 0) {
resolve('done');
}
});
});
};
const main = async () => {
await getFollowers('MozDevNet', 200);
console.log(output);
};
I gave up on the implementation using the Twitter package and switched to using axios instead.
require('dotenv').config();
const axios = require('axios');
const credentials = {
consumer_key: process.env.API_KEY,
consumer_secret: process.env.API_KEY_SECRET,
access_token_key: process.env.ACCESS_TOKEN,
access_token_secret: process.env.ACCESS_TOKEN_SECRET
};
const FOLLOWERS_LIST_ENDPOINT = "https://api.twitter.com/1.1/followers/list.json";
//documentation: https://developer.twitter.com/en/docs/authentication/oauth-2-0/application-only
const generateToken = async () => {
return process.env.BEARER_TOKEN;
}
//documentation: https://developer.twitter.com/en/docs/twitter-api/v1/accounts-and-users/follow-search-get-users/api-reference/get-followers-list
const getFollowers = async (screen_name, count, cursor) => {
let token = await generateToken();
let requestConfig = {
params: {
screen_name: screen_name,
count: count,
cursor: cursor,
include_user_entities: false
},
headers: {
Authorization: `Bearer ${token}`
}
};
let response = await axios.get(FOLLOWERS_LIST_ENDPOINT, requestConfig);
let users = response.data.users;
processUsers(users);
return response.data.next_cursor;
};
const processUsers = (users) => {
users.map(user => {
console.log(user.screen_name);
});
}
const main = async () => {
let cursor = -1;
while (cursor != 0) {
cursor = await getFollowers(process.env.SCREEN_NAME, 200, cursor);
}
}
main();
Related
I need your help, it turns out that I am trying to use the Hubstaff api. I am working on nodejs to make the connection, I followed the documentation (official hubstaff api documentation) and use the methods they give as implementation examples (example of implementation nodejs).
But I get the following error:
I don't know why this happens, and I can't find more examples of how I can implement this api. The openid-client lib is used to make the connection through the token and a state follow-up is carried out to refresh the token.
To be honest, I'm not understanding how to implement it. Can someone who has already used this API give me a little explanation? I attach the code
hubstaffConnect.util
const {
Issuer,
TokenSet
} = require('openid-client');
const fs = require('fs');
const jose = require('jose');
// constants
const ISSUER_EXPIRE_DURATION = 7 * 24 * 60 * 60; // 1 week
const ACCESS_TOKEN_EXPIRATION_FUZZ = 30; // 30 seconds
const ISSUER_DISCOVERY_URL = 'https://account.hubstaff.com';
// API URl with trailing slash
const API_BASE_URL = 'https://api.hubstaff.com/';
let state = {
api_base_url: API_BASE_URL,
issuer_url: ISSUER_DISCOVERY_URL,
issuer: {}, // The issuer discovered configuration
issuer_expires_at: 0,
token: {},
};
let client;
function loadState() {
return fs.readFileSync('./configState.json', 'utf8');
}
function saveState() {
fs.writeFileSync('./configState.json', JSON.stringify(state, null, 2), 'utf8');
console.log('State saved');
}
function unixTimeNow() {
return Date.now() / 1000;
}
async function checkToken() {
if (!state.token.access_token || state.token.expires_at < (unixTimeNow() + ACCESS_TOKEN_EXPIRATION_FUZZ)) {
console.log('Refresh token');
state.token = await client.refresh(state.token);
console.log('Token refreshed');
saveState();
}
}
async function initialize() {
console.log('API Hubstaff API');
let data = loadState();
data = JSON.parse(data);
if (data.issuer) {
state.issuer = new Issuer(data.issuer);
state.issuer_expires_at = data.issuer_expires_at;
}
if (data.token) {
state.token = new TokenSet(data.token);
}
if (data.issuer_url) {
state.issuer_url = data.issuer_url;
}
if (data.api_base_url) {
state.api_base_url = data.api_base_url;
}
if (!state.issuer_expires_at || state.issuer_expires_at < unixTimeNow()) {
console.log('Discovering');
state.issuer = await Issuer.discover(state.issuer_url);
state.issuer_expires_at = unixTimeNow() + ISSUER_EXPIRE_DURATION;
console.log(state.issuer);
}
client = new state.issuer.Client({
// For personal access token we can use PAT/PAT.
// This is only needed because the library requires a client_id where as the API endpoint does not require it
client_id: 'PAT',
client_secret: 'PAT',
});
saveState();
console.log('API Hubstaff initialized');
}
async function request(url, options) {
await checkToken();
let fullUrl = state.api_base_url + url;
return client.requestResource(fullUrl, state.token, options);
}
function tokenDetails() {
let ret = {};
if (state.token.access_token) {
ret.access_token = jose.JWT.decode(state.token.access_token);
}
if (state.token.refresh_token) {
ret.refresh_token = jose.JWT.decode(state.token.refresh_token);
}
return ret;
}
module.exports = {
initialize,
checkToken,
request,
tokenDetails
};
controller
const usersGet = async(req, res = response) => {
const response = await api.request('v2/organizations', {
method: 'GET',
json: true,
});
const body = JSON.parse(response.body);
res.render('organizations', {
title: 'Organization list',
organizations: body.organizations || []
});
};
I have tried using async-await, .then and now promise. I am quite new in javascript development.
The code
indexRouter.get('/dashboard', checkSignIn, async(request, response) => {
snapshot = await db.doc('users/accounts').get()
sites = snapshot.data()['sites']
const userId = request.session.userId
snapshot = await db.doc(`users/${userId}`).get()
var linkedSites = snapshot.data()['linked sites']
let getDs = new Promise((resolve,reject)=>{
console.log("1")
linkedSites.forEach((site) =>{
console.log("2")
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot)=>{
console.log("3")
snapshot.forEach((doc) => {
console.log("4")
console.log(doc.id)
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS.....
details.push({site:{'email': emailId, 'password': password, 'keys': keys}})
})
})
})
console.log("5")
console.log(details)
resolve(details)
}
);
getDs.then((details)=>{
console.log("rendering")
response.render('dashboard', {'details':details, 'linkedSites': linkedSites, 'sites': sites})
})
}
I am getting the response
1
2
2
5
[]
rendering
error: ...details not found in ejs
3
4
rsp121#gmail.com
3
4
test#gmail.com
According to the output, it seems like db.doc line after console.log(2) is getting executed after a certain time and resolve(details) is sent before.
Found a solution to the problem:
indexRouter.get('/dashboard', checkSignIn, async(request, response) => {
snapshot = await db.doc('users/accounts').get()
sites = snapshot.data()['sites']
const userId = request.session.userId
snapshot = await db.doc(`users/${userId}`).get()
var linkedSites = snapshot.data()['linked sites']
if(linkedSites){
const getSnapshot = (site) => {
return new Promise(resolve => {
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot) =>{
snapshot.forEach((doc) =>{
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS
details[site] = {'email': emailId, 'password': password, 'keys': keys}
resolve(true)
})
})
})
}
Promise.all(linkedSites.map(getSnapshot)).then(()=>{
console.log(linkedSites)
response.set('Cache-Control', 'no-store, no-cache, must-revalidate, private')
response.render('dashboard', {'details':details, 'linkedSites': linkedSites, 'sites': sites})
})
}
The problem is your promise resolved before the db.doc is resolved, and as your db.doc promise is inside a loop. So, you should be using the promise.all
The below code should work for you.
indexRouter.get("/dashboard", checkSignIn, async (request, response) => {
snapshot = await db.doc("users/accounts").get();
sites = snapshot.data()["sites"];
const userId = request.session.userId;
snapshot = await db.doc(`users/${userId}`).get();
var linkedSites = snapshot.data()["linked sites"];
let getDs = new Promise((resolve, reject) => {
const promises = [];
console.log("1");
linkedSites.forEach((site) => {
console.log("2");
promises.push(
new Promise((internalResolve) => {
db.doc(`users/${userId}`)
.collection(site)
.get()
.then((snapshot) => {
console.log("3");
snapshot.forEach((doc) => {
console.log("4");
console.log(doc.id);
emailId = doc.id;
keys = doc.data()["keys"];
var passwordEncrypt = doc.data()["password"];
password = cryptoJS;
details.push({
site: {
email: emailId,
password: password,
keys: keys,
},
});
internalResolve();
});
});
})
);
});
Promise.all(promises).then(() => {
console.log("5");
console.log(details);
resolve(details);
});
});
getDs.then((details) => {
console.log("rendering");
return response.render("dashboard", {
details: details,
linkedSites: linkedSites,
sites: sites,
});
});
});
More cleaner with async/await.
indexRouter.get("/dashboard", checkSignIn, async (request, response) => {
snapshot = await db.doc("users/accounts").get();
sites = snapshot.data()["sites"];
const userId = request.session.userId;
snapshot = await db.doc(`users/${userId}`).get();
var linkedSites = snapshot.data()["linked sites"];
console.log("1");
linkedSites.forEach(async (site) => {
console.log("2");
const snapshot = await db.doc(`users/${userId}`).collection(site).get();
console.log("3");
snapshot.forEach((doc) => {
console.log("4");
console.log(doc.id);
emailId = doc.id;
keys = doc.data()["keys"];
var passwordEncrypt = doc.data()["password"];
password = cryptoJS;
details.push({
site: {
email: emailId,
password: password,
keys: keys,
},
});
});
});
console.log("rendering");
return response.render("dashboard", {
details: details,
linkedSites: linkedSites,
sites: sites,
});
});
This is your code corrected, optimized and using last especifications of ECMA Script, the error is as the other comment said you are no waiting for the result of promises inside your "new Promise.." declaration. Dont foget to use try/catch inside async functions if you have not a top error handler.
The optimization is in firsts snapshot variables, this way you are getting data in parallel instead secuantially.
indexRouter.get('/dashboard', checkSignIn, async (request, response) => {
try {
let details = [];
const userId = request.session.userId
let [snapshot1, snapshot2] = await Promise.all([db.doc('users/accounts').get(), await db.doc(`users/${userId}`).get()])
let sites = snapshot1.data()['sites']
var linkedSites = snapshot2.data()['linked sites'];
await Promise.all(
linkedSites.map((site) =>
db.doc(`users/${userId}`).collection(site).get()
.then((snapshot) => {
snapshot.forEach((doc) => {
emailId = doc.id
keys = doc.data()['keys']
var passwordEncrypt = doc.data()['password']
password = cryptoJS
details.push({ site: { 'email': emailId, 'password': password, 'keys': keys } })
})
})
)
)
response.render('dashboard', { 'details': details, 'linkedSites': linkedSites, 'sites': sites })
} catch (e) {
//Render error you want
}})
linkedSites.map.... return an array of Promises that in the end are wrapped inside Promise.all and Promise.all wait until all promises are fullfilled or one of them is rejected in this last case your code goes to catch without reach the line response.render inside the try. You can avoid this catching locally the error of each promise in the map using
.then((snapshot) => {
...
}).catch(e=> { /*Do something with the rror*/})
I have an http trigger in cloud functions that is working, however I am getting some logs come back in a weird order. I am pretty sure I have not executed the promises correctly.
here is the code:
exports.createGame = functions.https.onRequest((req, res) => {
return cors(req, res, () => {
// Check for POST request
if (req.method !== "POST") {
res.status(400).send('Please send a POST request');
return;
}
const createGame = admin.firestore().collection('games')
generatePin()
function generatePin() {
...pin generating code goes here...
addGame(newGidToString)
}
function addGame(newGidToString) {
var getGame = admin.firestore().collection('games')
getGame = getGame.where('gid', '==', newGidToString)
getGame.get().then(snapshot => {
if (snapshot.empty) {
console.log('BODY ', req.body)
const promises = [];
const gameTitle = req.body.gametitle
const targetGID = req.body.target
const numPlayers = req.body.playercount.toString()
const newGID = newGidToString
const collections = ['games', 'clues', 'detours', 'messages', 'questions', 'tagstate']
collections.forEach(function (element) {
console.log('start loop', element)
let elementsRef = admin.firestore().collection(element)
elementsRef = elementsRef.where('gid', '==', targetGID)
elementsRef.get().then(snapshot => {
var batch = admin.firestore().batch()
snapshot.forEach(function (doc) {
// change the gid to the new game gid
let data = doc.data()
data.gid = newGID
if(data.template){
data.template = false
}
if(!data.parent) {
data.parent = targetGID
}
if (!data.playercount) {
data.playercount = numPlayers
}
if (data.gametitle) {
data.gametitle = gameTitle
}
let elementRef = admin.firestore().collection(element).doc()
batch.set(elementRef, data)
})
batch.commit().then(data => {
console.log('complete batch ', element)
})
})
})
res.send({ status: 200, message: 'Game: added.', pin: newGID})
console.log('completed');
} else {
console.log('found a match ', snapshot)
// re-run the generator for a new pin
generatePin()
}
})
}
})
})
Here is a screen shot of the console logs.
Notice the order of the logs. It would seem logical that completed would come at the end fo the loops.
Any help is great appreciated.
UPDATE:
function addGame(newGidToString) {
var getGame = admin.firestore().collection('games')
getGame = getGame.where('gid', '==', newGidToString)
getGame.get().then(snapshot => {
if (snapshot.empty) {
console.log('BODY ', req.body)
const promises = [];
const gameTitle = req.body.gametitle
const targetGID = req.body.target
const numPlayers = req.body.playercount.toString()
const newGID = newGidToString
const collections = ['games', 'clues', 'detours', 'messages', 'questions', 'tagstate']
collections.forEach(function (element) {
console.log('start loop', element)
let elementsRef = admin.firestore().collection(element)
elementsRef = elementsRef.where('gid', '==', targetGID)
// elementsRef.get().then(snapshot => {
const promGet = elementsRef.get(); //lets get our promise
promises.push(promGet); //place into an array
promGet.then(snapshot => { //we can now continue as before
var batch = admin.firestore().batch()
snapshot.forEach(function (doc) {
// change the gid to the new game gid
let data = doc.data()
data.gid = newGID
if(data.template){
data.template = false
}
if(!data.parent) {
data.parent = targetGID
}
if (!data.playercount) {
data.playercount = numPlayers
}
if (data.gametitle) {
data.gametitle = gameTitle
}
let elementRef = admin.firestore().collection(element).doc()
batch.set(elementRef, data)
})
batch.commit().then(data => {
console.log('complete batch ', element)
})
})
})
Promise.all(promises).then(() => {
res.send({ status: 200, message: 'Game: added.', pin: newGID })
console.log('completed');
});
// res.send({ status: 200, message: 'Game: added.', pin: newGID})
// console.log('completed');
} else {
console.log('found a match ', snapshot)
// re-run the generator for a new pin
generatePin()
}
})
}
SECOND UPDATE: (based off Keith's answer)
function addGame(newGidToString) {
var getGame = admin.firestore().collection('games')
getGame = getGame.where('gid', '==', newGidToString)
getGame.get().then(snapshot => {
if (snapshot.empty) {
console.log('BODY ', req.body)
const gameTitle = req.body.gametitle
const targetGID = req.body.target
const numPlayers = req.body.playercount.toString()
const newGID = newGidToString
const promises = [];
const collections = ['games', 'clues', 'detours', 'messages', 'questions', 'tagstate']
collections.forEach(function (element) {
console.log('start loop', element)
let elementsRef = admin.firestore().collection(element)
elementsRef = elementsRef.where('gid', '==', targetGID)
const getPromise = elementsRef.get(); //lets get our promise
promises.push(getPromise); //place into an array
getPromise.then(snapshot => { //we can now continue as before
var batch = admin.firestore().batch()
snapshot.forEach(function (doc) {
// change the gid to the new game gid
let data = doc.data()
data.gid = newGID
if(data.template){
data.template = false
}
if(!data.parent) {
data.parent = targetGID
}
if (!data.playercount) {
data.playercount = numPlayers
}
if (data.gametitle) {
data.gametitle = gameTitle
}
let elementRef = admin.firestore().collection(element).doc()
batch.set(elementRef, data)
})
batch.commit()
})
})
Promise.all(promises).then(() => {
res.send({ status: 200, message: 'Game: added.', pin: newGID })
console.log('completed from promise');
});
} else {
console.log('found a match ', snapshot)
// re-run the generator for a new pin
generatePin()
}
})
}
When looping with promises, one thing to remember is if you do a forEach, it's not going to wait.
It appears that what the OP wants to do is before he hits the complete, and does a res send, wants to make sure all promises have been completed.
Promise.all https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all , is exactly what this does.
So every time you create a promise if you push this onto an array, we can later use Promise.all to make sure all the promises are complete.
So if we replace
elementsRef.get().then(snapshot => {
with
const promGet = elementsRef.get(); //lets get our promise
promises.push(promGet); //place into an array
promGet.then(snapshot => { //we can now continue as before
Your promise is now stored for later so that Promise.all can check to make sure everything has complete. You could do this in one line, but I feel it's easier to see what's happening this way. eg->
promises.push(promGet.then(snapshot => {
Finally now we have got our promise array, before we do our res.send, lets make sure they have all completed.
Promise.all(promises).then(() => {
res.send({ status: 200, message: 'Game: added.', pin: newGID})
console.log('completed');
});
One thing to keep in mind when using Promises this way, is that you have a fair few things going on at the same time, sometimes this can be an issue, eg. connected to some service might put restrictions on the number of concurrent connections etc. So in these cases it might be an idea doing things in series or a mix of both. There are even libs that can do concurrecy, eg. Do only X amounts of promises at a time.
And finally, if you can use async / await, I'd advice using them, as it makes Promises so much nicer to work with.
I'm using an HTTP-triggered Firebase cloud function to make an HTTP request. I get back an array of results (events from Meetup.com), and I push each result to the Firebase realtime database. But for each result, I also need to make another HTTP request for one additional piece of information (the category of the group hosting the event) to fold into the data I'm pushing to the database for that event. Those nested requests cause the cloud function to crash with an error that I can't make sense of.
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require('request');
exports.foo = functions.https.onRequest(
(req, res) => {
var ref = admin.database().ref("/foo");
var options = {
url: "https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****",
json: true
};
return request(
options,
(error, response, body) => {
if (error) {
console.log(JSON.stringify(error));
return res.status(500).end();
}
if ("results" in body) {
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if ("name" in result &&
"description" in result &&
"group" in result &&
"urlname" in result.group
) {
var groupOptions = {
url: "https://api.meetup.com/" + result.group.urlname + "?sign=true&photo-host=public&key=****",
json: true
};
var categoryResult = request(
groupOptions,
(groupError, groupResponse, groupBody) => {
if (groupError) {
console.log(JSON.stringify(error));
return null;
}
if ("category" in groupBody &&
"name" in groupBody.category
) {
return groupBody.category.name;
}
return null;
}
);
if (categoryResult) {
var event = {
name: result.name,
description: result.description,
category: categoryResult
};
ref.push(event);
}
}
}
return res.status(200).send("processed events");
} else {
return res.status(500).end();
}
}
);
}
);
The function crashes, log says:
Error: Reference.push failed: first argument contains a function in property 'foo.category.domain._events.error' with contents = function (err) {
if (functionExecutionFinished) {
logDebug('Ignoring exception from a finished function');
} else {
functionExecutionFinished = true;
logAndSendError(err, res);
}
}
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1436:15)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
at /user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1479:13
at Object.forEach (/user_code/node_modules/firebase-admin/node_modules/#firebase/util/dist/index.node.cjs.js:837:13)
at validateFirebaseData (/user_code/node_modules/firebase-admin/node_modules/#firebase/database/dist/index.node.cjs.js:1462:14)
If I leave out the bit for getting the group category, the rest of the code works fine (just writing the name and description for each event to the database, no nested requests). So what's the right way to do this?
I suspect this issue is due to the callbacks. When you use firebase functions, the exported function should wait on everything to execute or return a promise that resolves once everything completes executing. In this case, the exported function will return before the rest of the execution completes.
Here's a start of something more promise based -
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require("request-promise-native");
exports.foo = functions.https.onRequest(async (req, res) => {
const ref = admin.database().ref("/foo");
try {
const reqEventOptions = {
url:
"https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=xxxxxx",
json: true
};
const bodyEventRequest = await request(reqEventOptions);
if (!bodyEventRequest.results) {
return res.status(200).end();
}
await Promise.all(
bodyEventRequest.results.map(async result => {
if (
result.name &&
result.description &&
result.group &&
result.group.urlname
) {
const event = {
name: result.name,
description: result.description
};
// get group information
const groupOptions = {
url:
"https://api.meetup.com/" +
result.group.urlname +
"?sign=true&photo-host=public&key=xxxxxx",
json: true
};
const categoryResultResponse = await request(groupOptions);
if (
categoryResultResponse.category &&
categoryResultResponse.category.name
) {
event.category = categoryResultResponse.category.name;
}
// save to the databse
return ref.push(event);
}
})
);
return res.status(200).send("processed events");
} catch (error) {
console.error(error.message);
}
});
A quick overview of the changes -
Use await and async calls to wait for things to complete vs. being triggered in a callback (async and await are generally much easier to read than promises with .then functions as the execution order is the order of the code)
Used request-promise-native which supports promises / await (i.e. the await means wait until the promise returns so we need something that returns a promise)
Used const and let vs. var for variables; this improves the scope of variables
Instead of doing checks like if(is good) { do good things } use a if(isbad) { return some error} do good thin. This makes the code easier to read and prevents lots of nested ifs where you don't know where they end
Use a Promise.all() so retrieving the categories for each event is done in parallel
There are two main changes you should implement in your code:
Since request does not return a promise you need to use an interface wrapper for request, like request-promise in order to correctly chain the different asynchronous events (See Doug's comment to your question)
Since you will then call several times (in parallel) the different endpoints with request-promise you need to use Promise.all() in order to wait all the promises resolve before sending back the response. This is also the case for the different calls to the Firebase push() method.
Therefore, modifying your code along the following lines should work.
I let you modifying it in such a way you get the values of name and description used to construct the event object. The order of the items in the results array is exactly the same than the one of the promises one. So you should be able, knowing that, to get the values of name and description within results.forEach(groupBody => {}) e.g. by saving these values in a global array.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
var rp = require('request-promise');
exports.foo = functions.https.onRequest((req, res) => {
var ref = admin.database().ref('/foo');
var options = {
url:
'https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****',
json: true
};
rp(options)
.then(body => {
if ('results' in body) {
const promises = [];
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if (
'name' in result &&
'description' in result &&
'group' in result &&
'urlname' in result.group
) {
var groupOptions = {
url:
'https://api.meetup.com/' +
result.group.urlname +
'?sign=true&photo-host=public&key=****',
json: true
};
promises.push(rp(groupOptions));
}
}
return Promise.all(promises);
} else {
throw new Error('err xxxx');
}
})
.then(results => {
const promises = [];
results.forEach(groupBody => {
if ('category' in groupBody && 'name' in groupBody.category) {
var event = {
name: '....',
description: '...',
category: groupBody.category.name
};
promises.push(ref.push(event));
} else {
throw new Error('err xxxx');
}
});
return Promise.all(promises);
})
.then(() => {
res.send('processed events');
})
.catch(error => {
res.status(500).send(error);
});
});
I made some changes and got it working with Node 8. I added this to my package.json:
"engines": {
"node": "8"
}
And this is what the code looks like now, based on R. Wright's answer and some Firebase cloud function sample code.
const functions = require("firebase-functions");
const admin = require("firebase-admin");
admin.initializeApp();
const request = require("request-promise-native");
exports.foo = functions.https.onRequest(
async (req, res) => {
var ref = admin.database().ref("/foo");
var options = {
url: "https://api.meetup.com/2/open_events?sign=true&photo-host=public&lat=39.747988&lon=-104.994945&page=20&key=****",
json: true
};
await request(
options,
async (error, response, body) => {
if (error) {
console.error(JSON.stringify(error));
res.status(500).end();
} else if ("results" in body) {
for (var i = 0; i < body.results.length; i++) {
var result = body.results[i];
if ("name" in result &&
"description" in result &&
"group" in result &&
"urlname" in result.group
) {
var groupOptions = {
url: "https://api.meetup.com/" + result.group.urlname + "?sign=true&photo-host=public&key=****",
json: true
};
var groupBody = await request(groupOptions);
if ("category" in groupBody && "name" in groupBody.category) {
var event = {
name: result.name,
description: result.description,
category: groupBody.category.name
};
await ref.push(event);
}
}
}
res.status(200).send("processed events");
}
}
);
}
);
I have a app.get which inside of it is quite a bit of logic. Which everything works great aside from some of the logic being called twice for some reason. I have noticed when I was saving something to by db that it would save two rows.
So I put a console.log in that area and sure enough it was logging it twice.
Any reason why this is happening?
app.get('/shopify/callback', (req, res) => {
const { shop, hmac, code, state } = req.query;
const stateCookie = cookie.parse(req.headers.cookie).state;
if (state !== stateCookie) {
return res.status(403).send('Request origin cannot be verified');
}
if (shop && hmac && code) {
// DONE: Validate request is from Shopify
const map = Object.assign({}, req.query);
delete map['signature'];
delete map['hmac'];
const message = querystring.stringify(map);
const providedHmac = Buffer.from(hmac, 'utf-8');
const generatedHash = Buffer.from(
crypto
.createHmac('sha256', config.oauth.client_secret)
.update(message)
.digest('hex'),
'utf-8'
);
let hashEquals = false;
try {
hashEquals = crypto.timingSafeEqual(generatedHash, providedHmac)
} catch (e) {
hashEquals = false;
};
if (!hashEquals) {
return res.status(400).send('HMAC validation failed');
}
// DONE: Exchange temporary code for a permanent access token
const accessTokenRequestUrl = 'https://' + shop + '/admin/oauth/access_token';
const accessTokenPayload = {
client_id: config.oauth.api_key,
client_secret: config.oauth.client_secret,
code,
};
request.post(accessTokenRequestUrl, { json: accessTokenPayload })
.then((accessTokenResponse) => {
const accessToken = accessTokenResponse.access_token;
// DONE: Use access token to make API call to 'shop' endpoint
const shopRequestUrl = 'https://' + shop + '/admin/shop.json';
const shopRequestHeaders = {
'X-Shopify-Access-Token': accessToken,
}
request.get(shopRequestUrl, { headers: shopRequestHeaders })
.then((shopResponse) => {
const response = JSON.parse(shopResponse);
const shopData = response.shop;
console.log('BEING CALLED TWICE...')
res.render('pages/brand_signup',{
shop: shopData.name
})
})
.catch((error) => {
res.status(error.statusCode).send(error.error.error_description);
});
})
.catch((error) => {
res.status(error.statusCode).send(error.error.error_description);
});
} else {
res.status(400).send('Required parameters missing');
}
});