We Keep Coding

Cloud function does not write second argument to Firebase

I have written a cloud function that runs every 5 minutes on my Firebase app. In essence, the function gathers trends data from the Google Trends website and parses the JSON into a variable.
After doing so I want to then connect to the Twitter API and search for tweets using the trending topics fetched in the first part.
My Issue seems to lie with the second part. It fetches the data but the remainder of the function does not wait for the result before writing to Firebase.
I have tried two different methods but both don't seem to work as intended. I am struggling to understand how the function should wait for the second part to gather and store the information before writing to Firebase.
Method 1
exports.callTo = functions.pubsub.schedule("5 * * * *").onRun((context) => {
let searchTrends;
const ts = Date.now();
const dateOb = new Date(ts);
const date = dateOb.getDate();
const month = dateOb.getMonth() + 1;
const year = dateOb.getFullYear();
const twitterTrends = [];
googleTrends.dailyTrends({
trendDate: new Date(year + "-" + month + "-" + date),
geo: "CA",
}, function(err, res) {
if (err) {
functions.logger.error(err);
} else {
searchTrends = JSON.parse(res).default.trendingSearchesDays[0]
.trendingSearches;
functions.logger.info(searchTrends);
for (let i = 0; i < searchTrends.length; i++) {
functions.logger.log(searchTrends[i].title.query);
T.get("search/tweets", {q: searchTrends[i].title.query, count: 1},
function(err, data, response) {
if (err) {
functions.logger.error(err);
}
functions.logger.info("Twitter data" +
JSON.stringify(data.statuses));
twitterTrends[i] = JSON.stringify(data.statuses);
});
}
const dbRef = admin.database().ref("searchTrends");
dbRef.set({google: searchTrends, twitter: twitterTrends});
}
});
});
Method 2
exports.callTo = functions.pubsub.schedule("5 * * * *").onRun((context) => {
let searchTrends;
const ts = Date.now();
const dateOb = new Date(ts);
const date = dateOb.getDate();
const month = dateOb.getMonth() + 1;
const year = dateOb.getFullYear();
const twitterTrends = [];
async function getTrends(){
googleTrends.dailyTrends({
trendDate: new Date(year + "-" + month + "-" + date),
geo: "CA",
}, function(err, res) {
if (err) {
functions.logger.error(err);
} else {
searchTrends = JSON.parse(res).default.trendingSearchesDays[0]
.trendingSearches;
functions.logger.info(searchTrends);
}
});
await getTwitterTrends();
}
async function getTwitterTrends(){
for (let i = 0; i < 1; i++) {
functions.logger.log(searchTrends[i].title.query);
T.get("search/tweets", {q: searchTrends[i].title.query, count: 1},
function(err, data, response) {
if (err) {
functions.logger.error(err);
} else {
functions.logger.info("Twitter data" +
JSON.stringify(data.statuses));
twitterTrends[i] = JSON.stringify(data.statuses);
}
});
}
return "done";
}
const dbRef = admin.database().ref("searchTrends");
dbRef.set({google: searchTrends, twitter: twitterTrends});
});

After checking your function it looks like a Promises issue. The reason you are seeing only the searchTrends data in Firestore is because the Firestore reference and upload is being done inside the callback for the dailyTrends method (taking for reference the method 1 code). However this does not wait for each request to the Twitter API to be resolved before writing to Firestore.
Based on the documentation for twit (which seems to be the wrapper you are using), it also supports standard promises. You could add each promise to an array, and then use Promise.all() to wait until they are all resolved to then write the data into Firestore. It would look something like this (which I haven’t tested since I don’t have Twitter API access).
exports.callTo = functions.pubsub.schedule("5 * * * *").onRun((context) => {
const ts = Date.now();
const dateOb = new Date(ts);
const date = dateOb.getDate();
const month = dateOb.getMonth() + 1;
const year = dateOb.getFullYear();
let searchTrends;
const twitterTrends = [];
const twPromises = [];
googleTrends.dailyTrends({
trendDate: new Date(year + "-" + month + "-" + date),
geo: "CA",
}, function(err, res) {
if (err) {
functions.logger.error(err);
} else {
searchTrends = JSON.parse(res).default.trendingSearchesDays[0]
.trendingSearches;
functions.logger.info(searchTrends);
for (let i = 0; i < searchTrends.length; i++) {
functions.logger.log(searchTrends[i].title.query);
twPromises.push(T.get("search/tweets", {q: searchTrends[i].title.query, count: 1})); // adds promises to the array
}
Promise.all(twPromises).then((responses) => { // runs when all promises from the array are resolved
responses.forEach((response) => {
twitterTrends.push(JSON.stringify(response.statuses));
})
const dbRef = admin.database().ref("searchTrends");
dbRef.set({google: searchTrends, twitter: twitterTrends});
})
}
});
});

Why can't I get the result sorted after using pagination?

I am learning how to do pagination. So I got a task to use skip and limit and then on top of that sort it. I've used REST API and query parameters to pass the things required like skip limit sort.
Now, my logic gets me to skip and limit running when I hit the GET API but sorting doesn't work. Can anyone provide a workaround and also explain why mine isn't working?
Code:
const getAliens = async (req, res) => {
try {
const skip = parseInt(req.query.skip);
const limit = parseInt(req.query.limit);
const sort = req.query.sort;
const sortOn = {};
if (sort != null) {
const key = sort.split(":")[0];
const value = sort.split(":")[1] == "asc" ? 1 : -1;
sortOn[key] === value;
}
const startIndex = (skip - 1) * limit;
const endIndex = skip * limit;
const response = await Alien.find()
.sort(sortOn)
.find()
.limit(limit)
.skip(startIndex)
.exec();
return ResponseUtils.success(res, response);
} catch (e) {
Log.error(e);
return ResponseUtils.error(res);
}
};

coin-base pro web socket current copy of order book (level 2)

hi is it possible to keep an up to date copy of the order book in an array from the level 2 web socket feed similar to how it is on the website or in the first message returned as snapshot
snapshot returned from level 2 update as type="snapshot"
this is what i am trying to recreate using the type="l2update"
the question is how do i use the l2updade to keep an accurate copy of the order book or at least get the top level ask and bid from the web socket instead of poling the "getProductOrderBook"
const websocket = new CoinbasePro.WebsocketClient(
["BTC-EUR"],
"wss://ws-feed.pro.coinbase.com",
null,
{ channels: [
{
"name": "level2"
}
]
});
websocket.on('message', data => {
var bids = [];
var asks = [];
var x= 0;
var x2 = 0;
if(data.changes[0][0] == "buy"){
if(x <= 100){
bids[x] = parseFloat(data.changes[0][1]);
x++;
}else{
x = 0;
bids[x] = parseFloat(data.changes[0][1]);
x++;
}
}
if(data.changes[0][0] == "sell"){
if(x2 <= 100){
asks[x2] = parseFloat(data.changes[0][1]);
x2++;
}else{
x2 = 0;
asks[x2] = parseFloat(data.changes[0][1]);
x2++;
}
}
console.log(data);
});

To do it directly in the array is useless complicated. I would do it in an object and would convert the object to an array when required. Something like (I assume product_id is always BTC-EUR):
let asks;
let bids;
websocket.on("message", data => {
if(data.type == "snapshot") {
asks = {};
bids = {};
data.asks.forEach(([price, amount]) => (asks[price] = parseFloat(amount)));
data.bids.forEach(([price, amount]) => (bids[price] = parseFloat(amount)));
return;
}
if(data.type == "l2update") data.changes.forEach(change => {
const [action, price, amount] = change;
const obj = action == "buy" ? bids : asks;
const parsed = parseFloat(amount);
if(parsed) obj[price] = parsed;
else delete obj[price];
});
});
Than you can always get the arrays with (or vice versa, didn't tested):
Object.entries(asks).map(([pr, am]) => [parseFloat(pr), am]).sort((a, b) => a[0] - b[0]);
Object.entries(bids).map(([pr, am]) => [parseFloat(pr), am]).sort((a, b) => b[0] - a[0]);
Hope this helps.

How do I optimize this synchronous "fuzzy search" function and turn it into an async function?

Problem
I'm trying to implement some sort of "fuzzy search" in my Node.js based project.
Fuzzy search is a search that returns results even if the string didn't match exactly.
I found this code in another stackoverflow thread. Code is below.
It's quite good, but the problem is - it's synchronous It slows down the whole program when it searches through a large array.
Question
ES6 methods are welcomed. Needs to work only in the latest Chrome, so any JS methods will work.
Are there any new JS methods that would optimize this function?
Am I doing something wrong there that makes it even slower?
Can I turn this function into an async function that returns a promise? Will it stop freezing the app during the search then?
Are there any better "fuzzy search" implementations you know of? (I found a module called fuzzysort, can't say if it's that much better though, it won't return "folder test" if you type "test folder" (wrong order) so it's not that good)
Code
Calling search function
searchArray is an array of paths it searches through, e.g.: ["C:\\test", "C:\\file.txt"...] (0.5 - 5 million paths)
searchQuery is a string without spaces, e.g.: filetxt
search () {
const fuzzySearch = this.fuzzySearch(this.searchQuery.toLowerCase(), this.searchArray)
let result = fuzzySearch.filter(element => element.relevance >= 0.3)
// sort by relevance
var sortedResults = result.sort((a, b) => parseFloat(b.relevance) - parseFloat(a.relevance)).map(item => item.name);
this.searchResults = sortedResults
},
The fuzzy search function
fuzzySearch (searchQuery, searchArray) {
const get_bigrams = function(string) {
const s = string.toLowerCase();
const v = new Array(s.length - 1);
for (let i = 0, end = v.length; i <= end; i++) {
v[i] = s.slice(i, i + 2);
}
return v;
};
const string_similarity = function(str1, str2) {
if ((str1.length > 0) && (str2.length > 0)) {
const pairs1 = get_bigrams(str1);
const pairs2 = get_bigrams(str2);
const union = pairs1.length + pairs2.length;
let hit_count = 0;
for (let x of Array.from(pairs1)) {
for (let y of Array.from(pairs2)) {
if (x === y) {
hit_count++;
}
}
}
if (hit_count > 0) {
return ((2.0 * hit_count) / union);
}
}
return 0.0;
};
let results = [];
for (let name of searchArray) {
// I added .match to use only the base filename (name+ext) not the whole path, and removed all characters
let filteredPath = name.match(/[^\\\/]+$/)[0].replace(/[^A-Za-z0-9.]+/g, '')
const relevance = string_similarity(searchQuery, filteredPath);
const obj = {name, relevance};
results.push(obj);
}
return results
},

Illegal return statement in JavaScript

A 'days' node with more than 1 child isn't getting removed. How can I fix this issue?
I need to ensure that my promise bubbles up to the last then() on the top-level. So I need a return before collectionRef.once. But that return statement now prevents the collectionRef.once from happening. I'm stuck!
Here's my code
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
const defaultDatabase = admin.database();
exports.deleteOldItems = functions.database.ref('/path/to/items/{pushId}')
.onWrite(event => {
var ref = event.data.ref.parent; // reference to the items
var now = Date.now();
var cutoff = now - 2 * 60 * 60 * 1000;
var oldItemsQuery = ref.orderByChild('timestamp').endAt(cutoff);
return oldItemsQuery.once('value', function(snapshot) {
// create a map with all children that need to be removed
var updates = {};
snapshot.forEach(function(child) {
updates[child.key] = null
});
// execute all updates in one go and return the result to end the function
return ref.update(updates);
}).then(function() {;
const theRef = event.data.ref;
const collectionRef = theRef.parent.child('days');
return collectionRef; // ILEGAL RETURN STATEMENT
collectionRef.once('value').then(messagesData => {
if(messagesData.numChildren() > 1) {
let updates = {};
updates['/days'] = null;
return defaultDatabase.ref().update(updates); // 'days' doesn't get removed even if it has more than 1 child (as in the image)!
}
})
});
});
Data structure: https://i.stack.imgur.com/gVn8S.jpg

exports.deleteOldItems = functions.database.ref('/path/to/items/{pushId}')
.onWrite(event => {
var ref = event.data.ref.parent // reference to the items
var now = Date.now()
var cutoff = now - 2 * 60 * 60 * 1000
var oldItemsQuery = ref.orderByChild('timestamp').endAt(cutoff)
return oldItemsQuery.once('value', function(snapshot) {
// create a map with all children that need to be removed
var updates = {}
snapshot.forEach(function(child) {
updates[child.key] = null
})
// execute all updates in one go and return the result to end the function
return ref.update(updates)
}).then(function() {
// const theRef = event.data.ref
const collectionRef = defaultDatabase.ref().child('/days')
// return collectionRef // ILEGAL RETURN STATEMENT
collectionRef.once('value').then(messagesData => {
console.log(`Hello messageData : ${messagesData.numChildren()}`)
if(messagesData.numChildren() > 1) {
const updates = {}
updates['/days'] = null
return defaultDatabase.ref().update(updates); // 'days' doesn't get removed even if it has more than 1 child (as in the image)!
}
})
})
use defaultDatabase.ref().child('/days') instead of using event.data.ref.parent
also please go through the documentation and learn how promises works it will help you future. For now these changes will work.Tested at my end.
videos you must watch
What's the difference between event.data.ref and event.data.adminRef? - #AskFirebase
Asynchronous Programming (I Promise!) with Cloud Functions for Firebase - Firecasts
you can subscribe their Firebase YouTube Channel to get latest updates and learn More.