First of all, please forgive me if this is a duplicate, I am new to coding and Javascript in general.
I have an async function that queries mongodb based on an objects passed in the function call. The function executes, and returns the results to a callback function which logs the results to the console, and then hangs. Ultimately, I want to take the results of the async query and then do something with them outside the original async function. I am not understanding why it hangs after it logs to the console.
const MongoClient = require('mongodb').MongoClient;
let fObj = {
field : {},
limit : 100
}
let cObj = {
dbName : 'myNewDatabase',
colName : 'newCollection'
}
async function findDoc(cObj,fObj) {
const url = 'mongodb://localhost:27017';
const client = new MongoClient(url, { useNewUrlParser: true });
try {
await client.connect();
const db = client.db(cObj.dbName);
const col = db.collection(cObj.colName);
console.log(`Connection Made to ${db.databaseName} database.`);
return await col.find(fObj.field).limit(fObj.limit).toArray();
client.close();
} catch (err) {
console.log(err.stack);
}
};
findDoc(cObj,fObj).then(function(result) {
console.log(result);
});
The code executes, and logs the results to the console, but then hangs. I have to ctrl-c out to get it to end. What am I missing?
I suppouse you're running your code with NodeJs. This implies that you have a promise hanging up, which keeps the server running. I assume this is because your connection to the DB is still open after you have found the document.
You need to move your client.close(); statement above the return statement, because it is never reached otherwise and your server will hang up forever.
Your code will look like this in the end:
const MongoClient = require('mongodb').MongoClient;
let fObj = {
field : {},
limit : 100
}
let cObj = {
dbName : 'myNewDatabase',
colName : 'newCollection'
}
async function findDoc(cObj,fObj) {
const url = 'mongodb://localhost:27017';
const client = new MongoClient(url, { useNewUrlParser: true });
try {
await client.connect();
const db = client.db(cObj.dbName);
const col = db.collection(cObj.colName);
console.log(`Connection Made to ${db.databaseName} database.`);
const result = await col.find(fObj.field).limit(fObj.limit).toArray();
client.close();
return result;
} catch (err) {
console.log(err.stack);
}
};
findDoc(cObj,fObj).then(function(result) {
console.log(result);
});
Also, I advise you to enclose your whole async function's body into the try clause. This way you will be able to effectively intercept any error. Imagine your new MongoClient failed to instantiate - you would end up with an uncaught error inside a promise, which isn't very nice.
Related
I am currently trying to return some data from a mongodb database query using callable web firebase functions. The functions are called. However, the fullName function returns {data: null} in my browsers console.
I have also tried return "test" and that also returns the same value of {data: null}
I am completely unsure why this is. And when I attempt to console.log
const data = {
totalInfections: totalInfections,
totalUsers: totalUsers,
totalCommands: totalCommands,
totalLiveInfections: totalLiveInfections,
totalLiveInfectedChannels: totalLiveInfectedChannels
}
console.log(data)
return data
Nothing comes up in my function logs (on my localhost), or in my terminal.
However, if I go to http://localhost:5001/pandemic-8955f/us-central1/fullName I get an error in my function logs which is
function[us-central1-fullName]
{
"severity": "WARNING",
"message": "Request has invalid method. GET"
}
function[us-central1-fullName]
{
"severity": "ERROR",
"message": "Invalid request, unable to process."
}
function[us-central1-fullName]
Finished "us-central1-fullName" in ~1s
I am unsure why this is occuring, and any pointers or advice would be very helpful.
exports.fullName = functions.https.onCall((lol, lol1) => {
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/?readPreference=primary&appname=MongoDB%20Compass&ssl=false";
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("discord_pandemic");
dbo.collection("ArrayStats").find().toArray(function(err, result) {
if (err) throw err;
const totalInfections = result[0].total_infections
const totalUsers = result[0].total_users
const totalCommands = result[0].total_commands
const totalLiveInfections = result[0].total_live_infections
const totalLiveInfectedChannels = result[0].total_live_infected_channels
const data = {
totalInfections: totalInfections,
totalUsers: totalUsers,
totalCommands: totalCommands,
totalLiveInfections: totalLiveInfections,
totalLiveInfectedChannels: totalLiveInfectedChannels
}
console.log(data)
return data
});
});
});
<body>
<input type="text" id="fName" placeholder="First Name">
<input type="text" id="lName" placeholder="Last Name">
<button onClick="addName()"> Add Name </button>
<script>
function addName() {
var fullName = firebase.functions().httpsCallable('fullName');
//For the fullName we have defined that fullName takes some data as a parameter
fullName({}).then((result) => {
console.log(result);
}).catch((error) => {
console.log(error);
})
}
</script>
</body>
There are asynchronous calls in your code, one to open the connection to Mongo, and then another one to load the data from Mongo. By the time the daa has loaded and your return data executes, the close } of your Function has long since executed - and nobody will ever see that return.
In most cases, you can bubble up the promise that you'd typically get back from the API you call when starting an asynchronous operation. But it seems that MongoClient.connect doesn't return a promise, so you'll have to constructor you own like this:
exports.fullName = functions.https.onCall((lol, lol1) => {
return new Promise((resolve, reject) => { // 👈 return a promise, so Cloud Functions knows to wait
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/?readPreference=primary&appname=MongoDB%20Compass&ssl=false";
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("discord_pandemic");
dbo.collection("ArrayStats").find().toArray(function(err, result) {
if (err) throw err;
const totalInfections = result[0].total_infections
const totalUsers = result[0].total_users
const totalCommands = result[0].total_commands
const totalLiveInfections = result[0].total_live_infections
const totalLiveInfectedChannels = result[0].total_live_infected_channels
const data = {
totalInfections: totalInfections,
totalUsers: totalUsers,
totalCommands: totalCommands,
totalLiveInfections: totalLiveInfections,
totalLiveInfectedChannels: totalLiveInfectedChannels
}
console.log(data)
resolve(data); // 👈 resolve the promise with the data for the user
});
});
});
});
Since you're new to this asynchronous behavior, I highly recommend checking out:
The Firebase documentation on when Cloud Functions are terminated.
I'm working on a personal project and am trying to understand the process logic that keeps my Node JS process from terminating after calling populateTransactions().
I think its because I need to close the DB (I'm not entirely clear on why), but when I do, the process terminates but the save() function of the model doesn't complete and the DB isn't written correctly.
When I let the script hang, eventually, it populates the DB correctly, but doesn't terminate.
console.log("This script populates the Transaction collection so that we have some sample data for Issue #31: Uninspected Transactions Component");
let Transaction = require('./models/transaction');
let User = require('./models/user');
let mongoose = require('mongoose');
// let mongoDB = 'mongodb+srv://<username>:<password>#cluster0.dsqmg.mongodb.net/<collection-name>?retryWrites=true&w=majority';
mongoose.connect(mongoDB, {useNewUrlParser: true, useUnifiedTopology: true});
let db = mongoose.connection;
db.on('error', console.error.bind(console, 'MongoDB connection error:'));
async function createTransaction(inspected, recurring, amount, note, startDateString, postDateString) {
let userQuery = await User.find({});
userQuery = userQuery[0];
let startDate = new Date(startDateString);
let postDate = new Date(postDateString);
let transaction = new Transaction({
user: userQuery._id,
inspected: inspected,
recurring: recurring,
amount: amount,
note: note,
startDate: startDate,
postDate: postDate
});
await transaction.save((err) => {
if(err){
console.log(err);
}
});
};
async function populateTransactions(){
await createTransaction(count,false, false, 563, "Numero Uno", "2012-12-05", "2012-12-06");
};
populateTransactions();
Yes, you have to close the connection, because the nodejs process will no finish when still have something is listening (db connection).
The data be not saved to the database, I guess you close db connection before the save action finish (you use callback style right here ??? The await keyword just affects with a Thenable object like a Promise).
...
await transaction.save(); // don't put callback as a parameter, save will return a Promise
await db.close(); // close connection
So I figured out that the issue was originating from
await transaction.save((err) => {
if(err){
console.log(err);
}
});
not following the await behavior. It turned out that the save() function doesn't return a promise if you pass a callback as a parameter, so I refactored the code so that it didn't use a callback and it worked as normal.
I'm trying to scrape a website with Puppeteer. I want to select the date of the last post inserted in my database and compare it to the dates taken by the scrape so I can see if the post is already in the database (using the date as the reference to see if it has been modified).
Here is my code:
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
password: '',
database: 'db_webcrawler_coches'
});
connection.connect((err) => {
if (err) throw err;
console.log('Connected!');
});
let lastpublishedDate;
let idCoches;
connection.query("SELECT id_coches, publish_date FROM coches ORDER BY publish_date DESC limit 1", function (err, row) {
if (err) throw err;
lastPublishedDate = row[0].publish_date;
idCoches = row[0].id_cochesNet;
console.log("Published in", lastPublishedDate);
console.log("Id Coches", idCoches);
});
const run = async () => {
try {
const options = {
headless: false,
};
...
const news = await page.evaluate(() => {
const idsList = [...document.querySelectorAll('div.mt-SerpList-item')].map(elem => elem.getAttribute("id")).filter(elem => elem.includes("#"))
const datePost = [...document.querySelectorAll('span.mt-CardAd-date')].map(elem => elem.innerText);
for(let i = 0; i < titlesCar.length; i++){
const finalDate = parsedDates[i];
if (finalDate > lastPublishedDate || idCoches !== idsList[i]){
console.log("Not repeated");
const carsList[i] = [
idsList[i],
parsedDates[i]
]
} else {
console.log("Repeated")
}
}
return carsList;
});
...
} catch (err) {
console.log(err);
await browser.close();
console.log("Browser Closed");
}
};
run();
As you can see I want to see if the date is the same or not as well as the id taken from the query. However, it appears an error that says Evaluation failed: ReferenceError: variable "lastPublishedDate" is not defined and I imagine that it will be the same with "idCoches". I wrote some console.logs to see when it crashes and it seems that it happens when reaches the function "news".
I'm not sure if it is because it is the scope or because of the function. What do you think I should do to make it work?
Could it be the scope?
Thank you!
EDIT: SOLVED!
I post it in the case that anyone faces a similar issue.
Indeed it was the scope, it is a problem related to Puppeteer. It seems that the function with page.evaluate() is unable to take any variable outside of it. To change it you need to add the page.evaluate in the following way:
await page.evaluate((variable_1, variable_2) => { /* ... */ }, variable_1, variable_2);
The callback to your Query probably does has not returned yet when the async function is run, so whatever your trying to reference is not defined.
I'm not sure if your mysql client supports promises, but if it does you could do something like this:
const run = async () => {
const row = await connection.query("SELECT id_coches, publish_date FROM coches ORDER BY publish_date DESC limit 1")
lastPublishedDate = row[0].publish_date;
idCoches = row[0].id_cochesNet;
...
}
If that does not work you could also run everything inside the callback of the query. Hope that helps.
I am new to JS and async operations. In a router of nodeJS using express, I have aggregated some data from mongo using mongoose. The data is weather data collected from different sites every 15 minutes interval. I processed the data with mongoose aggregate pipeline to get hourly data and group by each site. But the data needs a further process to get periods where for example relative humidity over 90% and assign scores to each period so I wrote some synchronous functions that target each site (each geojson object).
Mongoose looks something like that:
module.exports.filteredData = function (collection, dateInput) {
return collection.aggregate([
{
$addFields :{
DateObj: {
$dateFromString: {
dateString: "$DateTime",
format: '%Y-%m-%d'
}
},
}
},
{
$addFields :{
NewDateTimes: {
$dateFromParts:{
'year': {$year: '$DateObj'},
'month':{$month: '$DateObj'},
'day':{$dayOfMonth: '$DateObj'},
'hour': {$toInt: "$Time"}
}
}
}
}
...
synchronouse functions:
const calcDSV = function(featuresJSON){
// featuresJSON
const SVscore = [];
const tuEval = featuresJSON.features.properties.TU90; // array
const obArr = featuresJSON.features.properties.OB; // array
const periodObj = getPeriods(tuEval);// get period position
const paramObj = getParams(periodObj, obArr); // get parameters
const periodDate = getPeriodDate(featuresJSON, periodObj);
const removeTime = periodDate.beginDate.map(x=>x.split('T')[0]);
let hourly = paramObj.hourCounts;
let avgTemps = paramObj.avgTemps;
for(let i = 0;i<hourly.length; i++){
let score = assignScore(avgTemps[i], hourly[i]);
SVscore.push(score);
}
// output sv score for date
const aggreScore = accumScore(removeTime, SVscore);
aggreScore.DSVdate = aggreScore.Date.map(x=>new Date(x));
featuresJSON.features.properties.periodSV = SVscore;
featuresJSON.features.properties.Periods = periodDate;
featuresJSON.features.properties.DSVscore = aggreScore;
return featuresJSON;
}
Now I am stuck on how to apply those function on each site return by the mongoose aggregate pipeline on a post request:
router.post('/form1', (req, res, next)=>{
const emdate = new Date(req.body.emdate);
const address = req.body.address;
const stationDataCursor = stationData.filteredData(instantData, emdate);
stationDataCursor.toArray((err, result)=>{
if(err){
res.status(400).send("An error occurred in Data aggregation")
};
res.json(result.map(x=>calcDSV.calcDSV(x)));
})
});
I tried in the callback:
stationDataCursor.toArray((err, result)=>{
if(err){
res.status(400).send("An error occurred in Data aggregation")
};
res.json(result.map(async (x)=>await calcDSV.calcDSV(x))));
})
and using then():
stationDataCursor.toArray().then((docArr)=>{
let newfeature = await docArr.map(async (x)=> await calcDSV.calcDSV(x))));
res.json(newfeature);
})
or make calcDSV() returns new promise
return new Promise((rej, res)=>{
resolve(featuresJSON);
})
I would expect to see all sites with a new feature added in the HTTP response output. But most of the time, I got ReferenceError: error is not defined.
I think I have figured it out:
after all, have to make all synchronous functions asynchronous by prepending async to those functions;
rewrite this part in the post router function, especially the array map part. I read from this. and in the map() gonna have try...catch... in it, otherwise it won't work.
await stationDataCursor.toArray().then(async (docArr)=>{
const newfeature = await Promise.all(docArr.map(async function(x){
try{
const feature = await calcDSV.calcDSV(x);
return feature
} catch(err){
console.log("Error happened!!! ", err);
}
}));
res.json(newfeature)
})
Hope it helps.
I have a function that triggers on firebase database onWrite. The function body use two google cloud apis (DNS and Storage).
While the function is running and working as expected (mostly), the issue is that the Socket hang up more often than I'd like. (50%~ of times)
My questions are:
Is it similar to what the rest of the testers have experienced? Is it a well known issue that is outstanding or expected behavior?
the example code is as follows:
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const {credentials} = functions.config().auth;
credentials.private_key = credentials.private_key.replace(/\\n/g, '\n');
const config = Object.assign({}, functions.config().firebase, {credentials});
admin.initializeApp(config);
const gcs = require('#google-cloud/storage')({credentials});
const dns = require('#google-cloud/dns')({credentials});
const zoneName = 'applambda';
const zone = dns.zone(zoneName);
exports.createDeleteDNSAndStorage = functions.database.ref('/apps/{uid}/{appid}/name')
.onWrite(event => {
// Only edit data when it is first created.
const {uid, appid} = event.params;
const name = event.data.val();
const dbRef = admin.database().ref(`/apps/${uid}/${appid}`);
if (event.data.previous.exists()) {
console.log(`already exists ${uid}/${appid}`);
return;
}
// Exit when the data is deleted.
if (!event.data.exists()) {
console.log(`data is being deleted ${uid}/${appid}`);
return;
}
const url = `${name}.${zoneName}.com`;
console.log(`data: ${uid}/${appid}/${name}\nsetting up: ${url}`);
setupDNS({url, dbRef});
setupStorage({url, dbRef});
return;
});
function setupDNS({url, dbRef}) {
// Create an NS record.
let cnameRecord = zone.record('cname', {
name: `${url}.`,
data: 'c.storage.googleapis.com.',
ttl: 3000
});
zone.addRecords(cnameRecord).then(function() {
console.log(`done setting up zonerecord for ${url}`);
dbRef.update({dns: url}).then(res => console.log(res)).catch(err => console.log(err));
}).catch(function(err) {
console.error(`error setting up zonerecord for ${url}`);
console.error(err);
});
}
function setupStorage({url, dbRef}) {
console.log(`setting up storage bucket for ${url}`);
gcs.createBucket(url, {
website: {
mainPageSuffix: `https://${url}`,
notFoundPage: `https://${url}/404.html`
}
}).then(function(res) {
let bucket = res[0];
console.log(`created bucket ${url}, setting it as public`);
dbRef.update({storage: url}).then(function() {
console.log(`done setting up bucket for ${url}`);
}).catch(function(err) {
console.error(`db update for storage failed ${url}`);
console.error(err);
});
bucket.makePublic().then(function() {
console.log(`bucket set as public for ${url}`);
}).catch(function(err) {
console.error(`setting public for storage failed ${url}`);
console.error(err);
});
}).catch(function(err) {
console.error(`creating bucket failed ${url}`);
console.error(err);
});
}
I'm thinking your function needs to return a promise so that all the other async work has time to complete before the function shuts down. As it's shown now, your functions simply returns immediately without waiting for the work to complete.
I don't know the cloud APIs you're using very well, but I'd guess that you should make your setupDns() and setupStorage() return the promises from the async work that they're doing, then return Promise.all() passing those two promises to let Cloud Functions know it should wait until all that work is complete before cleaning up the container that's running the function.