We Keep Coding

Data not returning from async function with database connection

The goal is to call a function from my main script that connects to a database, reads a document from it, stores pieces of that document in a new object, and returns that object to my main script. The problem is I cannot get it all to work together. If I try one thing, I get the results but my program locks up. If I try something else I get undefined results.
Long story short, how do I open a database and retrieve something from it to another script.
The program is a quiz site and I want to return the quiz name and the questions.
const myDb = require('./app.js');
var myData = myDb.fun((myData) => {
console.log(myData.quizName);
});
Here is the script that tries to open the database and find the data
const { MongoClient } = require("mongodb");
const {mongoClient} = require("mongodb");
const uri = connection uri goes here but my name is hard coded into it at the moment so I removed for privacy
const client = new MongoClient(uri);
const fun = async (cback) => {
try {
await client.connect();
const database = client.db('Quiz-Capstone');
const quizzes = database.collection('Quiz');
const query = {quizName: "CIS01"};
const options = {
sort: {},
projection: {}
};
const quiz = await quizzes.findOne(query, options);
var quizObject = {
quizName: quiz.quizName,
quizQuestions: quiz.quizQuestions
}
//console.log(testOb);
} finally {
await client.close();
cback(quizObject);
}
}
fun().catch(console.dir);
module.exports = {
fun: fun
}
UPDATE: Still stuck. I have read several different threads here about asynchronous calls and callbacks but I cannot get my function located in one file to return a value to the caller located in another file.

NodeJS Async Programming

Totally new to programming with async functions. Also new to node.js which could be adding to my issue. I've read a lot and keep running into the similar problems and it seems like I've been randomly getting some portions of the async code to work, while others doesn't. Here is a simplified version of what I have:
Essentially I'm searching a site for music, scrapping all the results (scraper_start.js) and then it is sent to scrape_individual.js to gather data. It is currently able to get all the data, but when it downloads the album art it comes in "too late".
The image does get logged to the console, but only after info gets returned. Also if you have any good resources to learn async programming please share them - I haven't been able to find a website that is nice and clean and gets into examples big enough that they become realistic (such as multiple async functions working at once and sometimes relying upon each other). Please critic my code as well - I am trying to learn!
File scraper_start.js:
const rp = require('request-promise');
const cheerio = require('cheerio');
const scrape = require('./scrape_individual.js');
const base_url = 'https://www.test.ca';
const url = 'https://www.test.ca/search?mysearchstring';
rp(url)
.then(function(html)
{
const $ = cheerio.load(html);
var results = []
var hits = $('h3 > a').length;
console.log("TOTAL HITS: " + hits);
results = $('h3 > a').map(function(i,v){ return $(v).attr('href'); }).get()
return Promise.all(
results.map(function(url)
{
return scrape(base_url + url);
})
);
})
.then(function(my_data)
{
console.log(my_data);
});
File scrape_individual.js:
const rp = require('request-promise');
const cheerio = require('cheerio');
var info = {}
const scrape = function(url)
{
return rp(url)
.then(function(html)
{
const $ = cheerio.load(html);
if (!html.includes('contentType = "Podcast"'))
{
info = {
title: $('h2.bc-heading:first').text(),
img3: null};
img_data($('.bc-image-inset-border').attr('src'))
.then(function(v)
{
console.log(v);
info.img3 = v; // Log the value once it is resolved
})
.catch(function(v) {
});
return info;
}
})
};
function img_data(src)
{
return new Promise(function(resolve, reject)
{
const { createCanvas, loadImage } = require('canvas');
loadImage(src).then((image) =>
{
const canvas = createCanvas(image.width, image.height);
const ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0);
resolve(canvas.toDataURL());
});
});
}
module.exports = scrape;
UPDATE: New Code with ASYNC / AWAIT
scraper_start.js:
const rp = require('request-promise');
const cheerio = require('cheerio');
const scrape = require('./scrape_individual.js');
const base_url = 'https://www.test.ca';
const url = 'https://www.test.ca/search?mysearchstring';
var data = [];
async function get_links(url)
{
let html = await rp(url);
const $ = cheerio.load(html);
var results = [];
var hits = $('h3 > a').length;
console.log("TOTAL HITS: " + hits);
hrefs = $('h3 > a').map(function(i,v){ return $(v).attr('href'); }).get()
await Promise.all(hrefs.map(async (href) =>
{
let data_single = await scrape.scrape_book3(base_url + href);
data.push(data_single);
}));
//QUESTION AREA 1: This data works great with all info.
console.log(data);
return data
}
get_links(url);
//QUESTION AREA 2: This data gets printed before getting the actual data returned.
console.log(data);
scrape_individual.js:
const rp = require('request-promise');
const cheerio = require('cheerio');
var info = {}
//scrape2(url)
module.exports.scrape_individual = scrape2;
async function scrape2(url)
{
let html = await rp(url);
const $ = cheerio.load(html);
if (!html.includes('contentType = "Podcast"'))
{
let my_image = await img_data($('.bc-image-inset-border').attr('src'));
info = {title: $('h2.bc-heading:first').text(),
img3: my_image};
//console.log(info);
return info;
}
}
async function img_data(src)
{
const { createCanvas, loadImage } = require('canvas');
let image = await loadImage(src);
const canvas = createCanvas(image.width, image.height);
const ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0);
//console.log(canvas.toDataURL());
return canvas.toDataURL();
}
This code works great now. Also easier to understand. Please feel free to critic as I am trying to master this. My question now is more of a general coding question.
Within scraper_start.js where the end result ends up (data), I marked two comments with "QUESTION AREA 1" and "QUESTION AREA 2"
QUESTION AREA 1: works entirely fine, which I would assume because it is in the async function
QUESTION AREA 2: outside of async function, does not have the object returned yet as there is nothing to say await. Is there a way to make it wait?
My question is pretty loaded. I can't use await since it's not in an async function from my understanding. Does this mean all my code needs to be in functions if I want to maintain an important order? What is best practice? Why not call every function as async?
edit: Fixing typos
edit2: Added ASYNC / AWAIT modifications

Question Area 2 is returning before the data is captured because you are telling JS that the last function is not waiting on anyone, so it is run synchronously.
But the truth is that it is actually waiting for someone, it is waiting for get_links()
So a way to get to print the data would be:
async printer(){
const returnedData = await get_links(URL);
console.log(returnedData)
}
printer();
if you want to use the data returned from an async function you need to call it with await, so JS knows that it needs to wait for a resolved or rejected promise before going on, or else it will return a Promise<pending>. And all await need to be inside an async.
In the beginning, sounds like an endless circle but it really is not.
For instance in our example you don't need any other async to call the printer() (because no other function is depending on that one)
I hope it makes sense, but promises at the beginning need some time to be digested before understanding them.
Async/await in my opinion are a blessing to understand promises, once you get your head around it and how they function you are better prepared to understand the resolve/reject way

AWS Lambda - Only getting answer after the second test trigger

I'm developing an AWS Lambda in TypeScript that uses Axios to get data from an API and that data will be filtered and be put into a dynamoDb.
The code looks as follows:
export {};
const axios = require("axios");
const AWS = require('aws-sdk');
exports.handler = async (event: any) => {
const shuttleDB = new AWS.DynamoDB.DocumentClient();
const startDate = "2021-08-16";
const endDate = "2021-08-16";
const startTime = "16:00:00";
const endTime = "17:00:00";
const response = await axios.post('URL', {
data:{
"von": startDate+"T"+startTime,
"bis": endDate+"T"+endTime
}}, {
headers: {
'x-rs-api-key': KEY
}
}
);
const params = response.data.data;
const putPromise = params.map(async(elem: object) => {
delete elem.feat1;
delete elem.feat2;
delete elem.feat3;
delete elem.feat4;
delete elem.feat5;
const paramsDynamoDB = {
TableName: String(process.env.TABLE_NAME),
Item: elem
}
shuttleDB.put(paramsDynamoDB).promise();
});
await Promise.all(putPromise);
};
This all works kind of fine. If the test button gets pushed the first time, everything seems fine and is working. E.g. I received all the console.logs during developing but the data is not put into the db.
With the second try it is the same output but the data is successfully put into the Db.
Any ideas regarding this issue? How can I solve this problem and have the data put into the Db after the first try?
Thanks in advance!

you need to return the promise from the db call -
return shuttleDB.put(paramsDynamoDB).promise();
also, Promise.all will complete early if any call fails (compared to Promise.allSettled), so it may be worth logging out any errors that may be happening too.
Better still, take a look at transactWrite - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/DocumentClient.html#transactWrite-property to ensure all or nothing gets written

How to insert params into netlify functions?

I have an input that sends an api call on submit to the unsplash API. I am trying to convert this into a netlify function but am not sure how to pass the params from the input into the function. I am trying to keep the API key hidden. I've never worked with the qs package and looked up the docs but have not been able to quite figure it out.
script.js
const KEY = "" //secret
const URL = `https://api.unsplash.com/search/photos?page=1&per_page=50&client_id=${process.env.KEY}`;
const input = document.querySelector(".input");
const form = document.querySelector(".search-form");
const background = document.querySelector(".background");
const overlay = document.querySelector(".overlay");
const header = document.querySelector(".title");
let results = [];
search = (searchTerm) => {
let url = `${URL}&query=${searchTerm}`;//this should hit the netlify endpoint instead
return fetch(url)
.then((response) => response.json())
.then((result) => {
toggleStyles();
header.appendChild(form);
result.results.forEach((image) => {
const galleryItem = document.createElement("div");
galleryItem.className = "gallery-item";
const imageDiv = document.createElement("div");
imageDiv.className = "image-div";
document.querySelector(".results-page").appendChild(galleryItem);
galleryItem.appendChild(imageDiv);
imageDiv.innerHTML =
"<img class='image' src=" + image.urls.regular + ">";
form.classList.remove("toggle-show");
input.classList.add("header-expanded");
form.addEventListener("submit", (e) => {
e.preventDefault();
document.querySelector(".results-page").remove();
});
});
console.log(result.results);
return results;
});
};
toggleStyles = () => {
const resultsContainer = document.createElement("div");
resultsContainer.className = "results-page";
document.body.appendChild(resultsContainer);
};
input.addEventListener("focus", (e) => {
e.preventDefault();
input.style = "font-family: 'Raleway', sans-serif";
input.placeholder = "";
});
input.addEventListener("blur", (e) => {
e.preventDefault();
input.style = "font-family: FontAwesome";
input.value = "";
input.placeholder = "\uf002";
});
form.addEventListener("submit", (e) => {
e.preventDefault();
let searchTerm = input.value;
search(searchTerm);
});
token-hider.js
const axios = require("axios");
const qs = require("qs");
exports.handler = async function (event, context) {
// apply our function to the queryStringParameters and assign it to a variable
const API_PARAMS = qs.stringify(event.queryStringParameters);
console.log("API_PARAMS", API_PARAMS);
// Get env var values defined in our Netlify site UI
// TODO: customize your URL and API keys set in the Netlify Dashboard
// this is secret too, your frontend won't see this
const { KEY } = process.env;
const URL = `https://api.unsplash.com/search/photos?page=1&per_page=50&client_id=${KEY}`;
console.log("Constructed URL is ...", URL);
try {
const { data } = await axios.get(URL);
// refer to axios docs for other methods if you need them
// for example if you want to POST data:
// axios.post('/user', { firstName: 'Fred' })
return {
statusCode: 200,
body: JSON.stringify(data),
};
} catch (error) {
const { status, statusText, headers, data } = error.response;
return {
statusCode: error.response.status,
body: JSON.stringify({ status, statusText, headers, data }),
};
}
};
I added the KEY as an Environment variable in my netlify UI, and am able to hit the function's endpoint. Any help is greatly appreciated as I am new to serverless functions but want to learn JAMstack really badly.

I think you are asking two different things here - how to use secret keys in Netlify functions and how to pass parameters to it from your front end code.
You can define environment variables in your Netlify site settings. These variables can then be used in your Netlify functions via process.env. So if you called your variable SECRET_KEY, then your Netlify function code (not your front end code!) would be able to read it via process.env.SECRET_KEY.
Looking over your code, it looks like you understand that as you have it in your function, but you also try to use it in the client-side code. You can remove that.
Your code gets the query string parameters, and it looks like you just need to add them to the end of the URL you hit. Did you try that?

How to wait for the promise when using get in Firestore

I am just trying a simple get command with Firestore, using this code from Google it doesn't work because it's not waiting for the promise?
Earlier I had put only a snippet of code, this is the entirety of index.js -- I'm using Firestore with Dialogflow to build a Google Assistant app and trying to call a function from the welcome intent that gets a field from Firestore, then writes that field to a string (named question1), and then this string should be spoken by the assistant as part of the ssml response. I've been on this for at least 30 hours already, can't seem to comprehend promises in regards to intents, firestore, etc. I've tried about 10 different solutions, this one works, only it says "undefined" in other variations I have tried it would say undefined several times but after 2-3 passes the get command would be complete and then the variable would be read out. I'm just trying to figure out how to get the get command and variable set before moving onto the SSML response. Can anyone point me in the right direction?
'use strict';
const functions = require('firebase-functions'); //don't forget this one
// Import Admin SDK
var admin = require("firebase-admin");
admin.initializeApp(functions.config().firebase);
var db = admin.firestore();
const collectionRef = db.collection('text');
const Firestore = require('#google-cloud/firestore');
var doc;
var question1;
const url = require('url');
const {
dialogflow,
Image,
Permission,
NewSurface,
} = require('actions-on-google');
const {ssml} = require('./util');
const config = functions.config();
const WELCOME_INTENT = 'Default Welcome Intent';
const app = dialogflow({debug: true});
async function dbaccess(rando) {
console.log("dbaseaccess started")
var currentquestion2 = 'question-' + rando.toString();
var cityRef
try { return cityRef = db.collection('text').doc(currentquestion2).get();
console.log("get command completed")
//do stuff
question1 = cityRef.data().n111
} catch(e) {
//error!
}
console.log("one line above return something");
return rando;
}
app.fallback((conv) => {
// intent contains the name of the intent
// you defined in the Intents area of Dialogflow
const intent = conv.intent;
switch (intent) {
case WELCOME_INTENT:
var rando = Math.floor(Math.random() * 3) + 1;
dbaccess(rando);
const ssml =
'<speak>' +
question1 +
'</speak>';
conv.ask(ssml);
break;
exports.dialogflowFirebaseFulfillment = functions.https.onRequest(app);

You have 2 options: you can use async/await or you can use Promise.then() depending on how you want the code to execute.
Async/await:
async function databasetest {
var cityRef;
try{
cityRef = await db.collection('cities').doc('SF');
// do stuff
} catch(e) {
// error!
}
Promise.then():
db.collection('cities').doc('SF').then((cityRef) => {
cityRef.get()
.then(doc => { /* do stuff */ })
.catch(err => { /* error! */ });
});

maybe a little of work around could help you, I'm not sure yet how you are trying to implement it.
function databasetest () {
var cityRef = db.collection('cities').doc('SF');
return cityRef.get()
}
// so you can implement it like
databasetest().then(doc => {
if (!doc.exists) {
console.log('No such document!');
} else {
console.log('Document data:', doc.data());
}
})
.catch(err => {
console.log('Error getting document', err);
});
More context would help to understand your use case better :)