I am Korean and I do not speak English well so I ask for your understanding.
I am creating code that takes a search word and page number and crawls it.
let promises = []; //promise save
let totalData = []; //final data
let makeRequest = function(url) {
return new Promise((resolve, reject) => {
request(url, function(err, response, body){
if(body == null) { return reject('fail'); }
const $ = cheerio.load(body);
let parsingData = $('.g .r a');
console.log(parsingData);
for(let i = 0; i < parsingData.length; i++){
totalData.push(parsingData[i]);
}
return resolve('success');
});
});
}
//promises push
for(let i = 0; i < pageCount; i++){
let url = `https://google.com/search?q=${ searchWord }&start=${i}0`
promises.push(makeRequest(url));
}
(async function(){
try{
await Promise.all(promises);
console.log(totalData);
return res.json(true);
}
catch(error){
console.log(error);
return res.json(false);
}
});
I have confirmed that all promises change from pending to resolved.
enter image description here
enter image description here
I also noticed that the data I imported into the crawl was normal.
enter image description here
But I do not know why async and await do not work. console.log (totalData); does not execute.
This portion:
(async function(){
try{
await Promise.all(promises);
console.log(totalData);
return res.json(true);
}
catch(error){
console.log(error);
return res.json(false);
}
});
Looks as if you're trying to create an IIFE, but in this case you have created an anonymous function but you don't invoke it. So none of the code inside that function will execute. You need to add an extra set of parentheses to the end of that to invoke the function:
(async function(){
try{
await Promise.all(promises);
console.log(totalData);
return res.json(true);
}
catch(error){
console.log(error);
return res.json(false);
}
})(); // <-- note the extra () here
Related
I'm making a application in javascript (Nodejs), I'm kinda new to it. My code needs to do multiple congruent requests, I organized my code in async functions so I can linearly call them
my first code looked like this
async function Fa(param,param1,callback,error){
//SOME CODE
}
async function Fb(param){
//SOME CODE
}
async function Fc(param){
//SOME CODE
}
function Fd(param,callback,error){
//SOME CODE
}
and use it like this
Fa(param,param1,
(result,result1) => {
Fb(resultB) => {
Fc(resultB);
}
},
(error) => { /*handle error*/ }
);
Fd(param,
(result)=>{
//handle result
},
(error)=>{
//handle error
}
)
of course this is not the right way to go for me...
so I got creative and wrote this
async function Fa(param,param1){
var errorFun,resultFun;
function setOnError(error){errorFun = error;}
function setOnResult(result){resultFun = result;}
async function execute(){
//SOME CODE HERE
}
return {setOnError,setOneResult,execute}
//I had to write a execute function because `Fa` being an async function I couldn't access setError and other inner functions from outside
}
I'm not repeating all the functions but I hope you got the idea
so my code looks like this
var resultA,resultA1;
var fa = await Fa(param,param1);
fa.setOnError((error) => /*handle error*/ );
//I want to terminate my code here (all this being in a function) but I don't know how to do so because I can't even set a flag to understand if error function has been called because I have multiple function with error and setting multiple flags would be stupid
fa.setOnResult( (result,result1) => {resultA = result; resultA1 = result1} );
await fa.execute()
var fb = await Fb(param);
fb.setOnResult((result) => {Fc(result);})
await fb.execute();
var fd = await Fd(param);
fd.setOnResult(/*some code*/);
fd.setOnError(/*some code*/);
await fd.execute();
I like my second version more but I don't know how to handle the errror (I want to stop executing the main function) and I think it's a bit overkill..
Any suggestion will be appreciated, thank you
you can try this code.
if execute function throw an error, it will be caught by the try-catch block in the main function
async function Fa(param, param1) {
var errorFun, resultFun;
function setOnError(error) { errorFun = error; }
function setOnResult(result) { resultFun = result; }
async function execute() {
//SOME CODE HERE
if (error) {
throw new Error(error);
}
}
return { setOnError, setOnResult, execute }
}
async function main() {
try {
var fa = await Fa(param, param1);
fa.setOnError((error) => /*handle error*/ );
fa.setOnResult((result, result1) => { resultA = result; resultA1 = result1 });
await fa.execute();
var fb = await Fb(param);
fb.setOnResult((result) => { Fc(result); });
await fb.execute();
var fd = await Fd(param);
fd.setOnResult(/*some code*/);
fd.setOnError(/*some code*/);
await fd.execute();
} catch (error) {
// handle error
}
}
Consider these two functions when i call
getStatusAll(data)-
data=[[],['1'],['2'],['3']];
async function getStatusAll(data) {
console.log("In getStatusAll");
try{
let statusPromiseArray = data.map(async(value) => {
result= await this.fetchStatusDBs(value);
return result;
});
statusResolvedArray= await Promise.all(statusPromiseArray)
return statusResolvedArray;
}catch(err){
throw(err);
}
}
async function fetchStatusDBs(data) {
console.log("In fetchStatusDBs");
try{
//fetch status in dvf_req_id for an dvf_req_id
if(data.length==0){
console.log("1");
dvfStatus = await Promise.resolve("Disabled");
console.log("2");
trainingStatus = await Promise.resolve("Disabled");
console.log("3");
inferenceStatus = await Promise.resolve("Disabled");
}
else {
console.log("4");
dvfStatus = await Promise.resolve("Enabled");
console.log("5");
trainingStatus = await Promise.resolve("Enabled");
console.log("6");
inferenceStatus = await Promise.resolve("Enabled");
}
return [dvfStatus,trainingStatus,inferenceStatus];
}catch(err){
throw(err);
}
}
I am trying to resolve multiple Promises within a Promise.all
but the results is unexpected.
Actual Output-
In getStatusAll
In fetchStatusDBs
1
In fetchStatusDBs
4
In fetchStatusDBs
4
In fetchStatusDBs
4
2
5
5
5
3
6
6
6
[["Enabled","Enabled","Disabled"],["Enabled","Enabled","Enabled"],["Enabled","Enabled","Enabled"],["Enabled","Enabled","Enabled"]]
Expected Output-
In getStatusAll
inside map
In fetchStatusDBs
1
2
3
inside map
In fetchStatusDBs
4
5
6
inside map
In fetchStatusDBs
4
5
6
inside map
In fetchStatusDBs
4
5
6
[["Disabled","Disabled","Disabled"],["Enabled","Enabled","Enabled"],["Enabled","Enabled","Enabled"],["Enabled","Enabled","Enabled"]]
But changing fetchStatusDBs like this returns output in the correct format.
async function fetchStatusDBs(data) {
console.log("In fetchStatusDBs");
try{
if(data.length==0){
dvfStatus = "Disabled";
trainingStatus = "Disabled";
inferenceStatus = "Disabled";
}
else {
dvfStatus = "Enabled";
trainingStatus = "Enabled";
inferenceStatus = "Enabled";
}
return [dvfStatus,trainingStatus,inferenceStatus];
}catch(err){
throw(err);
}
}
Can somebody help me out?
You have several misunderstandings about async-await
async function getStatusAll(data) {
console.log("In getStatusAll");
try{
let statusPromiseArray = data.map(async(value) => { // map is sync
result= await this.fetchStatusDBs(value); // global result
return result; // return-await anti-pattern
});
statusResolvedArray= await Promise.all(statusPromiseArray) // global
return statusResolvedArray; // return-await anti-pattern
}catch(err){ // catch-throw anti-pattern
throw(err);
}
}
All of that can be written as -
function getStatusAll(data) {
return Promise.all(data.map(v => this.fetchStatusDBs(v)))
}
And any error will automatically bubble up. No need to catch and re-throw. This will do all the fetches in parallel. You could do the requests in serial if you wanted. This is shown to demonstrate proper use of await in async -
async function getStatusAll(data) {
const result = []
for (const value of data)
result.push(await this.fetchStatusDBs(value))
return result
}
Your main problem is that the status variables in fetchStatusDBs are undeclared and therefore implicitly global. Since your code is making multiple concurrent calls, which are interleaved at the await suspension points, they affect each other - a classic race condition. To fix this, just declare the variables with let or var.
const data=[[],['1'],['2'],['3']];
async function getStatusAll(data) {
console.log("In getStatusAll");
let statusPromiseArray = data.map(this.fetchStatusDBs, this);
return Promise.all(statusPromiseArray);
}
async function fetchStatusDBs(data) {
console.log("In fetchStatusDBs");
let dvfStatus, trainingStatus, inferenceStatus;
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
//fetch status in dvf_req_id for an dvf_req_id
if (data.length==0) {
console.log("1");
dvfStatus = await Promise.resolve("Disabled");
console.log("2");
trainingStatus = await Promise.resolve("Disabled");
console.log("3");
inferenceStatus = await Promise.resolve("Disabled");
} else {
console.log("4");
dvfStatus = await Promise.resolve("Enabled");
console.log("5");
trainingStatus = await Promise.resolve("Enabled");
console.log("6");
inferenceStatus = await Promise.resolve("Enabled");
}
return [dvfStatus, trainingStatus, inferenceStatus];
}
This will have the desired results (return value). To also get the expected output from the logs, you cannot use map with an asynchronous function and then wait for all the promises with Promise.all, but you need to make the calls in sequence:
async function getStatusAll(data) {
console.log("In getStatusAll");
const statusArray = [];
for (const value of data) {
// ^^^^^^^^^^^^^^^^^^^^^^^^^
statusArray.push(await this.fetchStatusDBs(value));
// ^^^^^
}
return statusArray ;
}
I am trying to loop through records in a database, in order to compile an array (cardsToInsert) that I will write to another database.
I was getting stuck because the array was writing to the database before the loop finished, I know I need to use promises / async functions to achieve what I want, but I'm pretty sure I'm doing something wrong with my promises.
The code works for a few loops (it goes for about 6-10 loops, it's supposed to loop 16 times), but then hangs while trying during wixData.get (or it hangs on a different promise that is part of buildCard).
// wixData.get is a function that returns a promise
async function loopCards(cardsToGet) {
let writeCard
let buildCard
for (let index = 0; index < cardsToGet.length; index++) {
const cardToGet = cardsToGet[index].card
buildCard = await wixData.get("Card", cardToGet)
.then((card) => {
return card
})
.catch((err) => {
let errorMsg = err;
return errorMsg
});
writeCard = await buildingCard(buildCard)
cardsToInsert.push(writeCard)
}
return cardsToInsert
}
What am I doing wrong? (or what is the key thing I'm doing wrong that is stopping this working, I'm sure there is plenty to be improved here!)
UPDATE
I've now updated the code and it loops through fine.
async function loopCards(cardsToGet) {
console.log('Start')
let writeCard
let buildCard
for (let index = 0; index < cardsToGet.length; index++) {
const cardToGet = cardsToGet[index].card
buildCard = wixData.get("Card", cardToGet)
.then(async (card) => {
writeCard = await buildingCard(card)
cardsToInsert.push(writeCard)
})
.catch((err) => {
let errorMsg = err;
return errorMsg
});
}
return cardsToInsert
}
How do I get it to wait for the loop to finish before finally returning cardsToInsert?
Your mix of async/await and .then is not really best practice
This should work, and will return once cardsToInsert is populated
async function loopCards(cardsToGet) {
const cardsToInsert = [];
for (let cardToGet of cardsToGet) {
try {
const card = await wixData.get("Card", cardToGet);
const writeCard = await buildingCard(card);
cardsToInsert.push(writeCard);
}
catch(err) {
let errorMsg = err;
return errorMsg;
}
}
return cardsToInsert;
}
better still, you really don't need to handle any errors here, since the calling function could do that
So it becomes even simpler
async function loopCards(cardsToGet) {
const cardsToInsert = [];
for (let cardToGet of cardsToGet) {
const card = await wixData.get("Card", cardToGet);
const writeCard = await buildingCard(card);
cardsToInsert.push(writeCard);
}
return cardsToInsert;
}
then using it could be like
loopCards(cards)
.then(result => doSomethingWihtResult)
.catch(error => handleError);
or if calling from an async function
try {
let result = await loopCards(cards);
// do something with result
} catch(error) {
// handle Error
}
While making a link Shortner script using nodejs, i encountered the following problem:
my program went on an infinite loop for a reason i ignore
here is the code:
function makeShort() {
var short = "";
var cond = true;
while(cond){
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for(var i = 0; i < length; i++){
short += possible.charAt(Math.floor(Math.random() * possible.length));
}
let query = {short:short};
Link.findOne(query, (err, link)=>{
if(err) throw err;
if(!link){
console.log("here");
cond = false;
}
});
}
return short;
}
to then use it here :
router.post('/', (req, res)=>{
let short = makeShort();
const newLink = new Link({
url: req.body.url,
short:short
});
newLink.save().then(link => {
res.json(link);
});
});
the idea is that i generate a random string (5 characters), and then, if it exists i create another one and so on.. until i find one that isn't used ( the database is empty btw so there is no reason for it to go infinite loop ).
You can loop over and test for values in your database using async/await. What we do is convert your function to an async function, then create a new function that will return a promise which will resolve true/false.
Next we call that function in the while loop and await for a result which will contain true/false we then set that to the variable cond and continue the loop.
It would look something like this:
async function makeShort(length) {
let cond = true;
while (cond) {
let short = (Math.random() * 1000).toString(32).replace(/\./g, '').substr(0, length);
let query = { short: short };
cond = await findOne(query);
}
return short;
}
function findOne(query) {
return new Promise(resolve => {
Link.findOne(query, (err, link) => {
if (err) resolve(false);
if (!link) {
return resolve(false);
}
return resolve(true);
});
})
}
We then can call it using let short = await makeShort() like this (we also have to the make (req, res) function async):
router.post('/', async (req, res) => {
let short = await makeShort();
const newLink = new Link({
url: req.body.url,
short: short
});
newLink.save().then(link => {
res.json(link);
});
});
Don't mix synchronous looping and asynchronous condition updating. Something like this is guaranteed to run the while body as many times as it can before that DoSomething call returns a result:
while(cond) {
// call something async. don't wait for a result.
DoSomething.asynchronous( () => { cond = false; });
// then immediately restart the iteration
}
So don't do that. Make your makeShort generate a shortform string asynchronously.
const symbols = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
const symbolCount = symbols.length;
function makeShort(howMany) {
howMany = howMany || 5;
let short = "";
while(howMany--) {
short += symbols[(Math.random() * symbolCount)|0];
}
return short;
}
Then, do your verification independently of this:
function assignShortForm(req, res) {
let short = makeShort();
verifyShortIsAvailable(
short,
success => {
// this short form was available
new Link({ url: req.body.url, short }).save().then(link => res.json(link));
}, error => {
// try again. RNG is not your friend, and this COULD run a very long time.
assignShortForm(req, res);
}
);
}
With your router using that function, not inlining it:
router.post('/', assignShortForm);
In this, verifyShortIsAvailable should do its work asynchronously:
verify verifyShortIsAvailable(short, resolve, reject) {
Link.findOne(query, (err, link) => {
if (err) return reject(err);
if (link) return reject("...");
return resolve();
});
}
while loops run synchronously, meaning they block the thread from further execution until they are complete. Because the link shortener is asynchronous, it is being blocked by the while loop.
To handle this code asynchronously, you can return a Promise
function makeShort() {
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
// note length was previously undefined in the comparison. use possible.length or another arbitrary value
for(var i = 0; i < possible.length; i++){
short += possible.charAt(Math.floor(Math.random() * possible.length));
}
let query = {short:short};
return new Promise((resolve, reject) => {
Link.findOne(query, (err, link) => {
if(err) return reject(err);
resolve(link)
});
})
}
Then you can use it like so...
let short = makeShort().then(shortLink => {
// do something with the link
}).catch(err => {
// handle the error
});
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Using_promises
async lsEntered(){
if(this.service.wd == '')
{
await basic((this.service.wd));
}
else
{
await basic(('/'+this.service.wd));
}
this.files = await JSON.parse(localStorage.getItem('FILENAMES'));
var filesList = document.getElementById(this.trackLine.toString());
var li;
for (var i = 0; i < this.files.length; i++) {
li = document.createElement('li');
li.appendChild(document.createTextNode(this.files[i].name));
filesList.appendChild(li);
}
localStorage.clear();
}
I want to wait until basic is finished and JSON.parse finishes before displaying the values in the DOM. I'm getting the values of the previous call every time which is tell me the async is no working. To be fair I don't have tons of TS experience.
Edit: This is basic I was hoping not to have to deal with it as it's a javascript function and fragily integrated into this app.
var basic = function (path) {
var ACCESS_TOKEN = '';
var dbx = new Dropbox({ accessToken: ACCESS_TOKEN });
dbx.filesListFolder({ path: path })
.then(function (response) {
localStorage.setItem('FILENAMES',JSON.stringify(response.entries));
console.log(response);
})
.catch(function (error) {
console.error(error);
});
return false;
}
let myPromise = new Promise((resolve, reject) => {
// Work you want to execute
resolve("I am done");
});
myPromise.then((successMessage) => {
// successMessage is whatever we passed in the resolve(...) function above.
console.log("Yay! " + successMessage);
});
You can only await promises1 and basic is not returning a promise. You need to do
return dbx.filesListfolder(...)...
Also consider what Bergi said in their comment.
1: Actually, you can await any value, but there is no point in awaiting something that is not a promise. By not returning the promise from basic, lsEntered won't wait for the local storage to be set.