I'm having a problem where for(var x=1; x < 6; x++) is getting called because too fast axios.get() is async, but I have no idea how to counter that without the solution being too complicated
const axios = require("axios");
const cheerio = require("cheerio");
function imdbGetData(id) {
var title, show, $;
var arr = [];
var airdates = [];
show = {
seasons: []
};
axios.get(`http://www.imdb.com/title/${id}/`).then((body) => {
$ = cheerio.load(body.data);
title = $("div h1").text()
});
for(var x=1; x < 6; x++) {
console.log(x); // Will count too 1,2,3,4,5,6
url = `http://www.imdb.com/title/${id}/episodes?season=${x}`
axios.get(url).then((body) => {
$ = cheerio.load(body.data);
console.log(x);// 6, 6, 6, 6
$("div .info .airdate").each(function(index, item) {
var airdate = String($(this).text());
airdates.push(airdate.trim());
});
$(".info strong a").each(function(i, item){
var airdate = airdates[i];
var epsiode_name = $(this).text()
if (epsiode_name && !epsiode_name.includes("#"))
arr.push({epsiode_name, airdate});
});
show.seasons.push(arr);
arr = []
// console.log(show.seasons);
});
setTimeout(() => {console.log(show.seasons)}, 10000) // ghetto
}
}
// season = {
// seasons: [[ {epsiode_name} ], [{Epsiode name}]]
// }
imdbGetData("tt2193021");
You can construct and push all promises to array, and then use Promise.all(arrayOfPromises). This way you will keep your asynchronous chain and you can easily handle results very similar to regular single asynchronous operation:
var promises = [];
for (var x = 1; x < 6; x++) {
url = `http://www.imdb.com/title/${id}/episodes?season=${x}`
promises.push(axios.get(url));
}
Promise.all(promises)
.then(body => {
// all results of promises will be in 'body' parameter
})
.catch(err => console.error(err));
You can also use async/await (in newer versions of Node.js), so you can make the code a little easier to read, I've made a few little changes to update progress too.
const axios = require("axios");
const cheerio = require("cheerio");
async function imdbGetData(id) {
var title, show, $;
var arr = [];
var airdates = [];
show = {
seasons: []
};
console.log('Getting from ' + `http://www.imdb.com/title/${id}/`);
let body = await axios.get(`http://www.imdb.com/title/${id}/`);
$ = cheerio.load(body.data);
title = $("div h1").text()
for(var x=1; x < 6; x++) {
console.log('Getting season: ' + x); // Will count too 1,2,3,4,5,6
url = `http://www.imdb.com/title/${id}/episodes?season=${x}`
let body = await axios.get(url);
$ = cheerio.load(body.data);
$("div .info .airdate").each(function(index, item) {
var airdate = String($(this).text());
airdates.push(airdate.trim());
});
$(".info strong a").each(function(i, item){
var airdate = airdates[i];
var epsiode_name = $(this).text()
if (epsiode_name && !epsiode_name.includes("#"))
arr.push({epsiode_name, airdate});
});
show.seasons.push(arr);
arr = []
}
console.log("Result: ", show.seasons);
}
imdbGetData("tt2193021");
You can simply use ES6 let instead of var , your code will be:
for(let i=0; i<length; i++){
asyncCall(function(){
console.log(i);// will print 0,1,2,3,...
});
}
Please check this article https://codeburst.io/asynchronous-code-inside-an-array-loop-c5d704006c99
Related
I'm doing an operation inside my function and I want to store my JSON data with AsyncStorage and use it elsewhere, but I'm getting an error in react native
this is my code block;
onPress={() => {
press = item.id;
// console.warn(press);
ars = options;
dd = JSON.stringify(ars);
cc = JSON.parse(dd);
for (var i = 0; i < cc.length; i++) {
if (cc[i].id == press) {
// console.warn(cc[i]);
var productData = cc[i];
var stri = JSON.stringify(cc[i]);
AsyncStorage.setItem('ProductData', stri);
var abc = AsyncStorage.getItem('ProductData');
console.warn(stri);
console.warn(abc);
}
}
}}>
how can i solve that problem?
thanks.
var abc = await AsyncStorage.getItem('ProductData');
add await as it is promise that all
so whole code will look like this
onPress={async () => {
press = item.id;
// console.warn(press);
ars = options;
dd = JSON.stringify(ars);
cc = JSON.parse(dd);
for (var i = 0; i < cc.length; i++) {
if (cc[i].id == press) {
// console.warn(cc[i]);
var productData = cc[i];
var stri = JSON.stringify(cc[i]);
AsyncStorage.setItem('ProductData', stri);
var abc =await AsyncStorage.getItem('ProductData');
console.warn(stri);
console.warn(abc);
}
}
}}>
add async to function and add await
async postList(arr){
console.log(arr);
console.log(arr[0]);
Debugger + console picture: https://prnt.sc/23q4jra
as seen in the picture of the debugger console.log(arr) returns an array with a value in the [0] position, the very next line arr[0] returns "undefined" and arr.length returns "0"
how is it possible?
the function that calls this function:
async mountedCall(){
var composedArray = await this.createList();
document.getElementById('listWrap_players').appendChild(await this.postList(composedArray));
},
createList():
async createList(){
var composedArray = [];
const id = document.getElementById('joinCode').innerHTML;
var player_count = null;
await firebase.database().ref('lobbies/' + id + '/playerCount/').once('value', (snapshot) => {
const data = snapshot.val();
player_count = data;
}).then(function() {
for(var i = 1; i <= player_count; i++){
var iStr = String(i);
const player_names_snapshot = firebase.database().ref('lobbies/' + id + '/players/' + iStr);
player_names_snapshot.once('value', (snapshot) => {
const data = snapshot.val();
composedArray.push(data);
}).then(function(){return;});
}
});
this.isLeader(id);
return composedArray;
},
UPDATE:
Tried to replace console.log with console.log(JSON.stringify(arr)) as suggested below
console.log(JSON.stringify(arr)) returns an empty array so I think it means I have synchronization problem in createList() or in mountedCall(), yet I cant seem to find it. I've used await and .then() in every location possible...
here are all the functions together:
async mountedCall(){
var composedArray = await this.createList();
document.getElementById('listWrap_players').appendChild(await this.postList(composedArray));
},
async removeAllChildNodes(list) {
while(list.firstChild){
list.removeChild(list.firstChild);
}
},
async postList(arr){
console.log(JSON.stringify(arr));
console.log(arr[0]);
var list = document.createElement('ul');
for(let i = 0; i < arr.length; i++){
var item = document.createElement('li');
item.appendChild(document.createTextNode(arr[i]));
list.appendChild(item);
}
const listContainer = document.getElementById('listWrap_players');
this.removeAllChildNodes(listContainer);
return list;
},
async createList(){
var composedArray = [];
const id = document.getElementById('joinCode').innerHTML;
var player_count = null;
await firebase.database().ref('lobbies/' + id + '/playerCount/').once('value', (snapshot) => {
const data = snapshot.val();
player_count = data;
}).then(function() {
for(var i = 1; i <= player_count; i++){
var iStr = String(i);
const player_names_snapshot = firebase.database().ref('lobbies/' + id + '/players/' + iStr);
player_names_snapshot.once('value', (snapshot) => {
const data = snapshot.val();
composedArray.push(data);
}).then(function(){return;});
}
});
return composedArray;
},
I'm new to promises and I know the issue is that I'm trying to read the array before the promises have resolved. If I just console.log the files they print fine, just need them in an array so I can iterate through them.
Can someone nudge me in the right direction? I'm using the recursive-readdir node module to grab a list of all files within the dir1 and dir2 directories and add them to an array (var sitemap).
var recursive = require("recursive-readdir");
var directories = ['dir1', 'dir2'];
var sitemap = [];
for(var i = 0; i < directories.length; i++) {
recursive(directories[i], ["*.pdf", "*.json", "*.xml", "*.jpg", "*.gif", "*.png", "*.css", "*.js", "*.flv", "*.swf", "*.ttf", "*.pptx", "*.doc", "*.docx"], function (err, files) {
for(var x = 0; x < files.length; x++) {
let url = 'example.com/' + files[x];
addURL(url);
}
});
}
function addURL(url) {
sitemap.push(url);
}
// Returns empty array
console.log(sitemap);
Below code will work, added console.log(sitemap) in Promise, please check
const recursive = require("recursive-readdir");
const directories = ['dir1', 'dir2'];
const sitemap = [];
const extns = ["*.pdf", "*.json", "*.xml", "*.jpg", "*.gif", "*.png", "*.css", "*.js", "*.flv", "*.swf", "*.ttf", "*.pptx", "*.doc", "*.docx"];
const allRequests = directories.map((item) => {
return recursive(item, extns);
});
Promise.all(allRequests).then((files) => {
for (let x = 0; x < files.length; x++) {
let url = 'example.com/' + files[x];
addURL(url);
}
console.log(sitemap);
});
function addURL(url) {
sitemap.push(url);
}
//console.log(sitemap);
Here is one of the approach using promise...will put up a better approach to handle multiple directories..
var recursive = require("recursive-readdir");
var directories = ['dir1', 'dir2'];
var sitemap = [];
const extns = ["*.pdf", "*.json", "*.xml", "*.jpg", "*.gif", "*.png", "*.css", "*.js", "*.flv", "*.swf", "*.ttf", "*.pptx", "*.doc", "*.docx"];
const path1 = recursive(directories[0], extns);
const path2 = recursive(directories[1], extns);
Promise.all([path1, path2]).then(function(files) {
for (let x = 0; x < files.length; x++) {
let url = 'example.com/' + files[x];
addURL(url);
}
console.log(sitemap);
});
function addURL(url) {
sitemap.push(url);
}
//console.log(sitemap);
Mediocre javascript developer here and need some help..
I want to make a GET call to a url several times in a for loop.
I am trying to use fetch, but because of the promise/timing situation, I can't figure out how to make it work.
Below is the code and it's entirely possible that the fetch approach doesn't make sense for what I am trying to do. Would appreciate your help either helping me with code or telling me I am idiot and advising an alternative :)
var fromAmt = 100;
var fromOOP = 50;
var fromGM = 50;
var fromCur = "USD"
var toCur = ["USD","EUR","INR","GBP","SGD"];
var adjAmt = [];
async function getConversionAsync(fcur,tcur,amt)
{
let response = await fetch('https://data.fixer.io/api/convert?access_key=xyxyxyxyxyxyxy&from=' + fcur + '&to=' + tcur + '&amount=' + amt);
let data = await response.json()
return data;
}
for (i = 0; i < toCur.length; i++) {
getConversionAsync(fromCur,toCur[0].toString(),fromAmt)
.then(data => display(data));
}
function display(thing){
adjAmt.push(thing.result);
}
document.getElementById("something").innerHTML = adjAmt[0].toString();
In your example, document.getElementById("something").innerHTML = adjAmt[0].toString(); is executed before anything is pushed to adjAmt. You need to wait for the loop calls to finish before displaying a result, and for this you could wrap everything inside an async function.
const fromAmt = 100;
const fromOOP = 50;
const fromGM = 50;
const fromCur = 'USD';
const toCur = ['USD', 'EUR', 'INR', 'GBP', 'SGD'];
const adjAmt = [];
const getConversionAsync = async (fcur, tcur, amt) => {
const response = await fetch(`https://data.fixer.io/api/convert?access_key=xyxyxyxyxyxyxy&from=${fcur}&to=${tcur}&amount=${amt}`);
return response.json();
}
function display(thing) {
adjAmt.push(thing.result);
}
(async () => {
for (i = 0; i < toCur.length; i += 1) {
const data = await getConversionAsync(fromCur, toCur[0], fromAmt);
display(data);
}
document.getElementById('something').innerHTML = adjAmt[0].toString();
})();
Some small changes to make it work without the API call, but you'll want to access the correct index in your loop. I don't know what the exact output you're wanting here but in this case I just joined all the values in the array.
Additionally, the setting of innerHTML needs to be done once all the values are retrieved from the API, so I would even suggest doing that when the loop terminates, or some other "done" type event.
Additionally, you can use Promise.all instead of a loop, which is what I would go with personally.
var fromAmt = 100;
var fromOOP = 50;
var fromGM = 50;
var fromCur = "USD"
var toCur = ["USD", "EUR", "INR", "GBP", "SGD"];
var adjAmt = [];
async function getConversionAsync(fcur, tcur, amt) {
let response = await sampleRequest()
/* let data = await response.json() */
return response;
}
for (i = 0; i < toCur.length; i++) {
const data = getConversionAsync(fromCur, toCur[i].toString(), fromAmt).then(data => {
display(data)
})
}
function display(thing) {
adjAmt.push(thing);
document.getElementById("something").innerHTML = adjAmt.join(', ')
}
function sampleRequest() {
return new Promise((resolve, reject) => {
resolve(Math.round(Math.random() * 1000))
})
}
<div id="something"></div>
i am an absolute beginner learning js and node.js and i am building a simple scraper. the code is scraping multiple domains . i would like to store information such as the titles, meta tags such as description, etc for each of the the domains scraped into a hash table but i have no idea how to proceed. can you shortly explain how to do it? here you have the code
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var arr = ["http://allrecipes.com/", "http://www.gossip.fr/" ];
console.log("Visiting pages now... ");
for (var i = 0; i < arr.length; i++) {
setTimeout(request, 5000 * i, arr[i], function (error, response, body) {
if(error) {
console.log("Error: " + error);
}
console.log("Status code: " + response.statusCode);
if(response.statusCode === 200) {
var $ = cheerio.load(body);
console.log("Page title: " + $('title').text());
}
});
}
i modify the code as you can see below but instead of storing into the hash the title for each domains scraped, it store the result just for the last domains of the array. see code below
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var output = {};
var arr = ["http://allrecipes.com/", "http://www.gossip.fr/", "http://www.clicrbs.com.br/rs/" ];
console.log("Visiting pages now... ");
for (var i = 0; i < arr.length; i++) {
var result = arr[i];
setTimeout(request, 5000 * i, arr[i], function (error, response, body) {
if(error) {
console.log("Error: " + error);
}
console.log("Status code: " + response.statusCode);
if(response.statusCode === 200) {
var $ = cheerio.load(body);
console.log("Page title: " + $('title').text());
}
{
output[result] = {error: error, title: $('title').text(), status: response.statusCode}
}
});
}
Use let in place of var. That way you create a new let variable in each iteration, instead of overwriting the old var.
Correct:
let output = {}
let a = ['a', 'b', 'c']
for(let i=0; i<a.length; i++) {
let result = a[i]; setTimeout(x => output[result] = i, i*5)
}
setTimeout(() => document.write(JSON.stringify(output)), 20)
Incorrect:
let output = {}
let a = ['a', 'b', 'c']
for(let i=0; i<a.length; i++) {
var result = a[i]; setTimeout(x => output[result] = i, i*5)
}
setTimeout(() => document.write(JSON.stringify(output)), 20)
You can read more about let and lexicographic scoping on Mozilla Developer Network.