We Keep Coding

strange error in nodejs when request promis used in for loop manner?

I have the following web data collector:
function start(urls) {
Promise.map(urls, requestPromise)
.map((htmlPage, index) => {
const $ = cheerio.load(htmlPage);
$(".fixedttitle2").each(function () {
mytxt = $(this).text();
myarray.push(mytxt);
});
mainarray[urls[index]] = myarray;
});
fs.writeFileSync("1.json", JSON.stringify(mainarray));
}
var urls = [];
for (i = 1; i <= 100; i++) {
urls = "https://thisurl.com/" + i.toString();
start(urls);
}
Now I want to check response of each request at first, How I can check the response code at first inorder to get rid of some URLs that return 500 Error? How I can handle it?

You might be looking for something like this.
scrape (née start) processes a single URL and returns a promise of [url, content], or if there's an error, [url, null].
main generates the list of URLs to scrape, then starts scrape for all of them.
Note that all 100 requests start at once; this may or may not be a problem for you.
Finally, when all of the scrape promises complete, their return values are gathered into response, and that's written into the JSON file.
This differs from the original in that the original kept re-writing the file as new content was scraped.
async function scrape(url) {
try {
const htmlPage = await requestPromise(url);
const $ = cheerio.load(htmlPage);
const texts = [];
$('.fixedttitle2').each(function () {
texts.push($(this).text());
});
return [url, texts];
} catch (err) {
console.error(`Error processing url: ${url}: ${err}`);
return [url, null];
}
}
async function main() {
const urls = [];
for (var i = 1; i <= 100; i++) {
urls.push(`https://thisurl.com/${i}`);
}
const response = await Promise.all(urls.map(scrape));
fs.writeFileSync('1.json', JSON.stringify(response));
}
If you'd like the requests to be done sequentially, you can await scrape() in the loop:
async function main() {
const response = [];
for (var i = 1; i <= 100; i++) {
const url = `https://thisurl.com/${i}`;
response.push(await scrape(url));
}
fs.writeFileSync('1.json', JSON.stringify(response));
}
You could also move the write file call into the loop if you wanted the same incremental behavior your original code had.
EDIT
You can also add a delay to the sequential loop:
// Basic promisified delay function
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
async function main() {
const response = [];
for (var i = 1; i <= 100; i++) {
const url = `https://thisurl.com/${i}`;
response.push(await scrape(url));
await delay(1000); // Wait for 1000 ms between scrapes
}
fs.writeFileSync('1.json', JSON.stringify(response));
}

Async await with Fetch

my fetch works perfectly with .then, but i want to step it up a notch by using async and await. It should wait for all 5 API calls, and then place the answer, instead, it shows answer on every API call
async function getPhotosFromAPI() {
for (let i = 1; i <= 5; i++) {
let albums = await fetch(
`https://jsonplaceholder.typicode.com/photos/?albumId=${i}`
);
let result = await albums.json();
let res = `<div class="album${i}"></div>`;
document.querySelector(".display-images").innerHTML += res;
for (let j = 1; j <= 5; j++) {
document.querySelector(
`.album${i}`
).innerHTML += `<img src="${result[j].url}"/>`;
}
}
console.log(result);
}
async function showPhotos() {
await getPhotosFromAPI();
document.getElementById("#loader").style.display = "none";
}
showPhotos();
document.getElementById("img").style.display = "block";
for (let i = 1; i <= 5; i++) {
fetch(`https://jsonplaceholder.typicode.com/photos/?albumId=${i}`)
.then((response) => response.json())
.then((json) => {
document.getElementById("img").style.display = "none";
const display = document.querySelector(".display-images");
const albumNo = document.querySelector(".album-no");
// document.getElementById('img').style.display = "block";
// document.getElementById('img').style.display = "none";]
display.innerHTML += `<div class="album-${i}>`;
for (let z = 1; z <= 5; z++) {
display.innerHTML += `<img id="img" alt="pic-from-album${json[i].albumId}" src="${json[z].url}"/>`;
}
display.innerHTML += `<div>`;
});
}

You should use a concurrent way of fetching like Promise.all to avoid round-trips
async function getPhotosFromAPI() {
let albums = await Promise.all(
Array(5).fill().map((elem, index) =>
fetch(`https://jsonplaceholder.typicode.com/photos/?albumId=${index+1}`)
)
)
let results = await Promise.all(
albums.map(album => album.json())
)
return results
}
//Display

You're asking for the code to wait until each fetch finishes by using await on fetch's return value (then again on the return value of json) in your loop. So it will do just that: wait until that request is complete before moving on to the next loop iteration.
If you don't want to do that, you need to start each fetch one after another and then wait for them all to complete. I'd probably break out the work for just one of them into a function, then call it five times, building an array of the promises it returns, then await Promise.all(/*...*/) those promises, something along these lines:
document.getElementById("img").style.display = "block";
// Fetch one item
const fetchOne = async (i) => {
const response = await fetch(`https://jsonplaceholder.typicode.com/photos/?albumId=${i}`);
if (!response.ok) {
throw new Error(`HTTP error ${response.status}`);
}
const data = await response.json();
document.getElementById("img").style.display = "none";
const display = document.querySelector(".display-images");
const albumNo = document.querySelector(".album-no");
// document.getElementById('img').style.display = "block";
// document.getElementById('img').style.display = "none";]
display.innerHTML += `<div class="album-${i}>`;
for (let z = 1; z <= 5; z++) {
display.innerHTML += `<img id="img" alt="pic-from-album${data[i].albumId}" src="${data[z].url}"/>`;
}
display.innerHTML += `<div>`;
};
// ...
await Promise.all(Array.from({length: 5}, (_, i) => fetchOne(i + 1)));
// All done
(I took the version with .then as my starting point for the above, since the two versions in your question were so different and you said the one with .then worked... Also note that I renamed the variable json to data, since it doesn't contain JSON [it's not a string], it contains the result of parsing JSON.)

Trying to set a recursive function inside a promise

let cancelDownload = true
const delay = () => {
return new Promise(() => {
if (cancelDownload === true){
setTimeout(() => {
console.log('Delaying...');
delay();
}, 1000);
}
else return null;
});
};
const cancelJob = async() => {
for(let i = 6; i>0; i--){
console.log('inside for ',i);
setTimeout(()=>{
cancelDownload = false
},4000)
await delay()
console.log('aaaaaaaa');
console.log(`the number is ${i}`);
}
}
cancelJob()
I am trying to write a delay function whereby once the condition is met delay is removed and all code is resumed but it seems that once the delay is done the code exits without executing the last two console logs

No recursion needed. Instead, you can:
Use setInteval to check your condition every second.
When the condition is correct, you need to resolve the promise.
Use clearInterval.
let cancelDownload = true
const delay = () => {
let intervalId;
return new Promise((resolve, reject) => {
const check = () => {
if (cancelDownload === true){
console.log('Delaying...');
} else {
clearInterval(intervalId);
resolve();
}
}
//check immediately
check();
//then check every second afterwards
intervalId = setInterval(check, 1000);
});
};
const cancelJob = async() => {
for(let i = 6; i>0; i--){
console.log('inside for ',i);
setTimeout(()=>{
cancelDownload = false
},4000)
await delay()
console.log('aaaaaaaa');
console.log(`the number is ${i}`);
}
}
cancelJob()
This can be generalised a bit in the following fashion - instead of hard-coding the condition, supply it as a callback. Then you can have a delay function without using global variables and it can wait for different things, not just one single variable.
const delayWhile = shouldWait => () => {
let intervalId;
return new Promise((resolve, reject) => {
const check = () => {
if (shouldWait()){
console.log('Delaying...');
} else {
clearInterval(intervalId);
resolve();
}
}
//check immediately
check();
//then check every second afterwards
intervalId = setInterval(check, 1000);
});
};
const cancelJob = async() => {
let cancelDownload = true;
const delay = delayWhile(() => cancelDownload);
for(let i = 6; i>0; i--){
console.log('inside for ',i);
setTimeout(()=>{
cancelDownload = false
},4000)
await delay()
console.log('aaaaaaaa');
console.log(`the number is ${i}`);
}
}
cancelJob()

The function you pass to the promise constructor (the promise executor function) is called with two arguments: a function to use to resolve the promise and a function to reject it (conventionally called resolve and reject). Your code should call one of those functions when your asynchronous work is done.
Yours isn't, so the promise never settles, and your code waits forever at the await.
But there are other issues:
1. If you call delay again, it creates a new promise. Your code using await only has access to the first promise, not the ones created by those recursive calls. There isn't really any reason to use recursion here at all.
2. All calls to the function share the same flag. So if we fix the issue with not fulfilling the promise, the loop does wait, but only once:
let cancelDownload = false;
const delay = () => {
return new Promise((resolve) => {
tick();
function tick() {
// Cancelled?
if (cancelDownload) {
// Yes, fulfill the promise
console.log("Flag is set, fulfilling");
resolve(null);
} else {
// No, wait another second
console.log("waiting 1000 and checking again");
setTimeout(tick, 1000);
}
}
});
};
const cancelJob = async () => {
setTimeout(() => {
console.log("Setting cancelDownload to true");
cancelDownload = true;
}, 4000);
for (let i = 6; i > 0; i--) {
console.log("inside for ", i);
await delay();
console.log("aaaaaaaa");
console.log(`the number is ${i}`);
}
};
cancelJob();
.as-console-wrapper {
max-height: 100% !important;
}
I may be mistaken that that second one is a problem for your use case, though, since in a comment on a now-deleted answer you said you wanted the loop only to wait once (the "whole loop" rather than just one iteration).
If you want a function that polls a flag (I don't recommend it, polling is generally not best practice, though sometimes you can't avoid it) and fulfills a promise when it's set, you could use AbortController:
const delay = (signal) => {
let done = false;
return new Promise((resolve) => {
tick();
function tick() {
// Cancelled?
if (signal.aborted) {
// Yes, fulfill the promise with null
console.log("Fulfilling with null");
resolve(null);
} else {
// No, wait another second
console.log("Waiting 1000 and checking again");
setTimeout(tick, 1000);
}
}
});
};
const cancelJob = async () => {
const controller = new AbortController();
setTimeout(() => {
console.log("Cancelling");
controller.abort();
}, 4000);
for (let i = 6; i > 0; i--) {
console.log("inside for ", i);
await delay(controller.signal);
console.log("aaaaaaaa");
console.log(`the number is ${i}`);
}
};
cancelJob();
.as-console-wrapper {
max-height: 100% !important;
}
That also only delays the loop once, because we're passing the same signal to all of the delay functions. Originally I was creating that inside the loop, like this:
const delay = (signal) => {
let done = false;
return new Promise((resolve) => {
tick();
function tick() {
// Cancelled?
if (signal.aborted) {
// Yes, fulfill the promise with null
console.log("Fulfilling with null");
resolve(null);
} else {
// No, wait another second
console.log("Waiting 1000 and checking again");
setTimeout(tick, 1000);
}
}
});
};
const cancelJob = async () => {
for (let i = 6; i > 0; i--) {
const controller = new AbortController();
setTimeout(() => {
console.log("Cancelling");
controller.abort();
}, 4000);
console.log("inside for ", i);
await delay(controller.signal);
console.log("aaaaaaaa");
console.log(`the number is ${i}`);
}
};
cancelJob();
.as-console-wrapper {
max-height: 100% !important;
}
...but then I saw the "whole loop" comment on the other answer.
Note: Normally when you have an asynchronous process with a cancel feature like that, it rejects on cancel with a cancel-specific rejection reason rather than fulfilling, but you get the idea.

One approach to cancellable promises is to supply the promise creator function with a writable parameter like cancelToken. The promise creator populates this parameter with a callback, which, when invoked, cancels this particular promise. This usually leads to simpler and more linear code.
let pause = () => new Promise(r => setTimeout(r, 100));
async function job(name, steps, cancelToken) {
let cancelled = false;
cancelToken.callback = () => cancelled = true;
for (let step = 1; step <= steps; step++) {
console.log(name, step);
await pause();
if (cancelled) {
console.log('CANCELLED');
break;
}
}
}
async function main() {
let cancelToken = {};
let job1 = job('ok', 5, cancelToken);
await job1;
let job2 = job('bad', 5, cancelToken);
setTimeout(cancelToken.callback, 100);
await job2;
}
main()
As a general hint, it's better to avoid new Promise and callbacks whether possible. Note how in this example Promise and setTimeout are only used for testing purposes (simulate real work) and the rest of the code won't change once we replace pause with something more useful like fetch.

Promise.all get slowest resolving promises

I have a few hundreds of things to render in parallel in an html5 canvas. These are drawn in parallel in a Promise.all call. Now, I would like to know which of these promise is the last to be resolved.
// get a promise that will resolve in between 0 and 5 seconds.
function resolveAfterSomeTime(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
}
const myPromises = [];
for (let i = 0; i < 100; i++) {
myPromises.push(resolveAfterSomeTime);
}
Promise.all(myPromises).then(() => {
// find out which promise was the last to resolve.
})
In my case, I have multiple classes with each a render() function. Some of these are heavier than others, but I want to know which ones.
I have something along these lines, and I would like to know which promise is the slowest to resolve, so that I can optimise it.

The best way I can think of is to use a counter indicating the number of promises that have resolved so far:
function resolveAfterSomeTime() {
return new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
}
const myPromises = [];
let resolveCount = 0;
for (let i = 0; i < 100; i++) {
myPromises.push(
resolveAfterSomeTime()
.then(() => {
resolveCount++;
if (resolveCount === 100) {
console.log('all resolved');
console.log('array item', i, 'took longest');
}
})
);
}

Here's a way where each promise sets the value of lastPromiseToResolve after resolving. The last promise to resolve would set it last.
// get a promise that will resolve in between 0 and 5 seconds.
function resolveAfterSomeTime(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
}
let lastPromiseToResolve = null
const myPromises = [];
for (let i = 0; i < 100; i++) {
const promise = resolveAfterSomeTime()
myPromises.push(promise.then(() => {
lastPromiseToResolve = promise // all the promises will set lastPromiseToResolve
}));
}
Promise.all(myPromises).then(() => {
console.log(lastPromiseToResolve) // this would be the last promise to resolve
})

You could time each promise. You could even assign an identifier to each one if you want to know specifically which is resolving. The timePromise function below takes an id and a function that returns a promise, times that promise, and logs the result. It doesn't change the result of the promises, so you can use myPromises as you normally would.
function resolveAfterSomeTime() {
return new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
}
// f is a function that returns a promise
function timePromise(id, f) {
const start = Date.now()
return f()
.then(x => {
const stop = Date.now()
console.log({id, start, stop, duration: (stop - start)})
return x
})
}
const myPromises = [];
for (let i = 0; i < 100; i++) {
myPromises.push(timePromise(i, resolveAfterSomeTime));
}
Promise.all(myPromises).then(() => {
// find out which promise was the last to resolve.
})
I'm not sure how you're creating your array of promises in your actual code, so it might not be straightforward to wrap each promise in a function that returns it. But you could likely adapt this to work with your situation.
If you aren't concerned with knowing exactly how long each takes, you could just have timePromise take a promise that's already started, and time from when timePromise is called to when it resolves. This wouldn't be as accurate, but would still give you a general idea, especially if one or a few promises are taking much longer than others.
Something like this:
function timePromise(id, p) {
const start = Date.now()
return p
.then(x => {
const stop = Date.now()
console.log({id, start, stop, duration: (stop - start)})
return x
})
}
const myPromises = [];
for (let i = 0; i < 100; i++) {
myPromises.push(timePromise(i, resolveAfterSomeTime()));
}

Javascript.Run Multi promises Synchronously

I want to request a website for 40 times.
I want this to be synchronously, like 10 requests 4 times.
This is My code for 1 request - 40 times:
'use strict';
var request = require('request');
var co = require('co');
function callUrl(url) {
return new Promise((resolve, reject) => {
request.get(url, (e, r, b) => {
if (e) reject(e)
else
resolve(`Response from ${url}`);
});
})
}
co(function*() {
for (var i = 1; i < 41; i++) {
let rs = yield callUrl('https://www.google.com/?q=' + i);
// let rs = yield makeUrls(10,i);
console.log(rs);
}
});
I can make an array of promises, but I can't figure it out how to change the value of q to be different.

You don't want to run them synchronously - you want to synchronize them - those are different.
You'd use an array of promises together with Promise#all. When you create a promise the action is already being executed - only yield synchronizes things.
You can make 10 requests at once like so:
co(function*() {
for (var i = 1; i < 41;) {
var promises = [];
for(var lim = i + 10; i < Math.max(lim, 41); i++) {
promises.push(callUrl('https://www.google.com/?q=' + i));
}
let rs = yield Promise.all(promises); // wait for everything
console.log(rs); // an array of 10 results
});
Note that in addition to that, your code is still not very efficient - what happens if 9 out of 10 requests are really fast and one takes a minute? You'll only have one outgoing requests. You can use a library like bluebird which has a more efficient Promise.map method with a concurrency parameter.

this might work w/o using generators
const urls = [/*array of urls*/];
const initialPromise = request[urls[0]];
let promise = initialPromise;
for(let i= 1; i<40;i++){
let thenFunction = response => {
//do something with the response
return request(urls[i])
}
promise = promise.then(thenFunction)
}
the idea behind this is to build the chain of promises so the next one will waif for the previous one to finish