We Keep Coding

Script gets stuck somewhere in it's execution process

I've created a script using request and cheerio libraries to scrape links of different provinces from this webpage and then use those urls to parse the links of different offices from here. Finally, use those office links to scrape title from here.
When I run the script, I can see that it does it's job accordingly until it gets stuck somewhere in it's execution. When it gets stuck, it doesn't throw any error.
Here are the steps in image what the script is following:
Firstly, the script grabs links from here
Secondly, it grabs links from here
And finally, the script parses title from here
Here is what I've tried with:
const request = require('request');
const cheerio = require('cheerio');
const link = 'https://www.egyptcodebase.com/en/p/all';
const base_link = 'https://www.egyptcodebase.com/en/';
let getLinks = (link) => {
const items = [];
return new Promise((resolve, reject) => {
request(link, function(error, response, html) {
let $ = cheerio.load(html);
if (error) return reject(error);
try {
$('.table tbody tr').each(function() {
items.push(base_link + $(this).find("a[href]").attr("href"));
});
resolve(items);
} catch (e) {
reject(e);
}
});
});
};
let getData = (links) => {
const nitems = [];
const promises = links
.map(nurl => new Promise((resolve, reject) => {
request(nurl, function(error, response, html) {
let $ = cheerio.load(html);
if (error) return reject(error);
try {
$('.table tbody tr').each(function() {
nitems.push(base_link + $(this).find("a[href]").attr("href"));
});
resolve(nitems);
} catch (e) {
reject(e);
}
})
}))
return Promise.all(promises)
}
let FetchData = (links) => {
const promises = links
.map(turl => new Promise((resolve, reject) => {
request(turl, function(error, response, html) {
if (error) return reject(error);
let $ = cheerio.load(html);
try {
const title = $(".home-title > h2").eq(0).text();
console.log({
title: title,
itemLink: turl
});
resolve(title);
} catch (e) {
reject(e);
}
})
}))
return Promise.all(promises)
}
(async function main() {
const result = await getLinks(link);
const resultSecond = await getData(result);
const merged = resultSecond.flat(1);
const resultFinal = await FetchData(merged);
for (const title of resultFinal) {
console.log(title);
}
})().catch(console.error);
How can I make the script finish it's execution process?
PS Although the script appears to be big, the functions used in there are alomost identical to each other except for the selectors.

Ok, so on testing this code, I ran across two problems right off the bat:
resultSecond, containing the data from getData(), returned an Array-like Object, not an Array, so I wasn't able to use the flat(). So I created a function toArray that converts Objects to Arrays and added another variable after resultSecond called resultThird and used this function on resultSecond, turning it to an array.
flat() did not exist in the Array prototype, so I had to add it manually.
After resolving those issues, I was able to run your code, and experienced the hang you were talking about.
An ECONNRESET error occurred, and then proceeded to make probably a couple thousand requests before hanging. An ECONNRESET usually results from not handling asynchronous network errors or the server you're requesting decides to kill the connection. Not sure how the request module would handle such an event, but it seems like the module could potentially not be handling the network errors or terminated connections properly.
The issue was you were making 15,000 requests to this sites API, so the API probably had a rate limiter, saw the amount of requests and terminated most of them, but allowed a couple thousand to go through, but since you're not handling the terminated connections-- most likely due to the request module swallowing those errors-- it's "hanging" there with the node process not exiting.
So I batched the requests into intervals of 300 using the async module and it worked like a charm. No terminated connections because I didn't reach the rate limit. You could probably up the interval limit higher than 300.
However, I would suggest not using the request module and use another http module like axios, which most likely handle these issues. You should consider using async when you're doing a ton of asynchronous requests. It has so many helpful methods. Lmk if you need more explanation to what the async module is doing here, but I'd advise reading the documentation first: https://caolan.github.io/async/v3/docs.html#mapLimit
const request = require('request');
const cheerio = require('cheerio');
const _async = require('async');
const link = 'https://www.egyptcodebase.com/en/p/all';
const base_link = 'https://www.egyptcodebase.com/en/';
const toArray = (obj) => {
const arr = [];
for (const prop in obj) {
arr.push(obj[prop])
}
return arr;
}
Object.defineProperty(Array.prototype, 'flat', {
value: function(depth = 1) {
return this.reduce(function (flat, toFlatten) {
return flat.concat((Array.isArray(toFlatten) && (depth>1)) ? toFlatten.flat(depth-1) : toFlatten);
}, []);
}
});
let getLinks = (link) => {
const items = [];
return new Promise((resolve, reject) => {
request(link, function(error, response, html) {
let $ = cheerio.load(html);
if (error) return reject(error);
try {
$('.table tbody tr').each(function() {
items.push(base_link + $(this).find("a[href]").attr("href"));
});
resolve(items);
} catch (e) {
reject(e);
}
});
});
};
let getData = (links) => {
const nitems = [];
const promises = links
.map(nurl => new Promise((resolve, reject) => {
request(nurl, function(error, response, html) {
let $ = cheerio.load(html);
if (error) return reject(error);
try {
$('.table tbody tr').each(function() {
nitems.push(base_link + $(this).find("a[href]").attr("href"));
});
return resolve(nitems);
} catch (e) {
return reject(e);
}
})
}))
return Promise.all(promises)
}
let FetchData = (links) => {
const limit = 300;
return new Promise((resolve, reject) => {
const itr = (col, cb) => {
request(col, function(error, response, html) {
if (error) cb(error)
let $ = cheerio.load(html);
try {
const title = $(".home-title > h2").eq(0).text();
console.log({
title: title,
itemLink: col
});
cb(null, title);
} catch (e) {
cb(e);
}
})
}
_async.mapLimit(links, limit, itr, function(err, results) {
if (err) reject(err);
return resolve(results);
})
})
}
(async function main() {
const result = await getLinks(link);
const resultSecond = await getData(result);
const resultThird = toArray(resultSecond);
const merged = resultThird.flat(1);
const resultFinal = await FetchData(merged);
for (const title of resultFinal) {
console.log("title: ", title);
}
})().catch(err => console.log(err))
//good to listen to these
process.on('uncaughtException', err => { console.log(err) });
process.on('unhandledRejection', err => { console.log(err) });

Cheerio. Can't get data inside TRY body after handling error of another function

There is a code like this:
const axios = require('axios');
const cheerio = require('cheerio');
let data = null;
const parseNewCorporations = async (iter) => {
let link2 = 'https://www.finanzen.net/aktien/' + iter.finanzen_net + '-aktie';
try{
await axios.get(link2)
.then(res => res.data)
.then(res => {
let html = res;
$ = cheerio.load( html, { decodeEntities: false } );
let bigData = iter;
let price = $('div.snapshot-headline div.col-sm-7 div.row.quotebox:first-child div.col-xs-5.col-sm-4.text-sm-right.text-nowrap').text();
let currency = $('div.snapshot-headline div.col-sm-7 div.row.quotebox:first-child div.col-xs-5.col-sm-4.text-sm-right.text-nowrap span').text();
price = price.replace(currency, '').replace(',', '.');
})
}
catch(e){
console.log(e.message, ', id =', iter.id, ", finanzen_net = "+iter.finanzen_net);
await getAdditionPriceBilanzGuv(iter);
}
};
const getAdditionPriceBilanzGuv = async (iter) => {
//console.log('111', iter); // **here the code works correctly**
let link = 'https://www.finanzen.net/bilanz_guv/'+ iter.finanzen_net;
try{
await axios.get(link)
.then(res => res.data)
.then(res => {
console.log('getAdditionPriceBilanzGuv', iter);
// **here the code works NOT correctly**
})
}
catch(e){
if(e.message == 'Request path contains unescaped characters'){
console.log('Request path contains unescaped characters');
console.log({paramSubLink: iter.finanzen_net, corporations_id: iter.id});
}
else{
console.log('paramCorporationsId: ', iter.id);
//console.log('err- ', e);
}
}
};
function getApiData(){
// get request
return axios.get("https://seo-gmbh.eu/invest/daily.php" , {
})
.then(response => {
return response.data;
})
.catch(function (error) {
console.log(error);
});
}
async function new_corporations() {
data = await getApiData();
let ii = 1;
for (let iter of data.new_corporations) {
//await parseNewCorporations(iter);
ii++;
await setTimeout(function(){
parseNewCorporations(iter);
}, ii*3000);
}
//console.log(arrayCurrency);
}
new_corporations();
After calling the parseNewCorporations () function, the catch () exception is triggered as a result of which
the corresponding messages can be seen in the console.
The problem is that when this error appears, you need to run the following function getAdditionPriceBilanzGuv () with the iter parameter, and inside the body oftry {}you need to get this parameter, which cannot be done.
At the very beginning (outside the body of try {}) of this function (where it is indicated by a comment that the code works), it is possible to get this parameter.
Question:
What am I missing and how can I get this parameter in a newly called function inside the body of try {}?
If it is impossible to do this, what alternative implementations can be for solving this problem?
P.S. In this case, a parsing library is used cheerio

If you use async / await, you don't want thens
let res = await axios.get(link2)
let $ = cheerio.load(res.data)

Promise all with inner function to be executed

I want to retrieve different HTML body at once and as soon as all results are available work with that content.
My callback solution which works looks like this (probably only relevant to read if the idea is not clear, otherwise skip ;)):
const request = require('request')
const argLength = process.argv.length
const result_array = []
let counter = 0
function executeRequest () {
for (start = 2; start <= argLength - 1; start++) {
const copy = start
function callback (error, res, body) {
const startCopy = copy
if (error) {
console.log('error')
return
}
result_array[startCopy - 2] = body.toString().length
counter++
if (counter === argLength - 2) {
console.log(result_array)
}
}
request(process.argv[start], callback)
}
}
executeRequest()
Now I try to make it running with Promises like this:
const httpRequest = require('request')
const argumentLength = process.argv.length
function fillURLArray () {
resultArray = []
for (start = 2; start < argumentLength; start++) {
resultArray[start - 2] = process.argv[start]
}
return resultArray
}
const urls = fillURLArray()
let counter = 0
function readHttp () {
const resultArray = []
Promise.all(urls.map(url => httpRequest(url, (error, res, body) => {
console.log(body.toString())
resultArray[counter++] = body.toString()
}))).then(value => {
console.log('promise counter: ' + counter++)
console.log(resultArray)
console.log('called')
})
}
readHttp()
I tried already several attempts with different promise chains but every time I get either not a result or just an empty array. So obviously the .then() function is called before the array is actually filled (at least I guess so since console.log(body.toString()) appears to print the content some time later)
Any idea how to solve this with promises?
Thank you

request is not returning promise object so have created a method that return promise object on which you do Promise.all.
function requestPromise(url){
return new Promise((resovle,reject) => {
httpRequest(url, (error, res, body) => {
if(err){
reject(err);
}
resolve(body.toString());
});
});
}
function readHttp () {
const resultArray = []
Promise.all(urls.map(url => requestPromise(url))).then(values => {
console.log("counter => ",values.length);
resultArray = resultArray.concat(values);
console.log("values=> ",values);
console.log("resultArray=> ",resultArray);
});
}

httpRequest does not return a promise so you have to make one yourself, also your resultArray is not necessary:
const makeRequest = url => new Promise((resolve, reject) => httpRequest(url, (error, res) => error ? reject(error) : resolve(res)));
Promise.all(urls.map(makeRequest))
.then(result => {
console.log(result.map(res => res.body.toString()));
console.log('called');
});

async whilst with request promise

I need to iterate the http request once previous is completed. I am using request promise along with async js. My snippet is
const _getAllProduct = (token, cb) => {
const _condition = () => last !== true;
const recordCount = 50;
let last = false;
let currentPage = 0;
console.log(`\n2. Getting All Products`);
let options = {
headers: {'Content-Type': 'application/json'},
json: true
};
let allProducts = [];
const _iterate = callback => {
options.url = `${domain.url}/all?page=${currentPage}&size=${recordCount}`;
console.log(`Options: ${JSON.stringify(options)}`);
console.log(`Here`);
rp(options)
.then(res => {
last = res.last;
allProducts = _.concat(allProducts, res.content);
currentPage++;
callback(null, true);
})
.catch(error => callback(error));
};
async.whilst(_condition, _iterate, error => {
if (error) return cb(error);
console.log(`Total %d records fetched from domain`, allProducts ? _.size(allProducts) : 0);
return cb(null, allProducts);
});
};
The issue is I am getting warning the moment request is completed. Warning is promise created but not returned. My specification and requirements are using async.js and request-promise module.
(node:4159) Warning: a promise was created in a handler at /home/xavient/Desktop/node/kwa/syncing/utils/product-synb-job.js:65:8 but was not returned from it, see
at new Promise (/home/xavient/Desktop/node/kwa/syncing/node_modules/bluebird/js/release/promise.js:79:10)

Promises with http.get node.js

Im am doing nodeschool exercises , the
This problem is the same as the previous problem (HTTP COLLECT) in
that you need to use http.get(). However, this time you will be
provided with three URLs as the first three command-line arguments.
You must collect the complete content provided to you by each of the
URLs and print it to the console (stdout). You don't need to print out
the length, just the data as a String; one line per URL. The catch is
that you must print them out in the same order as the URLs are
provided to you as command-line arguments.
in other words , i am to register 3 http.get request , and print data recieved from it in order.
I am trying to do it with promises = another get request wont be called untill the first on didnt end.
My code looks like this
var http=require("http");
var collect=[];
var dat=[];
for( var i = 2 ; i < process.argv.length;i++){
collect.push(process.argv[i]);
}
function chainIt(array,callback){
return array.reduce(function(promise,item){
return promise.then(function(){
return callback(item)
})
},Promise.resolve())
}
function getIt(item){
return http.get(item,function(response){
response.on("data",function(data){
dat.push(data);
})
})
}
chainIt(collett,function(item){
return getIt(item)
})
}).then(function(){
collect.forEach(function(x){
console.log(x);
})
})
But i actually print no data = i fail the exercise.
I do not see any bug here , but im just starting with promises and node. Where is the mistake?

For educational purposes, I recently wrote a wrapper for the http and https modules that uses native Promises. That said, I recommend using a library, such a request; that makes things simpler, has unit test coverage, as is maintained by the open source community. Also, my wrapper does a naive string concatenation with the response chunks, which I'm not convinced is the most performant way of building up the response body.
FYI: this requires Node.js 4 or above, although the methodology is pretty much the same in Node 0.x.x.
'use strict';
const http = require('http');
const url = require('url');
module.exports = {
get(url) {
return this._makeRequest('GET', url);
},
_makeRequest(method, urlString, options) {
// create a new Promise
return new Promise((resolve, reject) => {
/* Node's URL library allows us to create a
* URL object from our request string, so we can build
* our request for http.get */
const parsedUrl = url.parse(urlString);
const requestOptions = this._createOptions(method, parsedUrl);
const request = http.get(requestOptions, res => this._onResponse(res, resolve, reject));
/* if there's an error, then reject the Promise
* (can be handled with Promise.prototype.catch) */
request.on('error', reject);
request.end();
});
},
// the options that are required by http.get
_createOptions(method, url) {
return requestOptions = {
hostname: url.hostname,
path: url.path,
port: url.port,
method
};
},
/* once http.get returns a response, build it and
* resolve or reject the Promise */
_onResponse(response, resolve, reject) {
const hasResponseFailed = response.status >= 400;
var responseBody = '';
if (hasResponseFailed) {
reject(`Request to ${response.url} failed with HTTP ${response.status}`);
}
/* the response stream's (an instance of Stream) current data. See:
* https://nodejs.org/api/stream.html#stream_event_data */
response.on('data', chunk => responseBody += chunk.toString());
// once all the data has been read, resolve the Promise
response.on('end', () => resolve(responseBody));
}
};
EDIT: I only just realised you're new to Promises. Here's an example of how to use this wrapper:
'use strict';
const httpService = require('./httpService'); // the above wrapper
// get one URL
httpService.get('https://ron-swanson-quotes.herokuapp.com/v2/quotes').then(function gotData(data) {
console.log(data);
});
// get multiple URLs
const urls = [
'https://ron-swanson-quotes.herokuapp.com/v2/quotes',
'http://api.icndb.com/jokes/random'
];
/* map the URLs to Promises. This will actually start the
* requests, but Promise.prototype.then is always called,
* even if the operation has resolved */
const promises = urls.map(url => httpService.get(url));
Promise.all(promises).then(function gotData(responses) {
/* responses is an array containing the result of each
* Promise. This is ordered by the order of the URLs in the
* urls array */
const swansonQuote = responses[0];
const chuckNorrisQuote = responses[1];
console.log(swansonQuote);
console.log(chuckNorrisQuote);
});

Using Promise.all is the most efficient solution for this. You could also use async/await like below to solve this.
const http = require('http');
const bl = require('bl');
async function httpGet(url) {
return new Promise((resolve, reject) => {
http.get(url, response => {
response.setEncoding('utf8');
response.pipe(bl((err, data) => {
if (err) {
reject(err);
}
resolve(data.toString());
}));
});
});
}
async function main() {
const data1 = await httpGet(process.argv[2]);
const data2 = await httpGet(process.argv[3]);
const data3 = await httpGet(process.argv[4]);
console.log(data1);
console.log(data2);
console.log(data3);
}
main();

I don't think this exercise was meant to be solved with promises. I found my old exercice folder, this is how I did it without promises or extra libraries:
var http = require('http');
var urls = process.argv.slice(2);
// counts the number of requests done
var done = 0;
// stores the requests result
var result = [];
// this will be called by each http.get and they will provide their index
function callback(index, data) {
result[index] = data;
done++;
// all requests are done, log everything
if (done == urls.length) {
result.forEach(console.log);
}
}
function processUrl(url, index) {
var finalData = '';
http.get(url, function(response) {
response.setEncoding('utf8');
response.on('data', function(data) {
finalData += data;
});
response.on('error', console.error);
response.on('end', function() {
// console.log(finalData);
callback(index, finalData);
})
});
}
urls.forEach(processUrl);
Don't worry, you'll have enough promises to play with in the promise-it-wont-hurt workshop.

A bit late to the party here :)
Unfortunately none of the answers here uses the builtin util module in node.
Here is how to promisify http.get using util module and it is working correctly with typescript:
import util from "util";
const httpGetPromisified = util.promisify(
(url: string, cb: (err: any, result: IncomingMessage) => void) =>
http.get(url, (res) => cb(null, res))
);
// with promise.then
httpGetPromisified("http://www.google.com").then((res) => {
// res here has type http.IncomingMessage
console.log(res.statusCode);
});
// with async/await
const res = await httpGetPromisified("http://www.google.com");
console.log(res.statusCode);

Here's my solution after going through this thread:
var http = require('http');
var bl = require('bl')
promises = [
promiseLoad(process.argv[2]),
promiseLoad(process.argv[3]),
promiseLoad(process.argv[4])
];
Promise.all(promises).then(function(res) {
for(i=0; i<promises.length; i++) {
console.log(res[i]);
}
});
function promiseLoad(url) {
var body = '';
return new Promise(function(resolve, reject) {
http.get(url, function (response) {
response.setEncoding('utf8');
response.pipe(bl(function (err, data) {
resolve(data.toString())
}))
})
});
}
Here's the official solution in case you want to compare notes:
var http = require('http')
var bl = require('bl')
var results = []
var count = 0
function printResults () {
for (var i = 0; i < 3; i++) {
console.log(results[i])
}
}
function httpGet (index) {
http.get(process.argv[2 + index], function (response) {
response.pipe(bl(function (err, data) {
if (err) {
return console.error(err)
}
results[index] = data.toString()
count++
if (count === 3) {
printResults()
}
}))
})
}
for (var i = 0; i < 3; i++) {
httpGet(i)
}

const http = require('http');
const urls = process.argv.slice(2);
let callCount = 0;
const cb = (res) => {
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk.toString();
});
res.on('end', () => {
callCount += 1;
console.log(rawData);
if (callCount < urls.length) {
getData(urls[callCount]);
}
});
res.on('error', (error) => {
console.log(error);
});
};
const getData = (url) => {
http.get(url, cb);
};
getData(urls[callCount]);

Here's how I did it:
async myFunc = function {
let url = 'http://your.data/file';
let promise = new Promise((resolve, reject) => {
var data = '';
https.get(url, res => {
res.on('data', chunk => { data += chunk })
res.on('end', () => {
resolve(data);
})
})
});
let result = await promise; // wait until the promise resolves
doStuffWithResult(result);
};

One of the method is by using 'Q' library.
First create function that will hit URL and return promise
var Q = require('q);
function getIt(item){
return http.get(item,function(response){
return Q.resolve(response); // Return response
OR
return Q.resolve(error); // Return error
})
})
}
var urls = ['url1','url2','url3']; // list of urls
Q.spread(urls.map(getIt))
.then(function(res1,res2,res3){
// res1 is response for url1 and on
//Once all calls are finished you will get results here
});