I am writing a simple Javascript script which uses XMLHttpRequest objects to query an API. There are two main functions within the script; routeLength(startID,endID); and getSystemSecurity(systemID);
The first method returns a Promise object, within with an API call is made:
function routeLength (startID, endID) {
return new Promise((resolve, reject) => {
const url = `the request URL`;
const xml = new XMLHttpRequest();
xml.onload = function () {
var systems = this.response.split(',');
const length = systems.length - 1;
var routeSecurity = 2;
var promises = [];
systems.forEach((element) => {
if (element[0] == '[') {
promises.push(
getSystemSecurity(element.substr(1, element.length - 1))
);
}
else if (element[element.length - 1] == ']') {
promises.push(
getSystemSecurity(element.substr(0, element.length - 1))
);
}
else promises.push(getSystemSecurity(element));
});
Promise.all(promises).then((values) => {
values.forEach((v) => {
if (v < routeSecurity) routeSecurity = v;
})
resolve({"jumps":length,"security":routeSecurity});
}).catch((error) => {
reject(error)
});
};
xml.open("GET", url);
xml.send();
})
}
This method makes a GET request to the API, and performs some logic with the result. The 'systems' array contains system IDs which I have checked are all valid (once the brackets are removed if needed). The promises pushed into the promises array call the following function:
function getSystemSecurity (systemID) {
return new Promise((resolve,reject) => {
const url = `${API_URL}/universe/systems/${systemID}`;
const xml = new XMLHttpRequest();
xml.onload = function () {
if (this.response.error) reject(this.response.error);
const security = this.response["security_status"];
if (security >= 0.5) resolve(HIGH_SECURITY);
else if (security < 0.5 && security > 0) resolve(LOW_SECURITY);
else if (security < 0) resolve(NULL_SECURITY);
else reject({"error":"Invalid security value " + security});
}
xml.onerror = reject("Error executing XMLHttpRequest with ID " + systemID + xml.statusText + xml.responseText);
xml.open("GET", url);
xml.send();
});
}
When I try to run this method, I can see all of the proper requests being made, both the initial request to /route/ as well as the /universe/systems/ routes. Using the developer console I can see that none of them have failed and they all have the expected data. However, I am told that the first promise had failed:
(index):1 Uncaught (in promise) Error executing XMLHttpRequest with ID 30002659
As you can see from the error, xml.statusText and xml.responseText aren't populated, so I cannot determine the actual error that is happening.
So why is Promise.all failing despite all of the requests working with 200 response codes?
The problem is in this line:
xml.onerror = reject("Error executing XMLHttpRequest with ID " + systemID + xml.statusText + xml.responseText);
You are not assigning an event handler, but executing it on the spot!
It should be:
xml.onerror = () => reject("Error executing XMLHttpRequest with ID " + systemID + xml.statusText + xml.responseText);
A general remark: please use the fetch API, it is so much easier to use, and it returns a promise out of the box, so that you can get rid of all those new Promise constructs.
Related
I am trying to use a promise with a setInterval function to keep checking to see if a file exists(as it could be still generating on the backend) and then once it is available it runs the renderpage function as shown in my setupFunction(). Currently this program just keeps running until the setInterval maxes out and then it resolves even if the file is there.
async checkFileExist(path, timeout = 20000) {
let totalTime = 0;
let checkTime = timeout / 10;
var self = this;
return await new Promise((resolve, reject) => {
const timer = setInterval(function () {
totalTime += checkTime;
let fileExists = self.fileExist(path);
if (fileExists || totalTime >= timeout) {
alert(totalTime);
clearInterval(timer);
resolve(fileExists);
}
}, checkTime);
})
},
fileExist(urlToFile) {
var xhr = new XMLHttpRequest();
xhr.open('HEAD', urlToFile, false);
xhr.send();
if (xhr.status == "404") {
alert('false 404 ' + urlToFile)
return false;
} else {
alert('true file found ' + urlToFile)
return true;
}
},
setupFunction() {
var self = this
if (this.filedir == null || this.filedir == "") {
self.loadingFile = false
} else {
this.checkFileExist(this.filedir).then(function (response) {
self.loadingFile = false;
self.renderPage(this.pageNum);
}, function (error) {
console.error("Failed!", error);
})
}
}
},
mounted() {
this.setupFunction()
},
watch: {
filedir: function () { // watch it
this.setupFunction()
},
}
That won't tell you if a file exists. It will tell you if a URL finds a resource. It's an important distinction since a URL could be handled by server-side code and not a simple static file, while a file might exist and not have a URL.
Making an HTTP request is the simplest way to find out if a URL finds a resource. There are other ways to make HTTP requests (such as the fetch API), but they aren't faster, just different.
A potentially faster (but much more complicated) way would be to use a Websocket. You need to have it open before you wanted to know if the URL exists (otherwise any time savings are lost to establishing the Websocket connection) and you'd need to write the server side code which reacted to the Websocket message by working out if the desired URL existed and telling the client.
I am trying to fetch JSON data from the WordPress Developer Reference site. I need to search a keyword without knowing if it's a function, class, hook, or method, which is part of the url I need to fetch. So I'm using Promise.all to cycle through all possible urls. It works if the response.status <= 299, throwing the error immediately, and if the response is ok, then it continues to .then. Fine, but occasionally it will return an ok status if the JSON exists and only returns an empty array. So I need to check if the JSON data is an empty array, which I can't seem to do in the first part. I can only check in the second part as far as I know. And if it throws the error it doesn't continue trying the other urls. Any suggestions?
var keyword = 'AtomParser';
const refs = ['function', 'hook', 'class', 'method'];
// Store the promises
let promises = [];
// Cycle through each type until we find one we're looking for
for (let t = 0; t < refs.length; t++) {
const url =
'https://developer.wordpress.org/wp-json/wp/v2/wp-parser-' +
refs[t] +
'?search=' +
keyword;
// console.log(url);
promises.push(fetch(url));
}
Promise.all(promises)
.then(function(response) {
console.log(response[0]);
// Get the status
console.log('Status code: ' + response[0].status);
if (response[0].status <= 299) {
// The API call was successful!
return response[0].json();
} else {
throw new Error('Broken link status code: ' + response[0].status);
}
})
.then(function(data) {
// This is the HTML from our response as a text string
console.log(data);
// Make sure we have data
if (data.length == 0) {
throw new Error('Empty Array');
}
// ref
const reference = data[0];
// Only continue if not null or empty
if (reference !== null && reference !== undefined && data.length > 0) {
// Success
// Return what I want from the reference
}
})
.catch(function handleError(error) {
console.log('Error' + error);
});
Is there some way to get the JSON data in the first part so I can check if it's in an array while I'm checking the response status?
I would recommend encapsulating the success / failure logic for individual requests, then you can determine all the resolved and rejected responses based on the result of that encapsulation.
For example
const checkKeyword = async (ref, keyword) => {
const params = new URLSearchParams({ search: keyword });
const res = await fetch(
`https://developer.wordpress.org/wp-json/wp/v2/wp-parser-${encodeURIComponent(
ref
)}?${params}`
);
if (!res.ok) {
throw new Error(`${res.status}: ${await res.text()}`);
}
const data = await res.json();
if (data.length === 0) {
throw new Error(`Empty results for '${ref}'`);
}
return { ref, data };
};
Now you can use something like Promise.any() or Promise.allSettled() to find the first successful request or all successful requests, respectively
const keyword = "AtomParser";
const refs = ["function", "hook", "class", "method"];
const promises = refs.map((ref) => checkKeyword(ref, keyword));
// First success
Promise.any(promises)
.then(({ ref, data }) => {
console.log(ref, data);
})
.catch(console.error);
// All successes
Promise.allSettled(promises)
.then((responses) =>
responses.reduce(
(arr, { status, value }) =>
status === "fulfilled" ? [...arr, value] : arr,
[]
)
)
.then((results) => {
// results has all the successful responses
});
For whatever reason I couldn't get Phil's answer to work, so I ended up doing the following which works fine for me. (This is for a discord bot in case you're wondering what the other stuff is all about).
var keyword = 'AtomParser';
const refs = ['function', 'hook', 'class', 'method'];
// Store the successful result or error
let final: any[] = [];
let finalError = '';
// Cycle through each type until we find one we're looking for
for (let t = 0; t < refs.length; t++) {
const url =
'https://developer.wordpress.org/wp-json/wp/v2/wp-parser-' +
refs[t] +
'?search=' +
keyword;
console.log(url);
// Try to fetch it
await fetch(url)
.then(function (response) {
console.log(response);
// Get the status
console.log('Status code: ' + response.status);
if (response.status > 299) {
finalError = '`' + refs[t] + '` does not exist.';
throw new Error(finalError);
} else {
// The API call was successful!
return response.json();
}
})
.then(function (data) {
// This is the HTML from our response as a text string
console.log(data);
// Make sure we have data
if (data.length == 0) {
finalError = "Sorry, I couldn't find `" + keyword + '`';
throw new Error(finalError);
}
// Only continue if not null or empty
if (data[0] !== null && data[0] !== undefined && data.length > 0) {
for (let d = 0; d < data.length; d++) {
// Add it to the final array
final.push(data[d]);
}
}
})
.catch(function handleError(error) {
console.log(error);
});
}
if (final.length > 0) {
for (let f = 0; f < final.length; f++) {
// ref
const reference = final[f];
// Get the link
const link = reference.link;
// Get the title
var title = reference.title.rendered;
title = excerpt.replace('>', '>');
// Get the excerpt
var excerpt = reference.excerpt.rendered;
excerpt = excerpt.replace('<p>', '');
excerpt = excerpt.replace('</p>', '');
excerpt = excerpt.replace('<b>', '**');
excerpt = excerpt.replace('</b>', '**');
console.log(excerpt);
message.reply(
new discord.Embed({
title: `${title}`,
url: link,
description: `${excerpt}\n\n`,
footer: {
text: `WordPress Developer Code Reference\nhttps://developer.wordpress.org/`,
},
})
);
}
} else if (finalError != '') {
message.reply(finalError);
} else {
message.reply('Something went wrong...');
}
wp module
#Phil's answer puts you on the right track but I want to expand on some of his ideas. Use of URLSearchParamas is great but you can improve by using the high-level URL API and forego encodeURIComponent and constructing search params manually. Notice I'm putting this code in its own wp module so I can separate concerns more easily. We don't want all of this code leaking into your main program.
// wp.js
import { fetch } from "whatwg-fetch" // or your chosen implementation
const baseURL = "https://developer.wordpress.org"
async function search1(path, query) {
const u = new URL(path, baseURL)
u.searchParams.set("search", query)
const result = await fetch(u)
if (!result.ok) throw Error(`Search failed (${result.status}): ${u}`)
return result.json()
}
search1 searches one path, but we can write search to search all the necessary paths. I don't think there's any reason to get fancy with each path here, so just write them out -
// wp.js (continued)
function search(query) {
const endpoints = [
"/wp-json/wp/v2/wp-parser-function",
"/wp-json/wp/v2/wp-parser-hook",
"/wp-json/wp/v2/wp-parser-class",
"/wp-json/wp/v2/wp-parser-method"
]
return Promise
.all(endpoints.map(e => search1(e, query)))
.then(results => results.flat())
}
export { search }
main module
Notice we only exported search as search1 is internal to the wp module. Let's see how we can use it in our main module now -
// main.js
import { search } from "./wp.js"
for (const result of await search("database"))
if(result.guid.rendered)
console.log(`${result.title.rendered}\n${result.guid.rendered}\n`)
In this example, we first search for "database" -
wp_should_replace_insecure_home_url()
https://developer.wordpress.org/reference/functions/wp_should_replace_insecure_home_url/
wp_delete_signup_on_user_delete()
https://developer.wordpress.org/reference/functions/wp_delete_signup_on_user_delete/
get_post_datetime()
https://developer.wordpress.org/reference/functions/get_post_datetime/
wp_ajax_health_check_get_sizes()
https://developer.wordpress.org/reference/functions/wp_ajax_health_check_get_sizes/
wp_should_replace_insecure_home_url
https://developer.wordpress.org/reference/hooks/wp_should_replace_insecure_home_url/
comments_pre_query
https://developer.wordpress.org/reference/hooks/comments_pre_query/
users_pre_query
https://developer.wordpress.org/reference/hooks/users_pre_query/
WP_Object_Cache
http://developer.wordpress.org/reference/classes/wp_object_cache/
wpdb
http://developer.wordpress.org/reference/classes/wpdb/
WP_REST_Menu_Items_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_menu_items_controller/prepare_item_for_database/
WP_REST_Global_Styles_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_global_styles_controller/prepare_item_for_database/
WP_REST_Menus_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_menus_controller/prepare_item_for_database/
WP_REST_Templates_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_templates_controller/prepare_item_for_database/
WP_REST_Application_Passwords_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_application_passwords_controller/prepare_item_for_database/
wpdb::db_server_info()
https://developer.wordpress.org/reference/classes/wpdb/db_server_info/
WP_REST_Attachments_Controller::insert_attachment()
https://developer.wordpress.org/reference/classes/wp_rest_attachments_controller/insert_attachment/
WP_Debug_Data::get_database_size()
https://developer.wordpress.org/reference/classes/wp_debug_data/get_database_size/
WP_REST_Meta_Fields::update_multi_meta_value()
https://developer.wordpress.org/method/wp_rest_meta_fields/update_multi_meta_value/
another search example
Now let's search for "image" -
for (const result of await search("image"))
if(result.guid.rendered)
console.log(`${result.title.rendered}\n${result.guid.rendered}\n`)
get_adjacent_image_link()
https://developer.wordpress.org/reference/functions/get_adjacent_image_link/
get_next_image_link()
https://developer.wordpress.org/reference/functions/get_next_image_link/
get_previous_image_link()
https://developer.wordpress.org/reference/functions/get_previous_image_link/
wp_robots_max_image_preview_large()
https://developer.wordpress.org/reference/functions/wp_robots_max_image_preview_large/
wp_getimagesize()
https://developer.wordpress.org/reference/functions/wp_getimagesize/
is_gd_image()
https://developer.wordpress.org/reference/functions/is_gd_image/
wp_show_heic_upload_error()
https://developer.wordpress.org/reference/functions/wp_show_heic_upload_error/
wp_image_src_get_dimensions()
https://developer.wordpress.org/reference/functions/wp_image_src_get_dimensions/
wp_image_file_matches_image_meta()
https://developer.wordpress.org/reference/functions/wp_image_file_matches_image_meta/
_wp_check_existing_file_names()
https://developer.wordpress.org/reference/functions/_wp_check_existing_file_names/
edit_custom_thumbnail_sizes
https://developer.wordpress.org/reference/hooks/edit_custom_thumbnail_sizes/
get_header_image_tag_attributes
https://developer.wordpress.org/reference/hooks/get_header_image_tag_attributes/
image_editor_output_format
https://developer.wordpress.org/reference/hooks/image_editor_output_format/
wp_image_src_get_dimensions
https://developer.wordpress.org/reference/hooks/wp_image_src_get_dimensions/
wp_get_attachment_image
https://developer.wordpress.org/reference/hooks/wp_get_attachment_image/
image_sideload_extensions
https://developer.wordpress.org/reference/hooks/image_sideload_extensions/
wp_edited_image_metadata
https://developer.wordpress.org/reference/hooks/wp_edited_image_metadata/
wp_img_tag_add_loading_attr
https://developer.wordpress.org/reference/hooks/wp_img_tag_add_loading_attr/
wp_image_file_matches_image_meta
https://developer.wordpress.org/reference/hooks/wp_image_file_matches_image_meta/
get_custom_logo_image_attributes
https://developer.wordpress.org/reference/hooks/get_custom_logo_image_attributes/
Custom_Image_Header
http://developer.wordpress.org/reference/classes/custom_image_header/
WP_Image_Editor_Imagick
http://developer.wordpress.org/reference/classes/wp_image_editor_imagick/
WP_Embed
http://developer.wordpress.org/reference/classes/wp_embed/
WP_Image_Editor
http://developer.wordpress.org/reference/classes/wp_image_editor/
WP_Customize_Background_Image_Setting
http://developer.wordpress.org/reference/classes/wp_customize_background_image_setting/
WP_Customize_Header_Image_Setting
http://developer.wordpress.org/reference/classes/wp_customize_header_image_setting/
WP_Image_Editor_GD
http://developer.wordpress.org/reference/classes/wp_image_editor_gd/
WP_Customize_Header_Image_Control
http://developer.wordpress.org/reference/classes/wp_customize_header_image_control/
WP_REST_Server::add_image_to_index()
https://developer.wordpress.org/reference/classes/wp_rest_server/add_image_to_index/
WP_REST_URL_Details_Controller::get_image()
https://developer.wordpress.org/reference/classes/wp_rest_url_details_controller/get_image/
WP_Image_Editor::get_default_quality()
https://developer.wordpress.org/reference/classes/wp_image_editor/get_default_quality/
WP_Theme_JSON::get_blocks_metadata()
https://developer.wordpress.org/reference/classes/wp_theme_json/get_blocks_metadata/
WP_Image_Editor_Imagick::pdf_load_source()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/pdf_load_source/
WP_Image_Editor_Imagick::write_image()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/write_image/
WP_Image_Editor_Imagick::maybe_exif_rotate()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/maybe_exif_rotate/
WP_Image_Editor_Imagick::make_subsize()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/make_subsize/
WP_Image_Editor_GD::make_subsize()
https://developer.wordpress.org/reference/classes/wp_image_editor_gd/make_subsize/
empty search result
Searching for "zzz" will yield no results -
for (const result of await search("zzz"))
if(result.guid.rendered)
console.log(`${result.title.rendered}\n${result.guid.rendered}\n`)
<empty result>
I've been scouring similar problems but haven't seem to have found a solution that quite works on my end. So I'm working on a Discord bot that takes data from a MongoDB database and displays said data in the form of a discord embedded message using Mongoose. For the most part, everything is working fine, however one little section of my code is giving me trouble.
So I need to import an array of both all available users and the "time" data of each of those users. Here is the block of code I use to import said data:
for (i = 0;i < totalObj; i++){
timeArray[i] = await getData('time', i);
userArray[i] = await getData('user', i);
}
Now this for loop references a function I made called getData which obtains the data from MongoDB by this method:
async function getData(field, value){
var data;
await stats.find({}, function(err, result){
if(err){
result.send(err);
}else{
data = result[value];
}
});
if(field == "user"){
return data.user;
}else if (field == "time"){
return data.time;
}else{
return 0;
}
So that for loop is where my errors currently lie. When I try to run this code and display my data through a discord message, I get this error and the message does not get sent:
(node:13936) UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'time' of undefined
Now the strange thing is, this error does not happen every time. If I continue calling the command that triggers this code from my discord server, it's almost like a 50/50 shot if the command actually shows the message or instead gives this error. It is very inconsistent.
This error is confounding me, as the undefined part does not make sense to me. The objects that are being searched for in the mongoDB collection are definitely defined, and the for loop never exceeds the number of objects present. My only conclusion is that I'm doing something wrong with my asynchronous function design. I have tried altering code to use the getData function less often, or to not use awaits or asynchronous design at all, however this leaves my final discord message with several undefined variables and an eventual crash.
If anyone has any advice or suggestions, that would be very much appreciated. Just for reference, here is the full function that receives the data, sorts it, and prepares a string to be displayed on the discord server (though the error only seems to occur in the first for loop):
async function buildString(){
var string = "";
var totalObj;
var timeArray = [];
var userArray = [];
var stopSort = false;
await stats.find({}, function(err, result){
if(err){
result.send(err);
}else{
totalObj = result.length;
}
});
for (i = 0;i < totalObj; i++){
timeArray[i] = await getData('time', i);
userArray[i] = await getData('user', i);
}
while(!stopSort){
var keepSorting = false;
for(i = 0; i < totalObj ; i++){
var target = await convertTime(timeArray[i]);
for(j = i + 1 ; j < totalObj ; j++){
var comparison = await convertTime(timeArray[j]);
if(target > comparison){
//Switch target time with comparison time so that the lower time is up front
var temp = timeArray[i];
timeArray[i] = timeArray[j];
timeArray[j] = temp;
//Then switch the users around so that the user always corresponds with their time
var userTemp = userArray[i];
userArray[i] = userArray[j];
userArray[j] = userTemp;
//The loop will continue if even a single switch is made
keepSorting = true;
}
}
}
if(!keepSorting){
stopSort = true;
}
}
//String building starts here
var placeArray = [':first_place: **1st', ':second_place: **2nd', ':third_place: **3rd', '**4th', '**5th', '**6th', '**7th', '**8th', '**9th', '**10th'];
for(i = 0; i < totalObj; i++){
string = await string.concat(placeArray[i] + ": " + userArray[i] + "** - " + timeArray[i] + " \n\n");
console.log('butt');
}
console.log("This String:" + string);
return string;
}
I think problem is you are trying to await function with callback, it will not work => access to data.time may run before data = result[value]. If you need await callback, you can use custom Promise (or use util.promisify, more info here)
Promise:
function findStats(options) {
return new Promise((resolve, reject) => {
return stats.find(options, function (err, result) {
if (err) {
return reject(err)
}
return resolve(result)
})
})
}
utils.promisify
const util = require('util');
const findStats = util.promisify(stats.find);
Now you can use await in your function
async function getData(field, value) {
try {
const result = await findStats({})
const data = result.value
if (field === 'user') {
return data.user
}
if (field === 'time') {
return data.time
}
return 0
} catch (error) {
// here process error the way you like
// or remove try-catch block and sanitize error in your wrap function
}
}
Here's the complete code I'm trying to run. I will, however, censor the url of the website for personal reasons. I am trying to scrape titles from a very slow website which occasionally sends error status codes in the 4xx range because that, so to handle that I throw an error then retry fetching the same pages after a couple of seconds. The problem is that this error is never caught by the "catch" block. Any idea what I am doing wrong?
const URL = "https://webpage.com/page=";
const SELECTOR = ".post-title.entry-title>a";
const MAX_CONCURRENT_FETCH = 5;
const NB_OF_PAGES = 125;
const ERROR_WAIT_TIME = 20000;
const titles = [];
const parser = new DOMParser();
function fetchPages() {
helper(1, MAX_CONCURRENT_FETCH);
function helper(first, last) {
const requests = [];
console.log("\n" + "*".repeat(40));
for (let i = first; i <= last; i++) {
requests.push(fetch(URL + i));
console.log(`Fetching page: ${i}`);
}
console.log("*".repeat(40) + "\n");
try {
Promise.all(requests).then(responses => {
responses.forEach(async(response, i) => {
/* if no code errors, parse the page, extract the titles then add them to the "titles" array */
if (response.ok) {
const htmlPage = await response.text();
console.log("\n" + "*".repeat(40));
console.log(`Extracting titles from page: ${first+i}`);
console.log("*".repeat(40) + "\n");
const htmlObject = parser.parseFromString(htmlPage, "text/html");
htmlObject.querySelectorAll(SELECTOR).forEach(node => {
titles.push(node.textContent);
console.log(`Title: ${node.textContent}`);
});
}
/* code error, get out of the forEach method by throwing an error */
else {
console.log("*".repeat(40) + "\n");
console.log("Throwing error...");
throw {
response: response
}
}
});
/* keep fetching until the last page */
if (last + 1 < NB_OF_PAGES)
helper(last + 1, last + MAX_CONCURRENT_FETCH);
/* once all pages have been fetched, show the result on screen */
else showResult();
});
}
/* catch the error that was throw inside the forEach method, show the status code error and how long before the pages will be fetched again */
catch (err) {
console.log("Error captured...");
console.log(`Status Code: ${err.response.status}, retrying in ${ERROR_WAIT_TIME/1000} seconds.`);
console.log("*".repeat(40) + "\n");
setTimeout(() => helper(first, last), ERROR_WAIT_TIME);
}
}
}
function showResult() {
const uniqueTitles = [...new Set(titles)].sort();
const titlesUl = document.createElement("ul");
uniqueTitles.forEach(title => {
const titleLi = document.createElement("li");
titleLi.textContent = title;
titlesUl.appendChild(titleLi);
});
document.body.innerHTML = "";
document.body.appendChild(titlesUl);
}
fetchPages();
The try...catch statement contains Promises.all(). This is a non-blocking asynchronous function. The try...catch only applies to initializing the Promises.all() function, it does not catch resolved Promises.
Also note that you are sending all fetch requests in parallel, this may overload the server somewhat. It would be more polite to fire requests one by one using async await.
OK, so I have a situation where I cannot just fire thousands of requests to an API server.
I have a Node process (no UI) that I need to have process each API response/update sequentially, waiting for completion before sending the next request.
I may be making this more complicated than I think - not sure. I can only figure out how to do this with recursive calls, but this results in a stack overflow as there can be thousands of records. The general process is this:
get rows from SQL table with ID's (result)
formulate and send of an API call to retrieve ID's info
if returned data has image data, write it back to SQL table
wait on this process so not to bombard API server with thousands of requests all at once
repeat until last ID is processed (can be thousands, more than stack space)
Here's sample code (not actual so ignore syntax errors if any)...
UPDATED: actual running code with sensitive items removed
var g_con = null; //...yeah I know, globals are bad
//
// [ found updating ]
//
function getSetImage(result, row, found) {
if(row >= result.length) { //...exit on no row or last row processed
con.end();
return;
}
item = result[row]; //...next SQL row
if((item !== undefined) && (item.autoid !== undefined)) {
//...assemble API and send request
//
let url = 'https://...API header...'
+ item.autoid
+ '...API params...';
request(url, (error, response, body) => {
if(response.statusCode !== 200)
throw('Server is not responding\n' + response.statusMessage);
let imageData = JSON.parse(body);
if((imageData.value[0] !== undefined) &&
(imageData.value[0].DETAIL !== undefined) &&
(imageData.value[0].DETAIL.Value.length) ) {
//...post back to SQL
//
found++;
console.log('\n' + item.autoid + '/['+ item.descr + '], ' + 'Found:' + found);
qry = 'update inventory set image = "'+imageData.value[0].DETAIL.Value+'" where autoid = "'+item.autoid+'";';
g_con.query(qry, (err) => {
if (err) {
console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
throw err.message;
}
});
row++;
setTimeout(()=>{getSetImage(result, row, found)}, 0); //...nested call after SQL
} else {
row++;
process.stdout.write('.'); //...show '.' for record, but no image
setTimeout(()=>{getSetImage(result, row, found)}, 0); //...nested call after SQL
}
}); //...request callback
}
// } else {
// throw '\nERROR! result['+row+'] undefined? Images found: '+found;
// }
}
//
// [ main lines ]
//
(() => {
let params = null;
try {
params = JSON.parse(fs.readFileSync('./config.json'));
//...load autoids array from SQL inventory table - saving autoids
// autoids in INVENTRY join on par_aid's in INVENTRYIMAGES
//
g_con = mysql.createConnection(params.SQLConnection);
g_con.connect((err) => { if(err) {
console.log('ERROR:',err.message);
throw err.message;
}
});
//...do requested query and return data or an error
//
let qry = 'select autoid, descr from inventory order by autoid;';
g_con.query(qry, (err, results, flds) => {
if (err || flds === undefined) {
console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
throw err.message;
}
console.log('Results length:',results.length);
let row = 0;
let found = 0;
getSetImage(results, row, found);
});
}
catch (err) {
console.log('Error parsing config parameters!');
console.log(err);
}
})();
So here's the answer using Promises (except for MySQL):
//
// [ found updating ]
//
async function getSetImage(data) {
for(let item of data) {
if(item && item.autoid) {
//...assemble API and send request
//
let url = g_URLHeader + g_URLPartA + item.autoid + g_URLPartB;
let image = await got(url).json().catch(err => {
console.log(err);
err.message = 'API server is not responding';
throw err;
});
if(image && image.value[0] && image.value[0].DETAIL &&
image.value[0].DETAIL.Value.length ) {
console.log('\nFound: ['+item.autoid+' - '+item.descr
+ '] a total of ' + g_found + ' in ' + g_count + ' rows');
g_found++;
//...post back to SQL
//
let qry = 'update inventory set image = "'
+ image.value[0].DETAIL.Value
+ '" where autoid = "'
+ item.autoid+'";';
await g_con.query(qry, (err) => {
if (err) {
console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
throw err.message;
}
});
} else {
process.stdout.write('.'); //...show '.' for record, but no image
} //...if/else image.value
g_count++;
} //...if item
} //...for()
}
As I've said in all my comments, this would be a ton simpler using promises and async/await. To do that, you need to switch all your asynchronous operations over to equivalents that use promises.
Here's a general outline based on the original pseudo-code you posted:
// use got() for promise version of request
const got = require('got');
// use require("mysql2/promise" for promise version of mysql
async function getSetImage(data) {
for (let item of data) {
if (item && item.id) {
let url = uriHeader + uriPartA + item.id + uriPartB;
let image = await got(url).json().catch(err => {
// log and modify error, then rethrow
console.log(err);
err.msg = 'API Server is not responding\n';
throw err;
});
if (image.value && image.value.length) {
console.log('\nFound image for ' + item.id + '\n');
let qry = 'update inventory set image = "' + image.value + '" where id = "' + item.id + '";';
await con.query(qry).catch(err => {
console.log('ERROR:', err.message, '\nSQL:[' + err.sql + ']\n');
throw err;
});
}
} else {
// no image data found
process.stdout.write('.'); //...show '.' for record, but no image
}
}
}
//...sql query is done, returning "result" - data rows
getSetImage(result).then(() => {
console.log("all done");
}).catch(err => {
console.log(err);
});
Some notes about this code:
The request() library is no longer getting new features and is in maintenance mode and you need to change to a different library to get built-in promise support. You could use request-promise (also in maintenance mode), but I recommend one of the newer libraries such as got() that is more actively being developed. It has some nice features (automatically checks status for you to be 2xx, built-in JSON parsing, etc...) which I've used above to save code.
mysql2/promise has built-in promise support which you get with const mysql = require('mysql2/promise');. I'd recommend you switch to it.
Because of the user of async/await here, you can just loop through your data in a regular for loop. And, no recursion required. And, no stack build-up.
The way promises work by default, any rejected promises will automatically terminate the flow here. The only reason I'm using .catch() in a couple places is just for custom logging and tweaking of the error object. I then rethrow which propagates the error back to the caller for you.
You can tweak the error handling to your desire. The usual convention with promises is to throw an Error object (not a string) and that's often what callers are expecting to see if the promise rejects.
This code can be easily customized to log errors and continue on to subsequent items in the array. Your original code did not appear to do that so I wrote it to abort if it got an error.