So I was following along an MDN article on promises and was wondering how to modify the following code to be able to work for any number of files (not just 3).
function fetchAndDecode(url) {
return fetch(url).then(response => {
if(!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
} else {
if(response.headers.get("content-type") === "image/jpeg") {
return response.blob();
} else if(response.headers.get("content-type") === "text/plain") {
return response.text();
}
}
})
.catch(e => {
console.log(`There has been a problem with your fetch operation for resource "${url}": ` + e.message);
})
.finally(() => {
console.log(`fetch attempt for "${url}" finished.`);
})
}
let coffee = fetchAndDecode('coffee.jpg');
let tea = fetchAndDecode('tea.jpg');
let description = fetchAndDecode('description.txt');
Promise.all([coffee, tea, description]).then(values => {
console.log(values);
// Store each value returned from the promises in separate variables; create object URLs from the blobs
let objectURL1 = URL.createObjectURL(values[0]);
let objectURL2 = URL.createObjectURL(values[1]);
let descText = values[2];
// Display the images in <img> elements
let image1 = document.createElement('img');
let image2 = document.createElement('img');
image1.src = objectURL1;
image2.src = objectURL2;
document.body.appendChild(image1);
document.body.appendChild(image2);
// Display the text in a paragraph
let para = document.createElement('p');
para.textContent = descText;
document.body.appendChild(para);
});
MDN specifically notes that "If you were improving this code, you might want to loop through a list of items to display, fetching and decoding each one, and then loop through the results inside Promise.all(), running a different function to display each one depending on what the type of code was. This would make it work for any number of items, not just three." I'm not sure how to do this though, and would appreciate help. Thanks.
The second part of the code could be generalised as follows:
let urls = ['coffee.jpg', 'tea.jpg', 'description.txt'];
Promise.all(urls.map(fetchAndDecode)).then(values => {
let elem;
for (let value of values) {
if (value instanceof Blob) {
elem = document.createElement('img');
elem.src = URL.createObjectURL(value);
} else if (typeof value === "string") {
elem = document.createElement('p');
elem.textContent = value;
} else {
console.log("unexpected value type");
continue;
}
document.body.appendChild(elem);
}
});
const resources = ['coffee.jpg', 'tea.jpg', 'description'];
const resourceRequests = resources.map(fetchAndDecode);
Promise.all(resourceRequests.then(values => {
...
Is one way to implement the suggestion. This approach allows for easier modification of the list of resources, but doesn't really change any of the Promise code.
The .map code above is equivalent to (resource => fetchAndDecode(resource)) since fetchAndDecode takes only the first argument that .map would pass to it.
Related
I am trying to fetch JSON data from the WordPress Developer Reference site. I need to search a keyword without knowing if it's a function, class, hook, or method, which is part of the url I need to fetch. So I'm using Promise.all to cycle through all possible urls. It works if the response.status <= 299, throwing the error immediately, and if the response is ok, then it continues to .then. Fine, but occasionally it will return an ok status if the JSON exists and only returns an empty array. So I need to check if the JSON data is an empty array, which I can't seem to do in the first part. I can only check in the second part as far as I know. And if it throws the error it doesn't continue trying the other urls. Any suggestions?
var keyword = 'AtomParser';
const refs = ['function', 'hook', 'class', 'method'];
// Store the promises
let promises = [];
// Cycle through each type until we find one we're looking for
for (let t = 0; t < refs.length; t++) {
const url =
'https://developer.wordpress.org/wp-json/wp/v2/wp-parser-' +
refs[t] +
'?search=' +
keyword;
// console.log(url);
promises.push(fetch(url));
}
Promise.all(promises)
.then(function(response) {
console.log(response[0]);
// Get the status
console.log('Status code: ' + response[0].status);
if (response[0].status <= 299) {
// The API call was successful!
return response[0].json();
} else {
throw new Error('Broken link status code: ' + response[0].status);
}
})
.then(function(data) {
// This is the HTML from our response as a text string
console.log(data);
// Make sure we have data
if (data.length == 0) {
throw new Error('Empty Array');
}
// ref
const reference = data[0];
// Only continue if not null or empty
if (reference !== null && reference !== undefined && data.length > 0) {
// Success
// Return what I want from the reference
}
})
.catch(function handleError(error) {
console.log('Error' + error);
});
Is there some way to get the JSON data in the first part so I can check if it's in an array while I'm checking the response status?
I would recommend encapsulating the success / failure logic for individual requests, then you can determine all the resolved and rejected responses based on the result of that encapsulation.
For example
const checkKeyword = async (ref, keyword) => {
const params = new URLSearchParams({ search: keyword });
const res = await fetch(
`https://developer.wordpress.org/wp-json/wp/v2/wp-parser-${encodeURIComponent(
ref
)}?${params}`
);
if (!res.ok) {
throw new Error(`${res.status}: ${await res.text()}`);
}
const data = await res.json();
if (data.length === 0) {
throw new Error(`Empty results for '${ref}'`);
}
return { ref, data };
};
Now you can use something like Promise.any() or Promise.allSettled() to find the first successful request or all successful requests, respectively
const keyword = "AtomParser";
const refs = ["function", "hook", "class", "method"];
const promises = refs.map((ref) => checkKeyword(ref, keyword));
// First success
Promise.any(promises)
.then(({ ref, data }) => {
console.log(ref, data);
})
.catch(console.error);
// All successes
Promise.allSettled(promises)
.then((responses) =>
responses.reduce(
(arr, { status, value }) =>
status === "fulfilled" ? [...arr, value] : arr,
[]
)
)
.then((results) => {
// results has all the successful responses
});
For whatever reason I couldn't get Phil's answer to work, so I ended up doing the following which works fine for me. (This is for a discord bot in case you're wondering what the other stuff is all about).
var keyword = 'AtomParser';
const refs = ['function', 'hook', 'class', 'method'];
// Store the successful result or error
let final: any[] = [];
let finalError = '';
// Cycle through each type until we find one we're looking for
for (let t = 0; t < refs.length; t++) {
const url =
'https://developer.wordpress.org/wp-json/wp/v2/wp-parser-' +
refs[t] +
'?search=' +
keyword;
console.log(url);
// Try to fetch it
await fetch(url)
.then(function (response) {
console.log(response);
// Get the status
console.log('Status code: ' + response.status);
if (response.status > 299) {
finalError = '`' + refs[t] + '` does not exist.';
throw new Error(finalError);
} else {
// The API call was successful!
return response.json();
}
})
.then(function (data) {
// This is the HTML from our response as a text string
console.log(data);
// Make sure we have data
if (data.length == 0) {
finalError = "Sorry, I couldn't find `" + keyword + '`';
throw new Error(finalError);
}
// Only continue if not null or empty
if (data[0] !== null && data[0] !== undefined && data.length > 0) {
for (let d = 0; d < data.length; d++) {
// Add it to the final array
final.push(data[d]);
}
}
})
.catch(function handleError(error) {
console.log(error);
});
}
if (final.length > 0) {
for (let f = 0; f < final.length; f++) {
// ref
const reference = final[f];
// Get the link
const link = reference.link;
// Get the title
var title = reference.title.rendered;
title = excerpt.replace('>', '>');
// Get the excerpt
var excerpt = reference.excerpt.rendered;
excerpt = excerpt.replace('<p>', '');
excerpt = excerpt.replace('</p>', '');
excerpt = excerpt.replace('<b>', '**');
excerpt = excerpt.replace('</b>', '**');
console.log(excerpt);
message.reply(
new discord.Embed({
title: `${title}`,
url: link,
description: `${excerpt}\n\n`,
footer: {
text: `WordPress Developer Code Reference\nhttps://developer.wordpress.org/`,
},
})
);
}
} else if (finalError != '') {
message.reply(finalError);
} else {
message.reply('Something went wrong...');
}
wp module
#Phil's answer puts you on the right track but I want to expand on some of his ideas. Use of URLSearchParamas is great but you can improve by using the high-level URL API and forego encodeURIComponent and constructing search params manually. Notice I'm putting this code in its own wp module so I can separate concerns more easily. We don't want all of this code leaking into your main program.
// wp.js
import { fetch } from "whatwg-fetch" // or your chosen implementation
const baseURL = "https://developer.wordpress.org"
async function search1(path, query) {
const u = new URL(path, baseURL)
u.searchParams.set("search", query)
const result = await fetch(u)
if (!result.ok) throw Error(`Search failed (${result.status}): ${u}`)
return result.json()
}
search1 searches one path, but we can write search to search all the necessary paths. I don't think there's any reason to get fancy with each path here, so just write them out -
// wp.js (continued)
function search(query) {
const endpoints = [
"/wp-json/wp/v2/wp-parser-function",
"/wp-json/wp/v2/wp-parser-hook",
"/wp-json/wp/v2/wp-parser-class",
"/wp-json/wp/v2/wp-parser-method"
]
return Promise
.all(endpoints.map(e => search1(e, query)))
.then(results => results.flat())
}
export { search }
main module
Notice we only exported search as search1 is internal to the wp module. Let's see how we can use it in our main module now -
// main.js
import { search } from "./wp.js"
for (const result of await search("database"))
if(result.guid.rendered)
console.log(`${result.title.rendered}\n${result.guid.rendered}\n`)
In this example, we first search for "database" -
wp_should_replace_insecure_home_url()
https://developer.wordpress.org/reference/functions/wp_should_replace_insecure_home_url/
wp_delete_signup_on_user_delete()
https://developer.wordpress.org/reference/functions/wp_delete_signup_on_user_delete/
get_post_datetime()
https://developer.wordpress.org/reference/functions/get_post_datetime/
wp_ajax_health_check_get_sizes()
https://developer.wordpress.org/reference/functions/wp_ajax_health_check_get_sizes/
wp_should_replace_insecure_home_url
https://developer.wordpress.org/reference/hooks/wp_should_replace_insecure_home_url/
comments_pre_query
https://developer.wordpress.org/reference/hooks/comments_pre_query/
users_pre_query
https://developer.wordpress.org/reference/hooks/users_pre_query/
WP_Object_Cache
http://developer.wordpress.org/reference/classes/wp_object_cache/
wpdb
http://developer.wordpress.org/reference/classes/wpdb/
WP_REST_Menu_Items_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_menu_items_controller/prepare_item_for_database/
WP_REST_Global_Styles_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_global_styles_controller/prepare_item_for_database/
WP_REST_Menus_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_menus_controller/prepare_item_for_database/
WP_REST_Templates_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_templates_controller/prepare_item_for_database/
WP_REST_Application_Passwords_Controller::prepare_item_for_database()
https://developer.wordpress.org/reference/classes/wp_rest_application_passwords_controller/prepare_item_for_database/
wpdb::db_server_info()
https://developer.wordpress.org/reference/classes/wpdb/db_server_info/
WP_REST_Attachments_Controller::insert_attachment()
https://developer.wordpress.org/reference/classes/wp_rest_attachments_controller/insert_attachment/
WP_Debug_Data::get_database_size()
https://developer.wordpress.org/reference/classes/wp_debug_data/get_database_size/
WP_REST_Meta_Fields::update_multi_meta_value()
https://developer.wordpress.org/method/wp_rest_meta_fields/update_multi_meta_value/
another search example
Now let's search for "image" -
for (const result of await search("image"))
if(result.guid.rendered)
console.log(`${result.title.rendered}\n${result.guid.rendered}\n`)
get_adjacent_image_link()
https://developer.wordpress.org/reference/functions/get_adjacent_image_link/
get_next_image_link()
https://developer.wordpress.org/reference/functions/get_next_image_link/
get_previous_image_link()
https://developer.wordpress.org/reference/functions/get_previous_image_link/
wp_robots_max_image_preview_large()
https://developer.wordpress.org/reference/functions/wp_robots_max_image_preview_large/
wp_getimagesize()
https://developer.wordpress.org/reference/functions/wp_getimagesize/
is_gd_image()
https://developer.wordpress.org/reference/functions/is_gd_image/
wp_show_heic_upload_error()
https://developer.wordpress.org/reference/functions/wp_show_heic_upload_error/
wp_image_src_get_dimensions()
https://developer.wordpress.org/reference/functions/wp_image_src_get_dimensions/
wp_image_file_matches_image_meta()
https://developer.wordpress.org/reference/functions/wp_image_file_matches_image_meta/
_wp_check_existing_file_names()
https://developer.wordpress.org/reference/functions/_wp_check_existing_file_names/
edit_custom_thumbnail_sizes
https://developer.wordpress.org/reference/hooks/edit_custom_thumbnail_sizes/
get_header_image_tag_attributes
https://developer.wordpress.org/reference/hooks/get_header_image_tag_attributes/
image_editor_output_format
https://developer.wordpress.org/reference/hooks/image_editor_output_format/
wp_image_src_get_dimensions
https://developer.wordpress.org/reference/hooks/wp_image_src_get_dimensions/
wp_get_attachment_image
https://developer.wordpress.org/reference/hooks/wp_get_attachment_image/
image_sideload_extensions
https://developer.wordpress.org/reference/hooks/image_sideload_extensions/
wp_edited_image_metadata
https://developer.wordpress.org/reference/hooks/wp_edited_image_metadata/
wp_img_tag_add_loading_attr
https://developer.wordpress.org/reference/hooks/wp_img_tag_add_loading_attr/
wp_image_file_matches_image_meta
https://developer.wordpress.org/reference/hooks/wp_image_file_matches_image_meta/
get_custom_logo_image_attributes
https://developer.wordpress.org/reference/hooks/get_custom_logo_image_attributes/
Custom_Image_Header
http://developer.wordpress.org/reference/classes/custom_image_header/
WP_Image_Editor_Imagick
http://developer.wordpress.org/reference/classes/wp_image_editor_imagick/
WP_Embed
http://developer.wordpress.org/reference/classes/wp_embed/
WP_Image_Editor
http://developer.wordpress.org/reference/classes/wp_image_editor/
WP_Customize_Background_Image_Setting
http://developer.wordpress.org/reference/classes/wp_customize_background_image_setting/
WP_Customize_Header_Image_Setting
http://developer.wordpress.org/reference/classes/wp_customize_header_image_setting/
WP_Image_Editor_GD
http://developer.wordpress.org/reference/classes/wp_image_editor_gd/
WP_Customize_Header_Image_Control
http://developer.wordpress.org/reference/classes/wp_customize_header_image_control/
WP_REST_Server::add_image_to_index()
https://developer.wordpress.org/reference/classes/wp_rest_server/add_image_to_index/
WP_REST_URL_Details_Controller::get_image()
https://developer.wordpress.org/reference/classes/wp_rest_url_details_controller/get_image/
WP_Image_Editor::get_default_quality()
https://developer.wordpress.org/reference/classes/wp_image_editor/get_default_quality/
WP_Theme_JSON::get_blocks_metadata()
https://developer.wordpress.org/reference/classes/wp_theme_json/get_blocks_metadata/
WP_Image_Editor_Imagick::pdf_load_source()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/pdf_load_source/
WP_Image_Editor_Imagick::write_image()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/write_image/
WP_Image_Editor_Imagick::maybe_exif_rotate()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/maybe_exif_rotate/
WP_Image_Editor_Imagick::make_subsize()
https://developer.wordpress.org/reference/classes/wp_image_editor_imagick/make_subsize/
WP_Image_Editor_GD::make_subsize()
https://developer.wordpress.org/reference/classes/wp_image_editor_gd/make_subsize/
empty search result
Searching for "zzz" will yield no results -
for (const result of await search("zzz"))
if(result.guid.rendered)
console.log(`${result.title.rendered}\n${result.guid.rendered}\n`)
<empty result>
I am unable to pass any object or arrays to IPCRenderer.
I am getting error when passing an object or array through ipcs, I have even tried to send by converting to string using JSON.stringify but it converts it into empty object string.
I have tried passing a fileList, an array of object & even an object nothing passes. only string or handwritten objects are working.
I've read that it uses Structured Clone Algorithm and fileList & Array is allowed by this algorithm
ERROR:
electron/js2c/renderer_init.js:74 Uncaught Error: An object could not be cloned.
at EventEmitter.i.send.i.send (electron/js2c/renderer_init.js:74)
at HTMLButtonElement.compressNow (ImageHandling.js:190)
I have tried many possible solutions but nothing worked
code:
const compressNow = () => {
ipcRenderer.send("image:compress", filess). ///This is the error.
// filess is a variable containing an array of selected files from an HTML input.
}
Now i have tried to send filess as JSON.stringify, i tried to send it as an object but nothing works unless i manually write a dummy object or string.
Here's My Github Repo for this project
Files With ErrorJ:-
ImageHandling.js
const fs = window.require('fs');
const {ipcRenderer} = require("electron")
const SELECT = (target) => document.querySelector(`${target}`)
var filess = []
const imgUploadInput = SELECT("#imgUploadInput")
const warning = SELECT("#warning")
const setImgBase64 = (imgEl, file) => {
const ReadAbleFile = fs.readFileSync(file.path).toString('base64')
let src = "data:image/png;base64," + ReadAbleFile
imgEl.setAttribute("src", src)
// El.src=src
// console.log(`FIXED IMAGE # ${imgEl} `,ReadAbleFile)
}
const renderImages = () => {
const files = filess && Array.from(filess)
const defaultImg = SELECT("#defaultImg")
const addImgBtn = SELECT("#addImgBtn")
imgUploadInput.disabled = true;
let numOfFiles = files.length
if (numOfFiles < 1) {
SELECT("#compressContainer").style.visibility = "hidden"
} else {
SELECT("#compressContainer").style.visibility = "visible"
}
if (numOfFiles > 49) {
warning.innerHTML = `<b style="font-weight:bold; color:red;">WARNING:</b><br/>
<span style="padding:10px;text-align:left">
Your processor/computer may not be able to process ${numOfFiles} Images at once, We recommend selecting less than 50 Images at once for better performance.
</span>
`;
}
addImgBtn.innerHTML = `LOADING.....`
if (defaultImg && numOfFiles > 0)
defaultImg.remove();
setTimeout(() => {
if (files && numOfFiles > 0) {
let displayImages = SELECT("#displayImages")
displayImages.innerHTML = ""
files ?. forEach((file, i) => {
let divEl = document.createElement("div")
let imgEl = document.createElement("img")
imgEl.src = file.path
imgEl.id = `PNG_${i}_${
btoa(file.name)
}`
divEl.className = "displayedImg"
imgEl.setAttribute("onclick", `document.getElementById('ImageView').src=this.src`)
const a = document.createElement("a")
a.appendChild(imgEl)
a.setAttribute("href", `#ViewImage`)
a.className = "perfundo__link"
divEl.appendChild(a)
divEl.className = "displayedImg perfundo"
displayImages.appendChild(divEl)
if (i == files.length - 1) {
warning.innerHTML = "";
updateNumOfImages();
}
imgEl.onerror = () => setImgBase64(imgEl, file) // converting to base64 only on error, this make performance better and help us avoid freezes. (before this i was converting all images to base64 wither errored or not that was making computer freez)
})
addImgBtn.innerHTML = "+ Add MORE"
imgUploadInput.disabled = false
findDuplicate()
}
}, 0);
}
const hasDuplicate=()=>{
let FileNames = [... filess.map(f => f.name)]
let duplicateFiles = filess.filter((file, i) => FileNames.indexOf(file.name) !== i)
return {FileNames,duplicateFiles,FilesLength:duplicateFiles.length}
}
const findDuplicate = (forceAlert = false) => {
if (filess && filess.length) {
let {FileNames} = hasDuplicate()
let {duplicateFiles} = hasDuplicate()
if (duplicateFiles.length) { // alert(``)
let countFiles = duplicateFiles.length
let fileStr = countFiles > 1 ? "files" : "file"
console.log("result from removeDup=> ", filess, " \n dupfilename=> ", FileNames, " \n dupfiles=> ", duplicateFiles)
let shouldNotAsk = localStorage.getItem("NeverAsk")
let msg = `You've selected ${
countFiles > 1 ? countFiles : "a"
} duplicate ${fileStr}`
let duplInner = `<span style='color:red'>
<b>WARNING</b>
<p style="margin:0px;line-height:1"> ${msg} . <button onClick="findDuplicate(true)" type="button" class="btn btn-danger btn-rounded btn-sm">REMOVE DUPLICATE</button></p>
</span>`
if (! shouldNotAsk || forceAlert) {
swal("DUPLICATE FILES DETECTED", `${msg} , Would you like to un-select duplicate ${fileStr} having same name?`, {
icon: 'warning',
dangerMode: true,
buttons: {
cancel: true,
...forceAlert ? {} : {
never: "Never Ask"
},
confirm: "Yes !"
}
}).then((Yes) => {
if (Yes == "never") {
localStorage.setItem("NeverAsk", true)
warning.innerHTML=duplInner
} else if (Yes) {
removeDuplicates()
}
})
} else {
warning.innerHTML=duplInner
}
}
}
}
const removeDuplicates = (showAlert=true) => {
let {FileNames} = hasDuplicate()
let {duplicateFiles} = hasDuplicate()
let duplicateFileNames = duplicateFiles.map(f => f.name)
let uniqueFiles = filess.filter((file) => ! duplicateFileNames.includes(file.name))
filess = [
... uniqueFiles,
... duplicateFiles
]
console.log("result from removeDup=> ", filess, " \n filename=> ", FileNames, " \n dupfiles=> ", duplicateFiles, "\n unique fil=> ", uniqueFiles)
renderImages()
if(showAlert){
swal("DONE", "Removed Duplicate Files ", {icon: 'success'}).then(() =>{
renderImages()
setTimeout(() => {
let hasDuplicateFiles = hasDuplicate().FilesLength
if(hasDuplicate){//Re-check if any duplicate files left after the current removal process.
removeDuplicates(false) //Re-run the function to remove remaining. false will make sure that this alert does not show and the loop does not continue.
}
renderImages()
}, 10);
})
}
}
const updateNumOfImages = () => {
warning.innerHTML = `
<span style="text-align:left; color:green">
Selected ${
filess.length
} Image(s)
</span>
`;
}
const compressNow = () => {
ipcRenderer.send("image:compress", filess)
// alert("WOW")
}
CompressBtn.addEventListener("click", compressNow)
imgUploadInput.addEventListener("change", (e) => {
let SelectedFiles = e.target.files
if (SelectedFiles && SelectedFiles.length) {
filess = [
... filess,
... SelectedFiles
]
renderImages()
}
})
// SELECT("#imgUploadInput").addEventListener("drop",(e)=>console.log("DROP=> ",e))
UPDATE:-
I REPLACED THIS:
const compressNow = () => {
ipcRenderer.send("image:compress",filess)
}
INTO THIS:-
const compressNow = () => {
filess.forEach(file => {
ipcRenderer.send("image:compress",file.path )
});
}
Now here i am sending the files one by one via forEach, actually its sending string "file path" so thats how its working i am still confused why do i have to do this? why can't i send whole fileList i assume that this loop method is a bad practice because it will consume more CPU its one additional loop however it won't be necessary if i am able to send the whole array.
See Behavior Changed: Sending non-JS objects over IPC now throws an exception. DOM objects etc. are not serializable. Electron 9.0 (and newer) throws "object could not be cloned" error when unserializable objects are sent.
In your code, File and FileList are DOM objects.
If you want to avoid using forEach, try this code:
const compressNow = () => {
const paths = filess.map(f => f.path);
ipcRenderer.send("image:compress", paths);
}
Can refer to electron github issue tracker for this issue (already closed)
Error: An object could not be cloned #26338
Docs for ipcRenderer.send(channel, ...args)
This issue mainly comes when we have non-cloneable values like function within an object in data we are sending via IPC, to avoid that we can use JSON.stringify() before sending and JSON.parse() later on receiving end, but doing so will cause to lose some of the values eg:
const obj = {
x :10,
foo : ()=>{
console.log('This is non-cloneable value')
}
}
console.log(JSON.stringify(obj))
output:{"x":10}
Instead of sending the images save them in fs and send the path
The simplest thing that could possibly work is to use lodash cloneDeep()
ipcMain.handle('stuffgetList', async () => {
return _.cloneDeep(await stuffStore.getList())
})
in the windows JSON.stringify()
in the main.js JSON.parse()
Remove :compress from. .send method and try
async onSubmit(formValue) {
this.isSubmitted = true;
if(this.selectedImageArray.length > 0) { // 4 images in this array
for (let index = 0; index < this.selectedImageArray.length; index++) { // Loop through this image array
await new Promise(resolve => {
setTimeout(()=> {
console.log('This is iteration ' + index);
var filePath = `images/tours/${this.selectedImageArray[index].name.split('.').slice(0,-1).join('.')}_${new Date(). getTime()}`;
const fileRef = this.storage.ref(filePath);
this.storage.upload(filePath, this.selectedImageArray[index]).snapshotChanges().pipe(
finalize(() => {
fileRef.getDownloadURL().subscribe((url) => {
formValue[`imageUrl${index+1}`] = url;
console.log(url);
});
})
).subscribe()
resolve();
}, 3000);
});
}
console.log('After loop execution');
// this.value(formValue);
}
}
After submitting the code it will download and print 3 urls and then it print 'after loop execution' then it print 4th one I don't understand why. See here in console
see in the image line no of code execution.
What I want to execute code in sequence after all images download then after it will go out of loop.
I wrote another version of this that hopefully works as you expect it to.
First we create an array of all the storage upload snapshot observables.
The we use concat() to run them all in sequence. (If you change from concat() to merge() they will all go at once)
The we use mergeMap to jump over to the getDownloadURL
Then in the subscribe we add the url to the formValues
Finally in the finalize we set the class propery "value" equal to the formValue.
onSubmit(formValue) {
const snapshotObservables = this.selectedImageArray.map(selectedImage => { // 4 images in this array
const filePath = `images/tours/${selectedImage.name.split('.').slice(0, -1).join('.')}_${new Date(). getTime()}`;
return combineLatest(this.storage.upload(filePath, selectedImage).snapshotChanges(), filePath);
});
concat(...snapshotObservables).pipe(
mergeMap(([snapshot, filePath]) => {
const fileRef = this.storage.ref(filePath);
return fileRef.getDownloadURL();
}),
finalize(() => {
this.value(formValue);
})
).subscribe(url => {
formValue[`imageUrl${index+1}`] = url;
});
}
I wrote a new function for multiple file upload
public multipleFileUpload(event, isEncodeNeeded?: Boolean):Array<any> {
if(!isEncodeNeeded){
isEncodeNeeded=false;
}
let fileList = [];
for (let index = 0; index < event.target.files.length; index++) {
let returnData = {};
let file: File = event.target.files[index];
let myReader: FileReader = new FileReader();
returnData['documentName'] = event.target.files[index]['name'];
returnData['documentType'] = event.target.files[index]['type'];
myReader.addEventListener("load", function (e) {
if (myReader.readyState == 2) {
returnData['document'] = isEncodeNeeded ? btoa(e.target['result']) : e.target['result'];
}
});
myReader.readAsBinaryString(file);
fileList.push(returnData);
}
return fileList;
}
In this function event is the event of the input and the isEncodeNeeded is conversion is needed. If this is true then it convert to base64 format.
The output format is
[{
"document": documentbyte,
"documentName": document name,
"documentType": file format
}]
Alright so my problem is that in the first set of console.log(streamXXXX)s, where XXXX are the various variables, when I read their values they all read as they should, while in the second set they read as undefined. Is this a scope issue? Maybe an Async issue? I tried adding awaits to each time I make a web request but nothing seems to work, and one of the most interesting parts about this is the fact that there are no errors?
Anyways, my code is listed below, as well as a link to test it out in Repl using a sample bot I created. Below that is the list of libraries required for said program to run. Thanks!
if (!message.member.voiceChannel) return message.channel.send(`You do realize you have to be in a voice channel to do that, right ${message.author.username}?`)
if (!message.member.voiceConnection) message.member.voiceChannel.join().then(async connection => {
let streamURL = args.slice(1).join(" ")
let streamTitle = "";
let streamThumb = "";
let streamAuth = "";
let streamAuthThumb = "";
if (streamURL.includes("https://www.youtube.com") || streamURL.includes("https://youtu.be/") && !streamURL.includes(' ')) {
youtube.getVideo(streamURL)
.then(async results => {
let {
body
} = await snekfetch.get(`https://www.googleapis.com/youtube/v3/channels?part=snippet&id=${results.channel.id}&fields=items%2Fsnippet%2Fthumbnails&key=${ytapikey}`).query({
limit: 800
})
streamTitle = results.title
streamThumb = results.thumbnails.medium.url
streamAuth = results.channel.title
streamAuthThumb = body.items[0].snippet.thumbnails.medium.url
console.log(streamURL)
console.log(streamTitle)
console.log(streamThumb)
console.log(streamAuth)
console.log(streamAuthThumb)
})
.catch(console.error)
} else if (!streamURL.includes("https://www.youtube.com") || !streamURL.includes("https://youtu.be/")) {
youtube.searchVideos(streamURL)
.then(async results => {
let {
body
} = await snekfetch.get(`https://www.googleapis.com/youtube/v3/channels?part=snippet&id=${results[0].channel.id}&fields=items%2Fsnippet%2Fthumbnails&key=${ytapikey}`).query({
limit: 800
})
streamURL = results[0].url
streamTitle = results[0].title
streamThumb = results[0].thumbnails.default.medium.url
streamAuth = results[0].channel.title
streamAuthThumb = body.items[0].snippet.thumbnails.medium.url
}).catch(console.error)
} else {
return message.reply("I can only play videos from YouTube (#NotSponsored).")
}
console.log(streamURL)
console.log(streamTitle)
console.log(streamThumb)
console.log(streamAuth)
console.log(streamAuthThumb)
const stream = ytdl(streamURL, {
filter: 'audioonly'
})
const dispatcher = connection.playStream(stream)
dispatcher.on("end", end => {
return
})
let musicEmbed = new Discord.RichEmbed()
.setAuthor(streamAuth, streamAuthThumb)
.setTitle(`Now Playing: ${streamTitle}`)
.setImage(streamThumb)
.setColor(embedRed)
.setFooter(`${streamAuth} - ${streamTitle} (${streamURL}`)
await message.channel.send(musicEmbed)
})
Link to test out the program on a sample bot I made
Modules you will need to test this:
discord.js
simple-youtube-api
node-opus
ffmpeg
ffbinaries
ffmpeg-binaries
opusscript
snekfetch
node-fetch
ytdl-core
Thanks again!
The reason why your output is undefined is due to the way promises work and how you structured your code:
let streamTitle = "";
// 1. Promise created
youtube.getVideo(streamURL)
// 2. Promise still pending, skip for now
.then(async results => {
// 4. Promise fulfilled
console.log(results.title); // 5. Logged actual title
});
console.log(streamTitle); // 3. Logged ""
You already have the correct approach for your snekfetch requests, just need to apply it to the YT ones as well:
let streamTitle = "";
const results = await youtube.getVideo(streamURL);
streamTitle = results.title;
console.log(streamTitle); // Desired output
I'm just starting to play around with Puppeteer (Headless Chrome) and Nodejs. I'm scraping some test sites, and things work great when all the values are present, but if the value is missing I get an error like:
Cannot read property 'src' of null (so in the code below, the first two passes might have all values, but the third pass, there is no picture, so it just errors out).
Before I was using if(!picture) continue; but I think it's not working now because of the for loop.
Any help would be greatly appreciated, thanks!
for (let i = 1; i <= 3; i++) {
//...Getting to correct page and scraping it three times
const result = await page.evaluate(() => {
let title = document.querySelector('h1').innerText;
let article = document.querySelector('.c-entry-content').innerText;
let picture = document.querySelector('.c-picture img').src;
if (!document.querySelector('.c-picture img').src) {
let picture = 'No Link'; } //throws error
let source = "The Verge";
let categories = "Tech";
if (!picture)
continue; //throws error
return {
title,
article,
picture,
source,
categories
}
});
}
let picture = document.querySelector('.c-picture img').src;
if (!document.querySelector('.c-picture img').src) {
let picture = 'No Link'; } //throws error
If there is no picture, then document.querySelector() returns null, which does not have a src property. You need to check that your query found an element before trying to read the src property.
Moving the null-check to the top of the function has the added benefit of saving unnecessary calculations when you are just going to bail out anyway.
async function scrape3() {
// ...
for (let i = 1; i <= 3; i++) {
//...Getting to correct page and scraping it three times
const result = await page.evaluate(() => {
const pictureElement = document.querySelector('.c-picture img');
if (!pictureElement) return null;
const picture = pictureElement.src;
const title = document.querySelector('h1').innerText;
const article = document.querySelector('.c-entry-content').innerText;
const source = "The Verge";
const categories = "Tech";
return {
title,
article,
picture,
source,
categories
}
});
if (!result) continue;
// ... do stuff with result
}
Answering comment question: "Is there a way just to skip anything blank, and return the rest?"
Yes. You just need to check the existence of each element that could be missing before trying to read a property off of it. In this case we can omit the early return since you're always interested in all the results.
async function scrape3() {
// ...
for (let i = 1; i <= 3; i++) {
const result = await page.evaluate(() => {
const img = document.querySelector('.c-picture img');
const h1 = document.querySelector('h1');
const content = document.querySelector('.c-entry-content');
const picture = img ? img.src : '';
const title = h1 ? h1.innerText : '';
const article = content ? content.innerText : '';
const source = "The Verge";
const categories = "Tech";
return {
title,
article,
picture,
source,
categories
}
});
// ...
}
}
Further thoughts
Since I'm still on this question, let me take this one step further, and refactor it a bit with some higher level techniques you might be interested in. Not sure if this is exactly what you are after, but it should give you some ideas about writing more maintainable code.
// Generic reusable helper to return an object property
// if object exists and has property, else a default value
//
// This is a curried function accepting one argument at a
// time and capturing each parameter in a closure.
//
const maybeGetProp = default => key => object =>
(object && object.hasOwnProperty(key)) ? object.key : default
// Pass in empty string as the default value
//
const getPropOrEmptyString = maybeGetProp('')
// Apply the second parameter, the property name, making 2
// slightly different functions which have a default value
// and a property name pre-loaded. Both functions only need
// an object passed in to return either the property if it
// exists or an empty string.
//
const maybeText = getPropOrEmptyString('innerText')
const maybeSrc = getPropOrEmptyString('src')
async function scrape3() {
// ...
// The _ parameter name is acknowledging that we expect a
// an argument passed in but saying we plan to ignore it.
//
const evaluate = _ => page.evaluate(() => {
// Attempt to retrieve the desired elements
//
const img = document.querySelector('.c-picture img');
const h1 = document.querySelector('h1')
const content = document.querySelector('.c-entry-content')
// Return the results, with empty string in
// place of any missing properties.
//
return {
title: maybeText(h1),
article: maybeText(article),
picture: maybeSrc(img),
source: 'The Verge',
categories: 'Tech'
}
}))
// Start with an empty array of length 3
//
const evaluations = Array(3).fill()
// Then map over that array ignoring the undefined
// input and return a promise for a page evaluation
//
.map(evaluate)
// All 3 scrapes are occuring concurrently. We'll
// wait for all of them to finish.
//
const results = await Promise.all(evaluations)
// Now we have an array of results, so we can
// continue using array methods to iterate over them
// or otherwise manipulate or transform them
//
results
.filter(result => result.title && result.picture)
.forEach(result => {
//
// Do something with each result
//
})
}
Try-catch worked for me:
try {
if (await page.$eval('element')!==null) {
const name = await page.$eval('element')
}
}catch(error){
name = ''
}