We Keep Coding

How to pass optional parameters using fetch vanilla JS

I'm using JavaScript fetch GET method to call an API. The API returns data; however, there are optional parameters I'd like to pass in to format the data response in a different way. How do you pass optional parameters using the fetch method?
async function getText(){
let passageParam = randomPassage();
//API credit
let Url = 'https://api.esv.org/v3/passage/text?q=' + passageParam + params;
console.log(Url);
//Await - Used with Async
//Suspend function exeeuction until the Async promise settles and returns its result
let response = await fetch(Url, {
method: 'GET',
headers: {
'Authorization': 'myToken'
},
params = {
'indent-poetry': False,
'include-headings': False,
'include-footnotes': False,
'include-verse-numbers': False,
'include-short-copyright': False,
'include-passage-references': False
}
});
if(response.ok){ // if HTTP-status is 200-299
// get the response body
let passage = await response.json();
populateUI(passageParam, passage.passages[0]);
//console.log(passage);
} else{
alert("HTTP-Error: " + response.status);
}
//Function to input json response to HTML
function populateUI(ref, verse){
//strip verse
document.getElementById('reference').innerHTML = ref;
document.getElementById('verse').innerHTML = verse;
}
}

When using fetch with GET, it's generally expecting parameters be sent via a Query String.
You can try something like this:
let passageParam = randomPassage();
let extraParams = '&indent-poetry=False&include-headings=False' +
'&include-footnotes=False&include-verse-numbers=False' +
'&include-short-copyright=False&include-passage-references=False';
let Url = 'https://api.esv.org/v3/passage/text?q=' + passageParam + extraParams;
console.log(Url);
Alternatively you can do something like this:
let passageParam = randomPassage();
let extraParams = {
'indent-poetry': 'False',
'include-headings': 'False',
'include-footnotes': 'False',
'include-verse-numbers': 'False',
'include-short-copyright': 'False',
'include-passage-references': 'False'
}
let Url = 'https://api.esv.org/v3/passage/text?q=' + passageParam +
'&' + (new URLSearchParams(extraParams)).toString();
console.log(Url);
And also delete the params expression.

Since you are making a GET request to a URL-EndPoint using fetch. The URL-EndPint will always return the same data format every time you call it.
And formatting a response is not in our hands in this case. To check all the response details, go to the network tab of Developer Console (do Ctrl+Shift+I), you can see the response headers and other related stuff that you have received in the response and see if any information is useful to you there itself.

Amazon MWS- continue to get signature mismatch error even though the signature matches scratchpad (Javascript)

I am trying to send a POST request to the /Finances endpoint and cannot get past the error that says:
<ErrorResponse xmlns="http://mws.amazonservices.com/schema/Finances/2015-05-01">
<Error>
<Type>Sender</Type>
<Code>SignatureDoesNotMatch</Code>
<Message>The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.</Message>
</Error>
</ErrorResponse>
I have taken the parameters right off of successful queries on Scratchpad and run them through my hashing process and am calculating signatures that match those calculated on Scratchpad so I know my signature process is accurate but even in those instances, I am still receiving the mismatched Signature error. This tells me that maybe there is something else wrong with my request, however I've tried many different variations and mimicked the documentation with no success.
I've tried ordering my parameters in ASCII alphabetical as well as in the order that scratchpad shows (which is not alphabetical) and neither works. There are no typos in any of my keys or extra spaces. I am lost on what could be causing this and would really appreciate some help if you've worked with mws. Below is my code:
const method = "POST";
const hostName = "mws.amazonservices.com";
const requestUri = "/Finances/2015-05-01";
const keyId = XXXXXXXXXXXX;
const action = "ListFinancialEvents";
const authToken = XXXXXXXXXx;
const sellerId = XXXXXXXXX;
const secretKey = XXXXXXXXXXXx;
const signatureVersion = "2";
const timestamp = new Date();
const formattedTimestamp = timestamp.toISOString();
const version = "2015-05-01";
const signatureMethod = "HmacSHA256";
const postedAfter = "2020-06-07T0:00:00Z";
const queryString =
"AWSAccessKeyId=" +
encodeURIComponent(keyId) +
"&Action=" +
encodeURIComponent(action) +
"&MWSAuthToken=" +
encodeURIComponent(authToken) +
"&PostedAfter=" +
encodeURIComponent(postedAfter) +
"&SellerId=" +
encodeURIComponent(sellerId) +
"&SignatureMethod=" +
encodeURIComponent(signatureMethod) +
"&SignatureVersion=" +
encodeURIComponent(signatureVersion) +
"&Timestamp=" +
encodeURIComponent(formattedTimestamp) +
"&Version=" +
encodeURIComponent(version);
const stringToSign =
method + "\n" + hostName + "\n" + requestUri + "\n" + queryString;
const signature = crypto
.createHmac("sha256", secretKey)
.update(stringToSign)
.digest("base64");
const queryBody = querystring.stringify({
AWSAccessKeyId: keyId,
Action: action,
MWSAuthToken: authToken,
PostedAfter: postedAfter,
SellerId: sellerId,
SignatureMethod: signatureMethod,
SignatureVersion: signatureVersion,
Timestamp: formattedTimestamp,
Version: version,
Signature: signature,
});
const options = {
hostname: hostName,
path: requestUri,
method: method,
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Content-Length": queryBody.length,
Host: "mws.amazonservices.com",
"User-Agent": "myMWSApp/1.0 (Language=Javascript)",
},
};
const req = https.request(options, (res) => {
console.log("statusCode", res.statusCode);
console.log("headers:", res.headers);
res.on("data", (d) => {
process.stdout.write(d);
});
});
req.on("error", (e) => {
console.error("ERROR:", e);
});
req.write(queryBody);
req.end();

Why does my for loop mess up all the parameters?

I am trying to parse some data from several web pages using javascript. I wrote a small parser for this purpose. The algorithm looks like this:
Open first URL from my .csv file
Find the data I need on the page
Save URL and data to a json file
My code executes 1. and 2. perfectly but sometimes messes up with number 3. Output looks like this:
URL 1 + data from URL 1 (correct line)
URL 2 + data from URL 2 (correct line)
URL 3 + data from URL 3 (correct line)
URL 4 + data from URL 4 (correct line)
URL 6(wrong URL) + data from another URL
URL 5(wrong URL) + data from another URL
URL 7 + data from URL 7 (correct line)
URL 8 + data from URL 8 (correct line)
URL 9 + data from URL 9 (correct line)
I assume the problem is that some pages load way too long which messes up the whole process. But I still don't understand why it sometimes saves the wrong data.
Heres my code:
var request = require('request');
var cheerio = require('cheerio');
var cloudscraper = require('cloudscraper');
var fs = require('fs');
var path = require('path');
var csvjson = require('csvjson');
//First, we read .csv file with our URL list
function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
end = urlList.length;
logs = [];
//here we start the loop reading and saving data from each url
for (let p = 0; p < end; p += 1){
grabTheData(urlList, p)
}
}
//this code extracts the data from the page and saves it to a json file
function grabTheData(urlList, p){
setTimeout(function() {
url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
getTheList()

The reason this is happening is that there is a potential mismatch between the callback result and the url variable in grabTheData.
Now there is a very quick fix for this, simple change the scope of the url variable like so:
function grabTheData(urlList, p){
setTimeout(function() {
// Set scope of url variable to block
let url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
This should keep your results in order.
Here's another (IMHO much better) option, using promises and avoiding the use of setTimeout to separate calls. This should avoid any potential race condition, since the Promise.all call will preserve order:
async function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
let promiseList = urlList.map(urlEntry => grabTheDataUpdated(urlEntry.ItemLink));
let logs = await Promise.all(promiseList);
fs.writeFileSync('./new_logs.json', JSON.stringify(logs, null, 4));
}
// Promisified version of cloudscraper.get
function getCloudScraperData(url) {
return new Promise((resolve, reject) => {
cloudscraper.get(url, (err, res, body) => {
if (err) {
reject(err);
} else {
resolve ( { url, res, body });
}
})
})
}
function getDataINeed(url, body) {
// Use cheerio to process data..
// Return mock data for now.. replace with actual data processed by cheerio..
return `data from ${url}`;
}
async function grabTheDataUpdated(url) {
try {
let result = await getCloudScraperData(url);
let dataINeed = getDataINeed(result.url, result.body);
return { url, dataINeed };
} catch (error) {
return { url, dataINeed: "Error occurred: " + error.message };
}
}

Async function must return a boolean value

I have a method that I am calling on the onsubmit event in the form tag.
So I need a true or false to be returned from the method.
I use an API to retrieve data, and according to the response from the API, I return true or false. But because it is an async function thats running, I cant get it right to wait for the response from the API, analyze it and then return my decision.
Any ideas on how I could solve this problem
function GetPolygonID()
{
document.getElementById("displayerror").innerHTML = "";
var retrievedpoly = document.getElementById('polygondetails').value;
var parts = retrievedpoly.split('coordinates');
var parttoadd = parts[1].substring(0, parts[1].length - 2) + "}";
console.log(parttoadd);
var myx = '{"name":"Polygon OneTwoThree","geo_json":{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates' + parttoadd;
var url = 'http://api.agromonitoring.com/agro/1.0/polygons?appid=apiid';
const request = async() => {
const response = await fetchPoly(url, myx);
const data = await response.json();
const errorCheck = await CheckInfo(data);
console.log("2: " + errorCheck);
return await errorCheck;
};
return request();
}
function CheckInfo(data)
{
let flag = false;
console.log(data);
if (JSON.stringify(data).includes("Geo json Area is invalid. Available range: 1 - 3000 ha"))
{
var myval = JSON.stringify(data);
//myval = myval.replace(/\\n/g,"<br/>");
parts = myval.split("\\n ").join(",").split("\\n");
console.log(parts);
var todisplay = parts[1].substring(10);
todisplay += ("<br/>" + parts[2].substring(10).replace(",", "<br/>").replace("c", "C"));
console.log(todisplay);
document.getElementById("displayerror").innerHTML = todisplay;
} else
{
flag = true;
}
console.log("1:" + flag);
return flag;
}
function fetchPoly(url, data)
{
return fetch(url, {
method: "POST", // *GET, POST, PUT, DELETE, etc.
mode: "cors", // no-cors, cors, *same-origin
cache: "no-cache", // *default, no-cache, reload, force-cache, only-if-cached
credentials: "same-origin", // include, *same-origin, omit
headers: {
"Content-Type": "application/json"
// "Content-Type": "application/x-www-form-urlencoded",
},
redirect: "follow", // manual, *follow, error
referrer: "no-referrer", // no-referrer, *client
body: data // body data type must match "Content-Type" header
});
}
I did try it with .then(), originally, then I broke it down like this, as I thought it would be easier to return a value here.
Essentially I need GetPolygonID() to return a boolean which it gets from CheckInfo(). CheckInfo() determines if the form should submit or not
Any thought on how I could fix this?
Thank You

GetPolygonID() function returns a Promise, so it must be either called with await or you can call then upon it:
var res = await GetPolygonID();
GetPolygonID().then(res => console.log(res));
You can make the whole function async:
async function GetPolygonID() {
document.getElementById("displayerror").innerHTML = "";
var retrievedpoly = document.getElementById('polygondetails').value;
var parts = retrievedpoly.split('coordinates');
var parttoadd = parts[1].substring(0, parts[1].length - 2) + "}";
console.log(parttoadd);
var myx = '{"name":"Polygon OneTwoThree","geo_json":{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates' + parttoadd;
var url = 'http://api.agromonitoring.com/agro/1.0/polygons?appid=apiid';
const response = await fetchPoly(url, myx);
const data = response.json();
const errorCheck = CheckInfo(data);
console.log("2: " + errorCheck);
return errorCheck;
}
Using an async function for a form validation, you can do this:
function onSubmit(form) {
GetPolygonID().then(res => res ? form.submit() : null);
return false;
}
...
<form method="POST" onsubmit="return onSubmit(this);">
...

I have used two ways for returning boolean from the async function as we all know when we use keyword async in functions then it will return a Promise, but what if we want boolean values simple true or false then we can use two ways for that.
By using anonymous function with async and await
const Abc=async ()=>{
return false
}
(async () => {
try{
console.log(await Abc())
}catch(e){
console.error(e)
}
})();
Another way is simply use then() catch()
Abc().then(res=>console.log(res)).catch(e=>console.error(e))

Sent a POST request with JS, and handling the data with Node

I'm adding a contact me section to a website. I want to be able to send the data from the forms with JS, and then receive and do something with the data with Node. I understand that there are frameworks and libraries that can handle this stuff, but I would like to build it from scratch so that I have a better understanding of what is happening.
I currently have a section of JS (see below) that is taking the form data, and sending it as a POST request to the node script, but I can't seem to wrap my head around what is happening with node, or how to receive the data with the node script. Any help in pointing me in the right direction is greatly appreciated.
const name = $(".name");
const email = $(".email");
const message = $(".message");
const submitButton = $(".submitButton");
const nameRegex = /([a-zA-Z\s-])/g;
const emailRegex = /^(([^<>()\[\]\\.,;:\s#"]+(\.[^<>()\[\]\\.,;:\s#"]+)*)|(".+"))#((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/g;
const messageRegex = /([a-zA-Z\s.,?!$%&])/gm;
const url = "../node/contactMeSubmissionHandler.js"
submitButton.click(function(){
let nameContents = name.val().match(nameRegex).join("");
let emailContents = email.val().match(emailRegex).join("");
let messageContents = message.val().match(messageRegex).join("");
// if (emailRegex.test(emailContents) == true) {
// let emailValid = emailContents;
// } else {
// console.log("Email is invalid");
// };
const data = {
email: emailContents,
name: nameContents,
message: messageContents
}
$.post(url, data, function(data, status){
console.log(`${data} and status is ${status}`);
})
})

I like to write from scratch too. Here is working code which is called from a command line to get a token.
// clientEx.js
var http = require('http');
var fs = require('fs');
const _SERVER = "dcsmail.net"; /* dcsmail.net */
// Callback function is used to deal with response
//
var callback = function (response)
{
// update stream with data
var body = '';
response.on('data', function(data) {
body += data;
});
response.on ('end', function()
{
// Data received completely.
fs.writeFileSync ("temp.lst", body, 'utf8');
// console.log ("clientEx.js received: " + body);
});
}
if ((process.argv[2] == null) || (process.argv[3] == null) || (process.argv[4] == null) || (process.argv[5] == null))
{
console.log ("clientEx.js usage:<user email> <user password> <destination> <GUID>");
}
else
{
var Ef_email = encodeURI (process.argv[2]);
var Ef_pass = encodeURI (process.argv[3]);
var Ef_dest = encodeURI (process.argv[4]);
var Ef_guid = encodeURI (process.argv[5]);
var post_data = ("f_email=" + Ef_email +
"\&" + "f_pass=" + Ef_pass +
"\&" + "f_dest=" + Ef_dest +
"\&" + "f_guid=" + Ef_guid);
// Options to be used by request
var options = {
host: _SERVER,
port: '80',
path: '/DCSM/tokenP10.php',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength (post_data)
}
};
// console.log ("clientEx.js using " + _SERVER + ":" + options.port + options.path);
// request the token from the host
try
{
var req = http.request (options, callback);
req.write (post_data);
req.end();
}
catch (error)
{
fs.writeFileSync ("temp.lst", "Host access failed\n", 'utf8');
}
}
You should be able to adapt that to your needs.

Use this code to create a server and check the log in console for different request attributes.
const http = require('http');
http
.createServer((request, response) => {
console.log(request);
response.end();
})
.listen(3000);
Make GET and POST request to http://localhost:3000/ and look for method, headers etc.
See more here and here.