We Keep Coding

how to handle string comparison and file writing using xlsx with nodejs

This script is to get the title of the webpage where the URL of the website will be passed from an excel file, check to see if the title contains the keyword, and then store that domain in the new excel file.
There is no issue with the partial code, but the title comparison does not work as expected. Does anyone have an idea how to fix it?
here is my code
var request = require("request");
var cheerio = require("cheerio");
const xlsx = require("xlsx");
jsonData = [{ Domain: "blogger.com" }, { Domain: "stackoverflow.com" }];
function fetchTitle(url, onComplete = null) {
request(url, function (error, response, body) {
var output = url; // default to URL
if (!error && (response && response.statusCode) === 200) {
var $ = cheerio.load(body);
console.log(`URL = ${url}`);
var title = $("head > title").text().trim();
console.log(`Title = ${title}`);
output = `[${title}] (${url})`;
var keywords = ["Developers", "blog"];
var results = [];
var UrlArray = [];
for (var i = 0; i < keywords.length; i++) {
var match = title.match(new RegExp(keywords.join("|"), "g"));
results.push(keywords[i]);
}
if (match.length > 0) {
UrlArray.push({
Domain: url,
Keywords: results,
Title: output,
});
finalJsonData = JSON.stringify(UrlArray);
const ws = xlsx.utils.json_to_sheet(UrlArray);
const wb = xlsx.utils.book_new();
xlsx.utils.book_append_sheet(wb, ws, "Responses");
xlsx.writeFile(wb, "output.xlsx");
}
} else {
console.log(
`Error = ${error}, code = ${response && response.statusCode}`
);
}
console.log(`output = ${output} \n\n`);
if (onComplete) onComplete(output);
});
}
jsonData.forEach(function (table) {
var tableName = table.Domain;
var URL = "http://" + tableName;
fetchTitle(URL);
});
When I execute the script, I am able to get the title, but when I compare it with the keyword, it is not working as expected. Keywords are not being stored. You can see how the output looks after executing the script.
The script shows that both domains have keywords, but only blogger is stored in the spreadsheet, even then keywords aren't stored

you're overwriting the file on each loop,
keywords is an array, so it doesn't get saved, furthermore, keywords column will always contain all keywords, not the matching ones...
as requests are async, you need to track them all, and write results only when all requests are finished.
try this:
match case insensitive, and store only matching keywords for that site, not all (I also added "no match" for domains with no match)
store results outside the loop
move writing results into a separate function
add request counter and callback to track requests
write results when requests are done
the code:
var request = require("request");
var cheerio = require("cheerio");
const xlsx = require("xlsx");
const jsonData = [{ Domain: "blogger.com" }, { Domain: "stackoverflow.com" }];
var UrlArray = [];
function writeResults() {
const finalJsonData = JSON.stringify(UrlArray);
const ws = xlsx.utils.json_to_sheet(UrlArray);
const wb = xlsx.utils.book_new();
xlsx.utils.book_append_sheet(wb, ws, "Responses");
xlsx.writeFile(wb, "output.xlsx");
}
function fetchTitle(url, onComplete = null) {
request(url, function (error, response, body) {
var output = url; // default to URL
if (!error && (response && response.statusCode) === 200) {
var $ = cheerio.load(body);
console.log(`URL = ${url}`);
var title = $("head > title").text().trim();
console.log(`Title = ${title}`);
output = `[${title}] (${url})`;
var keywords = ["Developers", "blog"];
var results = [];
for (var i = 0; i < keywords.length; i++) {
let match = title.match(new RegExp(keywords[i], "gi"));
if (match && match.length > 0) {
results.push(keywords[i]);
}
}
UrlArray.push({
Domain: url,
Keywords: results.length > 0 ? results.join(', ') : 'no match',
Title: output,
});
} else {
console.log(
`Error = ${error}, code = ${response && response.statusCode}`
);
}
console.log(`output = ${output} \n\n`);
if (onComplete) onComplete(output);
});
}
let counter = 0;
jsonData.forEach(function (table) {
var tableName = table.Domain;
var URL = "http://" + tableName;
fetchTitle(URL, ()=>{
counter++;
if(counter === jsonData.length) {
console.log(`all ${counter} requests done`);
writeResults();
}
});
});

AMQJS0011E Invalid state not connected. AWS IoT core

I'm trying to have some sort of real-time dashboard that can subscribe to AWS IoT core topics and maybe publish too.
I have found a couple of items online, but I can't figure it out.
This is what I currently have:
function p4(){}
p4.sign = function(key, msg) {
const hash = CryptoJS.HmacSHA256(msg, key);
return hash.toString(CryptoJS.enc.Hex);
};
p4.sha256 = function(msg) {
const hash = CryptoJS.SHA256(msg);
return hash.toString(CryptoJS.enc.Hex);
};
p4.getSignatureKey = function(key, dateStamp, regionName, serviceName) {
const kDate = CryptoJS.HmacSHA256(dateStamp, 'AWS4' + key);
const kRegion = CryptoJS.HmacSHA256(regionName, kDate);
const kService = CryptoJS.HmacSHA256(serviceName, kRegion);
const kSigning = CryptoJS.HmacSHA256('aws4_request', kService);
return kSigning;
};
function getEndpoint() {
const REGION = "eu-west-1";
const IOT_ENDPOINT = "blablablabla-ats.iot.eu-west-1.amazonaws.com";
// your AWS access key ID
const KEY_ID = "My-key";
// your AWS secret access key
const SECRET_KEY = "my-access-token";
// date & time
const dt = (new Date()).toISOString().replace(/[^0-9]/g, "");
const ymd = dt.slice(0,8);
const fdt = `${ymd}T${dt.slice(8,14)}Z`
const scope = `${ymd}/${REGION}/iotdevicegateway/aws4_request`;
const ks = encodeURIComponent(`${KEY_ID}/${scope}`);
let qs = `X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=${ks}&X-Amz-Date=${fdt}&X-Amz-SignedHeaders=ho
const req = `GET\n/mqtt\n${qs}\nhost:${IOT_ENDPOINT}\n\nhost\n${p4.sha256('')}`;
qs += '&X-Amz-Signature=' + p4.sign(
p4.getSignatureKey( SECRET_KEY, ymd, REGION, 'iotdevicegateway'),
`AWS4-HMAC-SHA256\n${fdt}\n${scope}\n${p4.sha256(req)}`
);
return `wss://${IOT_ENDPOINT}/mqtt?${qs}`;
}
// gets MQTT client
function initClient() {
const clientId = Math.random().toString(36).substring(7);
const _client = new Paho.MQTT.Client(getEndpoint(), clientId);
// publish method added to simplify messaging
_client.publish = function(topic, payload) {
let payloadText = JSON.stringify(payload);
let message = new Paho.MQTT.Message(payloadText);
message.destinationName = topic;
message.qos = 0;
_client.send(message);
}
return _client;
}
function getClient(success) {
if (!success) success = ()=> console.log("connected");
const _client = initClient();
const connectOptions = {
useSSL: true,
timeout: 3,
mqttVersion: 4,
onSuccess: success
};
_client.connect(connectOptions);
return _client;
}
let client = {};
function init() {
client = getClient();
client.onMessageArrived = processMessage;
client.onConnectionLost = function(e) {
console.log(e);
}
}
function processMessage(message) {
let info = JSON.parse(message.payloadString);
const publishData = {
retailer: retailData,
order: info.order
};
client.publish("sc/delivery", publishData);
}
$(document).ready(() => {
init();
client.subscribe("sc/orders/");
});
I keep getting the same error; AMQJS0011E Invalid state not connected., but I do see in the requests pane of chrome that there is an connection or so... What am I doing wrong here?
I don't see any of the logs anyhow...

Authentication Failed while making REST API call to GET container details in AZURE STORAGE BLOB

I am trying to obtain Container details in Azure Storage Blob. But it throws Auth Failed, I think there might be problems with my resource string formulation.
Here's the code:
const CryptoJS = require("crypto-js");
const request = require("request");
const parser = require("xml2json");
require("dotenv").config();
const account = process.env.ACCOUNT_NAME || "";
const key = process.env.ACCOUNT_KEY || "";
var strTime = new Date().toUTCString();
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo?restype:container`;
var secret = CryptoJS.enc.Base64.parse(key);
var hash = CryptoJS.HmacSHA256(strToSign, secret);
var hashInBase64 = CryptoJS.enc.Base64.stringify(hash);
var auth = `SharedKey ${account}:${hashInBase64}`;
const options = {
url: `https://${account}.blob.core.windows.net/demo?restype=container`,
headers: {
Authorization: auth,
"x-ms-date": strTime,
"x-ms-version": "2018-03-28",
},
};
function callback(error, response, body) {
console.log(error);
console.log(response.headers["Last-Modified"]);
console.log(response);
}
request(options, callback);
In the above example demo is a private container in my account.

Please try by changing the following line of code:
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo?restype:container`;
to
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo\nrestype:container`;

AWS IOT GetThingShadow API request sends response "Signature expired"

const https = require('https');
const crypto = require('crypto');
const utf8 = require('utf8');
const awsIoT = require('aws-iot-device-sdk');
const {AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY} = require('../../config');
const endpointFile = require('../../endpoint.json');
function sign(key, msg){
// since we are deriving key, refer getSignatureKey function
// we use binary format hash => digest() and not digest('hex')
return crypto.createHmac("sha256", key).update(utf8.encode(msg)).digest();
}
function getSignatureKey(key, date, region, service){
// deriving the key as follows:
// HMAC(HMAC(HMAC(HMAC("AWS4" + kSecret,"20150830"),"us-east-1"),"iam"),"aws4_request")
let kDate = sign(utf8.encode(`AWS4${key}`), date);
let kRegion = sign(kDate, region);
let kService = sign(kRegion, service);
let kSigning = sign(kService, 'aws4_request');
return kSigning;
}
function getTime(){
let timeObj = new Date().toISOString();
let arr = timeObj.split(':');
let len = arr[0].length;
let hh = arr[0].slice(len-2,len);
let mm = arr[1];
let ss = arr[2].slice(0,2);
return `${hh}${mm}${ss}`;
}
function getDate(){
let timeObj = new Date().toISOString();
let arr = timeObj.split('-');
let year = arr[0];
let month = arr[1];
let day = arr[2].slice(0,2);
return `${year}${month}${day}`;
}
const ENDPOINT = endpointFile.endpointAddress;
const THING_NAME = __dirname.split('/').pop();
const URI = `https://${ENDPOINT}/things/${THING_NAME}/shadow`;
console.log(URI);
// access key = access key ID + secret access key
// SigV4
// Signing Key derived from credential scope
// Step1
// Create a canonical request (CR)
// CanonicalRequest =
// HTTPRequestMethod + '\n' +
// CanonicalURI + '\n' +
// CanonicalQueryString + '\n' +
// CanonicalHeaders + '\n' +
// SignedHeaders + '\n' +
// HexEncode(Hash(RequestPayload))
const CONTENT_TYPE = "application/json; charset=utf-8";
const HTTP_REQUEST_METHOD = "GET";
// remove the protocol part from URI and query parameters (none in this case)
const CANONICAL_URI = URI.slice(URI.indexOf('/')+2, URI.length);
// console.log(`CANONICAL_URI: ${CANONICAL_URI}`);
const CANONICAL_QUERY_STRING = "";
const HOST = `${ENDPOINT}`;
const DATE = getDate();
const TIME = getTime();
const X_AMZ_DATE = `${DATE}T${TIME}Z`;
console.log(`X_AMZ_DATE: ${X_AMZ_DATE}`);
// note the trailing \n is present
const CANONICAL_HEADER = `content-type:${CONTENT_TYPE}\n`+
`host:${HOST}\n`+
`x-amz-date:${X_AMZ_DATE}\n`;
const SIGNED_HEADER = "content-type;host;x-amz-date";
// payload is the contents of request body
const PAYLOAD = "";
const PAYLOAD_HEX_HASH_ENCODED = crypto.createHash("sha256").update(utf8.encode(PAYLOAD)).digest("hex");
// string for signing CR_STRING = canonical request + metadata
const CANONICAL_REQUEST = `${HTTP_REQUEST_METHOD}\n`+
`${CANONICAL_URI}\n`+
`${CANONICAL_QUERY_STRING}\n`+
`${CANONICAL_HEADER}\n`+
`${SIGNED_HEADER}\n`+
`${PAYLOAD_HEX_HASH_ENCODED}`;
// Step2
// signing key STR_TO_SIGN
const HASH_ALGO = "AWS4-HMAC-SHA256";
const REGION = "us-east-2";
const SERVICE = "iot";
const CREDENTIAL_SCOPE = `${DATE}/`+
`${REGION}/`+
`${SERVICE}/`+
`aws4_request`;
const STRING_TO_SIGN = `${HASH_ALGO}\n`+
`${X_AMZ_DATE}\n`+
`${CREDENTIAL_SCOPE}\n`+
crypto.createHash("sha256")
.update(CANONICAL_REQUEST)
.digest("hex");
// Step3
const SECRET_KEY = AWS_SECRET_ACCESS_KEY;
const SIGNING_KEY = getSignatureKey(SECRET_KEY, DATE, REGION, SERVICE);
const SIGNATURE = crypto.createHmac("sha256", SIGNING_KEY).update(utf8.encode(STRING_TO_SIGN)).digest("hex");
// Step4
// Add SIGNATURE to HTTP request in a header or as a query string parameter
const ACCESS_KEY_ID = AWS_ACCESS_KEY_ID;
const AUTHORIZATION_HEADER = `${HASH_ALGO}`+
` Credential=`+
`${ACCESS_KEY_ID}`+
`/`+
`${CREDENTIAL_SCOPE}`+
`, SignedHeaders=`+
`${SIGNED_HEADER}`+
`, Signature=`+
`${SIGNATURE}`;
const HEADERS = {
'host':HOST,
'content-type':CONTENT_TYPE,
'Authorization':AUTHORIZATION_HEADER,
'x-amz-date':X_AMZ_DATE
};
const OPTIONS = {
hostname: HOST,
path: `/things/${THING_NAME}/shadow`,
headers: HEADERS
};
// send request
https.get(OPTIONS, res=>{
res.setEncoding("utf-8");
let body = "";
res.on("data", data=>{
body += data;
});
res.on("end", ()=>{
body = JSON.parse(body);
console.log(body);
});
});
On running this code the typical response I'm getting is
{ message: 'Signature expired: 20201017T000000Z is now earlier than 20201017T073249Z (20201017T073749Z - 5 min.)', traceId: 'b8f04573-2afd-d26a-5f2a-a13dd2dade3' }
I don't know what is going wrong or what to do to remove this error.
The ISO format is used here with this structure YYYYMMDDTHHMMSSZ
Signature expired: YYYYMMDDT000000Z is now earlier than YYYYMMDDT073249Z (YYYYMMDDT073749Z - 5 min.)
Why is HHMMSS always zero in the reported message?
What I'm trying to do is get "thing" shadow document by sending a request to the API referring to this (AWS_IOT_GetThingShadow API)
However, for authenticating my request I have to do a lot of other stuff which is stated here Signing AWS requests. I have simply performed the 4 tasks / steps mentioned in order to sign the request.
They have provided an example script (sigv4-signed-request-examples) in python which I followed to write my code.
I have been stuck on this for quite a while now. If anyone has any idea about this please help.
EDIT: The above problem was solved by using X_AMZ_DATE in STRING_TO_SIGN and HEADERS. I was wrongly using DATE. I have updated the above code accordingly.
New error I am gettiing is
{ message: 'Credential should be scoped to correct service. ', traceId: 'e711927a-11f4-ae75-c4fe-8cdc5a120c0d' }
I am not sure what is wrong with the credentials. I have set the REGION correctly. I am using SERVICE as iot which should be correct as well for requesting shadow API.
EDIT: It turns out iot is wrong. Changed SERVICE = "iotdata" and now I can successfully request shadow data. Solution found here. It is strange that I couldn't find it anywhere in the AWS docs. Another thing wrong was CANONICAL_URI = path in URI after domain and before query strings
So in my case it will be CANONICAL_URI = /things/${THING_NAME}/shadow

I am posting the correct final version of my code in case anyone is facing similar issue.
Three things were wrong in my original code.
X_AMZ_DATE (YYYYMMDDTHHMMSSZ) didn't use it in HEADERS and STRING_TO_SIGN. Hence, was getting Signature expired error.
SERVICE I thought would be iot but it is iotdata. Credential should be scoped to correct service error was resolved.
CANONICAL_URI should only contain part after the domain and before query parameters. Eg. If request URI is https://foo.bar.baz.com/foo1/foo2/foo3?bar1=baz1&bar2=baz2 then CANONICAL_URI = "/foo1/foo2/foo3"
const https = require('https');
const crypto = require('crypto');
const utf8 = require('utf8');
const {AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY} = require('../../config');
const endpointFile = require('../../endpoint.json');
function sign(key, msg){
// since we are deriving key, refer getSignatureKey function
// we use binary format hash => digest() and not digest('hex')
return crypto.createHmac("sha256", key).update(utf8.encode(msg)).digest();
}
function getSignatureKey(key, date, region, service){
// deriving the key as follows:
// HMAC(HMAC(HMAC(HMAC("AWS4" + kSecret,"20150830"),"us-east-1"),"iam"),"aws4_request")
let kDate = sign(utf8.encode(`AWS4${key}`), date);
let kRegion = sign(kDate, region);
let kService = sign(kRegion, service);
let kSigning = sign(kService, 'aws4_request');
return kSigning;
}
function getTime(){
let timeObj = new Date().toISOString();
let arr = timeObj.split(':');
let len = arr[0].length;
let hh = arr[0].slice(len-2,len);
let mm = arr[1];
let ss = arr[2].slice(0,2);
return `${hh}${mm}${ss}`;
}
function getDate(){
let timeObj = new Date().toISOString();
let arr = timeObj.split('-');
let year = arr[0];
let month = arr[1];
let day = arr[2].slice(0,2);
return `${year}${month}${day}`;
}
const ENDPOINT = endpointFile.endpointAddress;
const THING_NAME = __dirname.split('/').pop();
const URI = `https://${ENDPOINT}/things/${THING_NAME}/shadow`;
console.log(URI);
// access key = access key ID + secret access key
// SigV4
// Signing Key derived from credential scope
// Step1
// Create a canonical request (CR)
// CanonicalRequest =
// HTTPRequestMethod + '\n' +
// CanonicalURI + '\n' +
// CanonicalQueryString + '\n' +
// CanonicalHeaders + '\n' +
// SignedHeaders + '\n' +
// HexEncode(Hash(RequestPayload))
const CONTENT_TYPE = "application/json; charset=utf-8";
const HTTP_REQUEST_METHOD = "GET";
const CANONICAL_URI = `/things/${THING_NAME}/shadow`;
const CANONICAL_QUERY_STRING = "";
const HOST = `${ENDPOINT}`;
const DATE = getDate();
const TIME = getTime();
const X_AMZ_DATE = `${DATE}T${TIME}Z`;
// note the trailing \n is present
const CANONICAL_HEADER = `content-type:${CONTENT_TYPE}\n`+
`host:${HOST}\n`+
`x-amz-date:${X_AMZ_DATE}\n`;
const SIGNED_HEADER = "content-type;host;x-amz-date";
// payload is the contents of request body
const PAYLOAD = "";
const PAYLOAD_HEX_HASH_ENCODED = crypto.createHash("sha256").update(utf8.encode(PAYLOAD)).digest("hex");
// console.log(`Payload: ${PAYLOAD_HEX_HASH_ENCODED}`);
// string for signing CR_STRING = canonical request + metadata
const CANONICAL_REQUEST = `${HTTP_REQUEST_METHOD}\n`+
`${CANONICAL_URI}\n`+
`${CANONICAL_QUERY_STRING}\n`+
`${CANONICAL_HEADER}\n`+
`${SIGNED_HEADER}\n`+
`${PAYLOAD_HEX_HASH_ENCODED}`;
// Step2
// signing key STR_TO_SIGN
const HASH_ALGO = "AWS4-HMAC-SHA256";
const REGION = "us-east-2";
const SERVICE = "iotdata";
const CREDENTIAL_SCOPE = `${DATE}/`+
`${REGION}/`+
`${SERVICE}/`+
`aws4_request`;
const STRING_TO_SIGN = `${HASH_ALGO}\n`+
`${X_AMZ_DATE}\n`+
`${CREDENTIAL_SCOPE}\n`+
crypto.createHash("sha256")
.update(CANONICAL_REQUEST)
.digest("hex");
// Step3
const SECRET_KEY = AWS_SECRET_ACCESS_KEY;
const SIGNING_KEY = getSignatureKey(SECRET_KEY, DATE, REGION, SERVICE);
const SIGNATURE = crypto.createHmac("sha256", SIGNING_KEY).update(utf8.encode(STRING_TO_SIGN)).digest("hex");
// Step4
// Add SIGNATURE to HTTP request in a header or as a query string parameter
const ACCESS_KEY_ID = AWS_ACCESS_KEY_ID;
const AUTHORIZATION_HEADER = `${HASH_ALGO}`+
` Credential=`+
`${ACCESS_KEY_ID}`+
`/`+
`${CREDENTIAL_SCOPE}`+
`, SignedHeaders=`+
`${SIGNED_HEADER}`+
`, Signature=`+
`${SIGNATURE}`;
const HEADERS = {
'host':HOST,
'content-type':CONTENT_TYPE,
'Authorization':AUTHORIZATION_HEADER,
'x-amz-date':X_AMZ_DATE
};
const OPTIONS = {
hostname: HOST,
path: `/things/${THING_NAME}/shadow`,
headers: HEADERS
};
https.get(OPTIONS, res=>{
res.setEncoding("utf-8");
let body = "";
res.on("data", data=>{
body += data;
});
res.on("end", ()=>{
body = JSON.parse(body);
console.log(body);
});
});

Using Node.js to retrieve the results count from google

To give the specific details:
I am designing a Discord Bot that uses the eris library for private use. One requested feature is a googlefight command which needs to do the following:
Take user input.
Use input to generate a Google search URL.
Pull the resulting HTML page and extract from it the text from within <div class="sd" id="resultStats">.
Output this result as a message.
The currently existing related code is:
const Eris = require("eris");
const express = require('express');
const request = require('request');
const cheerio = require('cheerio');
const app = express();
...
bot.registerCommand("googlefight", (msg, args) => {
if(args.length === 0) {
return "Invalid input";
}
var arrayLength = args.length;
var searchURL = "https://www.google.com/search?q=";
for (var i = 0; i < arrayLength; i++) {
searchURL = searchURL.concat(args[i]);
if (i + 1 < arrayLength) {
searchURL = searchURL.concat("%20");
} else {
}
}
var text;
app.get('/', function(req, res){
request(searchURL, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
$(.sd).filter(function(){
var data = $(this);
text = data.children().first().text();
})
}
})
})
return text;
}, {
description: "Result counter",
fullDescription: "The bot will return the result count of a google search term.",
usage: "<text>"
});
I am unfamiliar with this type of work.
The part specifically that is broken is this section:
...
app.get('/', function(req, res){
request(searchURL, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
$(.sd).filter(function(){
var data = $(this);
text = data.children().first().text();
})
}
})
})
...