We Keep Coding

Authentication Failed while making REST API call to GET container details in AZURE STORAGE BLOB

I am trying to obtain Container details in Azure Storage Blob. But it throws Auth Failed, I think there might be problems with my resource string formulation.
Here's the code:
const CryptoJS = require("crypto-js");
const request = require("request");
const parser = require("xml2json");
require("dotenv").config();
const account = process.env.ACCOUNT_NAME || "";
const key = process.env.ACCOUNT_KEY || "";
var strTime = new Date().toUTCString();
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo?restype:container`;
var secret = CryptoJS.enc.Base64.parse(key);
var hash = CryptoJS.HmacSHA256(strToSign, secret);
var hashInBase64 = CryptoJS.enc.Base64.stringify(hash);
var auth = `SharedKey ${account}:${hashInBase64}`;
const options = {
url: `https://${account}.blob.core.windows.net/demo?restype=container`,
headers: {
Authorization: auth,
"x-ms-date": strTime,
"x-ms-version": "2018-03-28",
},
};
function callback(error, response, body) {
console.log(error);
console.log(response.headers["Last-Modified"]);
console.log(response);
}
request(options, callback);
In the above example demo is a private container in my account.

Please try by changing the following line of code:
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo?restype:container`;
to
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo\nrestype:container`;

AWS IOT GetThingShadow API request sends response "Signature expired"

const https = require('https');
const crypto = require('crypto');
const utf8 = require('utf8');
const awsIoT = require('aws-iot-device-sdk');
const {AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY} = require('../../config');
const endpointFile = require('../../endpoint.json');
function sign(key, msg){
// since we are deriving key, refer getSignatureKey function
// we use binary format hash => digest() and not digest('hex')
return crypto.createHmac("sha256", key).update(utf8.encode(msg)).digest();
}
function getSignatureKey(key, date, region, service){
// deriving the key as follows:
// HMAC(HMAC(HMAC(HMAC("AWS4" + kSecret,"20150830"),"us-east-1"),"iam"),"aws4_request")
let kDate = sign(utf8.encode(`AWS4${key}`), date);
let kRegion = sign(kDate, region);
let kService = sign(kRegion, service);
let kSigning = sign(kService, 'aws4_request');
return kSigning;
}
function getTime(){
let timeObj = new Date().toISOString();
let arr = timeObj.split(':');
let len = arr[0].length;
let hh = arr[0].slice(len-2,len);
let mm = arr[1];
let ss = arr[2].slice(0,2);
return `${hh}${mm}${ss}`;
}
function getDate(){
let timeObj = new Date().toISOString();
let arr = timeObj.split('-');
let year = arr[0];
let month = arr[1];
let day = arr[2].slice(0,2);
return `${year}${month}${day}`;
}
const ENDPOINT = endpointFile.endpointAddress;
const THING_NAME = __dirname.split('/').pop();
const URI = `https://${ENDPOINT}/things/${THING_NAME}/shadow`;
console.log(URI);
// access key = access key ID + secret access key
// SigV4
// Signing Key derived from credential scope
// Step1
// Create a canonical request (CR)
// CanonicalRequest =
// HTTPRequestMethod + '\n' +
// CanonicalURI + '\n' +
// CanonicalQueryString + '\n' +
// CanonicalHeaders + '\n' +
// SignedHeaders + '\n' +
// HexEncode(Hash(RequestPayload))
const CONTENT_TYPE = "application/json; charset=utf-8";
const HTTP_REQUEST_METHOD = "GET";
// remove the protocol part from URI and query parameters (none in this case)
const CANONICAL_URI = URI.slice(URI.indexOf('/')+2, URI.length);
// console.log(`CANONICAL_URI: ${CANONICAL_URI}`);
const CANONICAL_QUERY_STRING = "";
const HOST = `${ENDPOINT}`;
const DATE = getDate();
const TIME = getTime();
const X_AMZ_DATE = `${DATE}T${TIME}Z`;
console.log(`X_AMZ_DATE: ${X_AMZ_DATE}`);
// note the trailing \n is present
const CANONICAL_HEADER = `content-type:${CONTENT_TYPE}\n`+
`host:${HOST}\n`+
`x-amz-date:${X_AMZ_DATE}\n`;
const SIGNED_HEADER = "content-type;host;x-amz-date";
// payload is the contents of request body
const PAYLOAD = "";
const PAYLOAD_HEX_HASH_ENCODED = crypto.createHash("sha256").update(utf8.encode(PAYLOAD)).digest("hex");
// string for signing CR_STRING = canonical request + metadata
const CANONICAL_REQUEST = `${HTTP_REQUEST_METHOD}\n`+
`${CANONICAL_URI}\n`+
`${CANONICAL_QUERY_STRING}\n`+
`${CANONICAL_HEADER}\n`+
`${SIGNED_HEADER}\n`+
`${PAYLOAD_HEX_HASH_ENCODED}`;
// Step2
// signing key STR_TO_SIGN
const HASH_ALGO = "AWS4-HMAC-SHA256";
const REGION = "us-east-2";
const SERVICE = "iot";
const CREDENTIAL_SCOPE = `${DATE}/`+
`${REGION}/`+
`${SERVICE}/`+
`aws4_request`;
const STRING_TO_SIGN = `${HASH_ALGO}\n`+
`${X_AMZ_DATE}\n`+
`${CREDENTIAL_SCOPE}\n`+
crypto.createHash("sha256")
.update(CANONICAL_REQUEST)
.digest("hex");
// Step3
const SECRET_KEY = AWS_SECRET_ACCESS_KEY;
const SIGNING_KEY = getSignatureKey(SECRET_KEY, DATE, REGION, SERVICE);
const SIGNATURE = crypto.createHmac("sha256", SIGNING_KEY).update(utf8.encode(STRING_TO_SIGN)).digest("hex");
// Step4
// Add SIGNATURE to HTTP request in a header or as a query string parameter
const ACCESS_KEY_ID = AWS_ACCESS_KEY_ID;
const AUTHORIZATION_HEADER = `${HASH_ALGO}`+
` Credential=`+
`${ACCESS_KEY_ID}`+
`/`+
`${CREDENTIAL_SCOPE}`+
`, SignedHeaders=`+
`${SIGNED_HEADER}`+
`, Signature=`+
`${SIGNATURE}`;
const HEADERS = {
'host':HOST,
'content-type':CONTENT_TYPE,
'Authorization':AUTHORIZATION_HEADER,
'x-amz-date':X_AMZ_DATE
};
const OPTIONS = {
hostname: HOST,
path: `/things/${THING_NAME}/shadow`,
headers: HEADERS
};
// send request
https.get(OPTIONS, res=>{
res.setEncoding("utf-8");
let body = "";
res.on("data", data=>{
body += data;
});
res.on("end", ()=>{
body = JSON.parse(body);
console.log(body);
});
});
On running this code the typical response I'm getting is
{ message: 'Signature expired: 20201017T000000Z is now earlier than 20201017T073249Z (20201017T073749Z - 5 min.)', traceId: 'b8f04573-2afd-d26a-5f2a-a13dd2dade3' }
I don't know what is going wrong or what to do to remove this error.
The ISO format is used here with this structure YYYYMMDDTHHMMSSZ
Signature expired: YYYYMMDDT000000Z is now earlier than YYYYMMDDT073249Z (YYYYMMDDT073749Z - 5 min.)
Why is HHMMSS always zero in the reported message?
What I'm trying to do is get "thing" shadow document by sending a request to the API referring to this (AWS_IOT_GetThingShadow API)
However, for authenticating my request I have to do a lot of other stuff which is stated here Signing AWS requests. I have simply performed the 4 tasks / steps mentioned in order to sign the request.
They have provided an example script (sigv4-signed-request-examples) in python which I followed to write my code.
I have been stuck on this for quite a while now. If anyone has any idea about this please help.
EDIT: The above problem was solved by using X_AMZ_DATE in STRING_TO_SIGN and HEADERS. I was wrongly using DATE. I have updated the above code accordingly.
New error I am gettiing is
{ message: 'Credential should be scoped to correct service. ', traceId: 'e711927a-11f4-ae75-c4fe-8cdc5a120c0d' }
I am not sure what is wrong with the credentials. I have set the REGION correctly. I am using SERVICE as iot which should be correct as well for requesting shadow API.
EDIT: It turns out iot is wrong. Changed SERVICE = "iotdata" and now I can successfully request shadow data. Solution found here. It is strange that I couldn't find it anywhere in the AWS docs. Another thing wrong was CANONICAL_URI = path in URI after domain and before query strings
So in my case it will be CANONICAL_URI = /things/${THING_NAME}/shadow

I am posting the correct final version of my code in case anyone is facing similar issue.
Three things were wrong in my original code.
X_AMZ_DATE (YYYYMMDDTHHMMSSZ) didn't use it in HEADERS and STRING_TO_SIGN. Hence, was getting Signature expired error.
SERVICE I thought would be iot but it is iotdata. Credential should be scoped to correct service error was resolved.
CANONICAL_URI should only contain part after the domain and before query parameters. Eg. If request URI is https://foo.bar.baz.com/foo1/foo2/foo3?bar1=baz1&bar2=baz2 then CANONICAL_URI = "/foo1/foo2/foo3"
const https = require('https');
const crypto = require('crypto');
const utf8 = require('utf8');
const {AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY} = require('../../config');
const endpointFile = require('../../endpoint.json');
function sign(key, msg){
// since we are deriving key, refer getSignatureKey function
// we use binary format hash => digest() and not digest('hex')
return crypto.createHmac("sha256", key).update(utf8.encode(msg)).digest();
}
function getSignatureKey(key, date, region, service){
// deriving the key as follows:
// HMAC(HMAC(HMAC(HMAC("AWS4" + kSecret,"20150830"),"us-east-1"),"iam"),"aws4_request")
let kDate = sign(utf8.encode(`AWS4${key}`), date);
let kRegion = sign(kDate, region);
let kService = sign(kRegion, service);
let kSigning = sign(kService, 'aws4_request');
return kSigning;
}
function getTime(){
let timeObj = new Date().toISOString();
let arr = timeObj.split(':');
let len = arr[0].length;
let hh = arr[0].slice(len-2,len);
let mm = arr[1];
let ss = arr[2].slice(0,2);
return `${hh}${mm}${ss}`;
}
function getDate(){
let timeObj = new Date().toISOString();
let arr = timeObj.split('-');
let year = arr[0];
let month = arr[1];
let day = arr[2].slice(0,2);
return `${year}${month}${day}`;
}
const ENDPOINT = endpointFile.endpointAddress;
const THING_NAME = __dirname.split('/').pop();
const URI = `https://${ENDPOINT}/things/${THING_NAME}/shadow`;
console.log(URI);
// access key = access key ID + secret access key
// SigV4
// Signing Key derived from credential scope
// Step1
// Create a canonical request (CR)
// CanonicalRequest =
// HTTPRequestMethod + '\n' +
// CanonicalURI + '\n' +
// CanonicalQueryString + '\n' +
// CanonicalHeaders + '\n' +
// SignedHeaders + '\n' +
// HexEncode(Hash(RequestPayload))
const CONTENT_TYPE = "application/json; charset=utf-8";
const HTTP_REQUEST_METHOD = "GET";
const CANONICAL_URI = `/things/${THING_NAME}/shadow`;
const CANONICAL_QUERY_STRING = "";
const HOST = `${ENDPOINT}`;
const DATE = getDate();
const TIME = getTime();
const X_AMZ_DATE = `${DATE}T${TIME}Z`;
// note the trailing \n is present
const CANONICAL_HEADER = `content-type:${CONTENT_TYPE}\n`+
`host:${HOST}\n`+
`x-amz-date:${X_AMZ_DATE}\n`;
const SIGNED_HEADER = "content-type;host;x-amz-date";
// payload is the contents of request body
const PAYLOAD = "";
const PAYLOAD_HEX_HASH_ENCODED = crypto.createHash("sha256").update(utf8.encode(PAYLOAD)).digest("hex");
// console.log(`Payload: ${PAYLOAD_HEX_HASH_ENCODED}`);
// string for signing CR_STRING = canonical request + metadata
const CANONICAL_REQUEST = `${HTTP_REQUEST_METHOD}\n`+
`${CANONICAL_URI}\n`+
`${CANONICAL_QUERY_STRING}\n`+
`${CANONICAL_HEADER}\n`+
`${SIGNED_HEADER}\n`+
`${PAYLOAD_HEX_HASH_ENCODED}`;
// Step2
// signing key STR_TO_SIGN
const HASH_ALGO = "AWS4-HMAC-SHA256";
const REGION = "us-east-2";
const SERVICE = "iotdata";
const CREDENTIAL_SCOPE = `${DATE}/`+
`${REGION}/`+
`${SERVICE}/`+
`aws4_request`;
const STRING_TO_SIGN = `${HASH_ALGO}\n`+
`${X_AMZ_DATE}\n`+
`${CREDENTIAL_SCOPE}\n`+
crypto.createHash("sha256")
.update(CANONICAL_REQUEST)
.digest("hex");
// Step3
const SECRET_KEY = AWS_SECRET_ACCESS_KEY;
const SIGNING_KEY = getSignatureKey(SECRET_KEY, DATE, REGION, SERVICE);
const SIGNATURE = crypto.createHmac("sha256", SIGNING_KEY).update(utf8.encode(STRING_TO_SIGN)).digest("hex");
// Step4
// Add SIGNATURE to HTTP request in a header or as a query string parameter
const ACCESS_KEY_ID = AWS_ACCESS_KEY_ID;
const AUTHORIZATION_HEADER = `${HASH_ALGO}`+
` Credential=`+
`${ACCESS_KEY_ID}`+
`/`+
`${CREDENTIAL_SCOPE}`+
`, SignedHeaders=`+
`${SIGNED_HEADER}`+
`, Signature=`+
`${SIGNATURE}`;
const HEADERS = {
'host':HOST,
'content-type':CONTENT_TYPE,
'Authorization':AUTHORIZATION_HEADER,
'x-amz-date':X_AMZ_DATE
};
const OPTIONS = {
hostname: HOST,
path: `/things/${THING_NAME}/shadow`,
headers: HEADERS
};
https.get(OPTIONS, res=>{
res.setEncoding("utf-8");
let body = "";
res.on("data", data=>{
body += data;
});
res.on("end", ()=>{
body = JSON.parse(body);
console.log(body);
});
});

Node.js shortened url

I have just started learning to code about 5 days ago and what I'm struggling to achieve, is to have an rssfeed-to-twitter script that posts a shortened url instead of a full website/article feed url. I found a node.js module called TinyURL that could do that but i struggle to get it to work. Here's the full script:
var simpleTwitter = require('simple-twitter');
var fs = require('fs');
var http = require('http');
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type' : 'text/plain'});
res.end('RSS Twitter Bot\n');
}).listen(5693);
var timeInterval = 300000; // run every 30m
var timerVar = setInterval (function () {runBot()}, timeInterval);
function runBot(){
var lastCompleted = Date.parse(new Date(0));
console.log(lastCompleted);
try {
var lastcompletedData = fs.readFileSync('./lastCompleted.json', 'utf8');
var timeData = JSON.parse(lastcompletedData);
var lastCompletedFromFile = Date.parse(new Date(timeData.lastCompleted));
if ( isNaN(lastCompletedFromFile) == false ) {
lastCompleted = lastCompletedFromFile;
}
} catch (e) {
console.log(e);
}
fs.readFile('./config.json', 'utf8', function (err, data) {
if (err) console.log(err); // we'll not consider error handling for now
var configData = JSON.parse(data);
console.log(configData);
var twitter = new simpleTwitter( configData.consumerKey //consumer key from twitter api
, configData.consumerSecret //consumer secret key from twitter api
, configData.accessToken //access token from twitter api
, configData.accessTokenSecret //access token secret from twitter api
, 3600);
var dateNow = Date.parse(new Date());
var FeedParser = require('feedparser');
var request = require('request');
var req = request(configData.feedUrl);
var feedparser = new FeedParser();
req.on('error', function (error) {
console.log(error);
});
req.on('response', function (res){
var stream = this;
if (res.statusCode != 200 ) return this.emit('error', new Error('Bad status code'));
stream.pipe(feedparser);
});
feedparser.on('error', function(error) {
console.log(error);
});
feedparser.on('readable', function() {
var stream = this;
var meta = this.meta;
var item;
while (item = stream.read()) {
var itemDate = Date.parse(item.date);
//check to not publish older articles
if (itemDate > lastCompleted){
var titleLength = item.title.length;
var itemTitle = item.title;
var itemLink = item.link;
if (titleLength > 100) {
itemTitle = itemTitle.substring(0, 100);
}
twitter.post('statuses/update'
, {'status' : itemTitle + ' ' + itemLink + " " + configData.tags}
, function (error, data) {
console.dir(data);
});
console.log(itemTitle + ' ' + item.link + configData.tags);
}
}
//TO KNOW WHEN FROM TO START POSTING
var dateCompleted = new Date();
console.log('loop completed at ' + dateCompleted);
var outputData = {
lastCompleted : dateCompleted
}
var outputFilename = './lastCompleted.json';
fs.writeFile(outputFilename, JSON.stringify(outputData, null, 4), function(err) {
if(err) {
console.log(err);
} else {
console.log("JSON saved to " + outputFilename);
}
});
});
});
}
And this is the TinyURL node.js module
var TinyURL = require('tinyurl');
TinyURL.shorten('http://google.com', function(res) {
console.log(res); //Returns a tinyurl
});
Changing the 'http://google.com' string to itemLink var works just fine and prints it in the terminal as expected.
TinyURL.shorten(itemLink, function(res) {
console.log(res); //Returns a tinyurl
});
What i'm trying to achieve is:
twitter.post('statuses/update', {'status' : itemTitle + ' ' + tinyurlLink + " " + configData.tags}
How can i get the response turned into a e.g var tinyurlLink to replace the itemLink var? Any help would be much appreciated!

As suggested by #zerkms sending a tweet from inside the TinyURL.shorten worked!

Model importing as object, not function in Node.js project with Express

I have a node.js project that uses express. Within this project I have a models folder that contains different models used in the project.
In my MosaicParTileStreamerS3Only model I have the following:
'use strict';
var GlobalMercator = require('./GlobalMercator.js');
var S3 = require('./S3.js');
var binary = require('binary');
var bufferpack = require('bufferpack');
function MosaicParTileStreamerS3Only() {
};
Further into this model, I use S3 without any issues:
MosaicParTileStreamerS3Only.prototype.Init = function(filepath, index, s3config){
var retval = false;
this.s3 = new S3(s3config.access_key, s3config.secret_key, s3config.host);
var host = s3config.host;
var bucket = s3config.bucket;
filepath = s3config.tile_directory + filepath;
var request = this.s3.getObject(bucket, filepath, false, 0,
this.HEADER_SIZE + 5 * this.RESOLUTION_ENTRY_SIZE + this.TILE_COUNT_SIZE
+ 256 * this.TILE_ENTRY_SIZE);
...
Placing a breakpoint on the second line (this.s3 = new S3(s3config.access_key, s3config.secret_key, s3config.host);) shows that S3 is being brought in as a function.
I have the following in my S3Request.js:
'use strict';
var S3 = require('./S3.js');
var STDClass = require('stdclass');
var Curl = require('node-libcurl').Curl;
var parseString = require('xml2js').parseString;
function S3Request(verb, bucket, uri, endpoint){
this.headers = '';
this.endpoint = endpoint;
this.verb = verb;
this.bucket = bucket;
this.uri = (uri !== '' ? '/' + '%2F'.replace('/', encodeURIComponent(uri)) : '/');
...
}
Later on in this model I try using S3:
var test = new S3();
headers['Authorization'] = S3.getSignature(this.verb + '\n' +
this.headers['Content-MD5'] + "\n" + this.headers['Content-Type']
+ "\n" + this.headers["Date"] + amz + "\n" + this.resource);
I get the following error:
TypeError: S3 is not a function
Putting a breakpoint on var test = new S3(); shows that in this file S3 is considered an object:
Any clue as to what's going wrong?

The issue is that there is a cyclic dependency between S3 and S3Request.
See How to deal with cyclic dependencies in Node.js

How do I code a node.js proxy to use NTLMv2 authentication

I've tried to search through stackoverflow for a similar question but most people are asking about the client-side of the NTLMv2 protocol.
I'm implementing a proxy that is performing the server-side of the protocol to authenticate users connecting to the proxy.
I've coded a lot of the protocol but I'm now stuck because the documentation that should take me further is difficult to understand.
This is the best documentation I've found so far: http://www.innovation.ch/personal/ronald/ntlm.html, but how to deal with the LM and NT responses is oblivious to me.
The proxy is located on an application server. The domain server is a different machine.
Example code for the node proxy:
var http = require('http')
, request = require('request')
, ProxyAuth = require('./proxyAuth');
function handlerProxy(req, res) {
ProxyAuth.authorize(req, res);
var options = {
url: req.url,
method: req.method,
headers: req.headers
}
req.pipe(request(options)).pipe(res)
}
var server = http.createServer(handlerProxy);
server.listen(3000, function(){
console.log('Express server listening on port ' + 3000);
});
ProxyAuth.js code:
ProxyAuth = {
parseType3Msg: function(buf) {
var lmlen = buf.readUInt16LE(12);
var lmoff = buf.readUInt16LE(16);
var ntlen = buf.readUInt16LE(20);
var ntoff = buf.readUInt16LE(24);
var dlen = buf.readUInt16LE(28);
var doff = buf.readUInt16LE(32);
var ulen = buf.readUInt16LE(36);
var uoff = buf.readUInt16LE(40);
var hlen = buf.readUInt16LE(44);
var hoff = buf.readUInt16LE(48);
var domain = buf.slice(doff, doff+dlen).toString('utf8');
var user = buf.slice(uoff, uoff+ulen).toString('utf8');
var host = buf.slice(hoff, hoff+hlen).toString('utf8');
var lmresp = buf.slice(lmoff, lmoff+lmlen).toString('utf8');
var ntresp = buf.slice(ntoff, ntoff+ntlen).toString('utf8');
console.log(user, lmresp, ntresp);
/* NOW WHAT DO I DO? */
},
authorize: function(req, res) {
var auth = req.headers['authorization'];
if (!auth) {
res.writeHead(401, {
'WWW-Authenticate': 'NTLM',
});
res.end('<html><body>Proxy Authentication Required</body></html>');
}
else if(auth) {
var header = auth.split(' ');
var buf = new Buffer(header[1], 'base64');
var msg = buf.toString('utf8');
console.log("Decoded", msg);
if (header[0] == "NTLM") {
if (msg.substring(0,8) != "NTLMSSP\x00") {
res.writeHead(401, {
'WWW-Authenticate': 'NTLM',
});
res.end('<html><body>Header not recognized</body></html>');
}
// Type 1 message
if (msg[8] == "\x01") {
console.log(buf.toString('hex'));
var challenge = require('crypto').randomBytes(8);
var type2msg = "NTLMSSP\x00"+
"\x02\x00\x00\x00"+ // 8 message type
"\x00\x00\x00\x00"+ // 12 target name len/alloc
"\x00\x00\x00\x00"+ // 16 target name offset
"\x01\x82\x00\x00"+ // 20 flags
challenge.toString('utf8')+ // 24 challenge
"\x00\x00\x00\x00\x00\x00\x00\x00"+ // 32 context
"\x00\x00\x00\x00\x00\x00\x00\x00"; // 40 target info len/alloc/offset
type2msg = new Buffer(type2msg).toString('base64');
res.writeHead(401, {
'WWW-Authenticate': 'NTLM '+type2msg.trim(),
});
res.end();
}
else if (msg[8] == "\x03") {
console.log(buf.toString('hex'));
ProxyAuth.parseType3Msg(buf);
/* NOW WHAT DO I DO? */
}
}
else if (header[0] == "Basic") {
}
}
}
};
module.exports = ProxyAuth;
The /* NOW WHAT DO I DO? */ comment specifies where I am stuck.
I hope I put enough information there, but let me know if anything else is needed.