We Keep Coding

Node/Express API when getting mutlipel requests data gets mixed up and incorrect results occur

I have a nodejs/express backend api that when I get hundreds of requests within seconds I start to get mixed results where data is crossed between requests and leads to unexpected and incorrected results. Can someone point me to where I am incorrectly defining variables that when I get one of the await functions it causes the other variables to be overwritten with the next requests data and causing my issues.
router.post('/autoscan3', async(req,res,next) => {
let authrule = "autoscan"
console.log(req.body.companyDomain + " " + req.body.user + " for folder: " + req.body.folderid)
let totalscans = 0;
let client = await getDB();
let authorizationResults = await checkAuthorization(client, req.body.companyDomain);
const featureSet = authorizationResults.features;
let company = authorizationResults.company;
const clientid = authorizationResults.client_id;
const clientsecret = authorizationResults.secret;
let tenantid = await getTenantID(req.body.companyDomain, client);
let token = await msgraphservices.getClientCredentialsAccessToken(tenantid, clientid, clientsecret)
let scanResults = []
let folderscans = 0;
try{
for(let a = 0; a < req.body.messages.length; a++){
let senderAddress = req.body.messages[a].senderAddress;
let emailBody = req.body.messages[a].emailBody;
let messageid = req.body.messages[a].messageid;
let receivedDateTime = req.body.messages[a].receivedDateTime;
let user = req.body.messages[a].recieverAddress;
let subject = req.body.messages[a].subject;
let attachments = req.body.messages[a].attachments;
let links = req.body.messages[a].emailBodyLinks;
let headers = req.body.messages[a].headers
let knownaddress
if (senderAddress.includes(req.body.companyDomain)) {
knownaddress = 10
} else {
knownaddress = await searchForSender(client, senderAddress, req.body.user, token, "");}
let assessmentResults = await assessment(
messageid,
emailBody,
senderAddress,
user,
subject,
receivedDateTime,
company,
attachments,
links,
req.body.companyDomain,
client,
headers,
knownaddress,
featureSet,
authrule
)
console.log('adding to folderscans')
folderscans++
try {
await msgraphservices.updateUserCategory2(messageid, req.body.user, req.body.folderid, assessmentResults.riskFactor,token)
}
catch(e) {
console.log(`error on category tag for ${messageid} with user ${req.body.user}`);
console.log(e);
}
}
console.log(`folder scans ${folderscans}`);
totalscans = totalscans + folderscans
return res.status(200).json({status:"success", totalscans:totalscans});
}
catch(e) {
console.log(`error while trying to loop for user ${req.body.user}`)
console.log(e)
logapierror(e.stack, req.body.user, req.body)
return res.status(200).json({status:'error', totalscans:totalscans, error:e});
}
});

It's very likely let client = await getDB();
If multiple requests use the same snapshot of the database but don't know about each other, it's very likely that they're overwriting each other's data.

AMQJS0011E Invalid state not connected. AWS IoT core

I'm trying to have some sort of real-time dashboard that can subscribe to AWS IoT core topics and maybe publish too.
I have found a couple of items online, but I can't figure it out.
This is what I currently have:
function p4(){}
p4.sign = function(key, msg) {
const hash = CryptoJS.HmacSHA256(msg, key);
return hash.toString(CryptoJS.enc.Hex);
};
p4.sha256 = function(msg) {
const hash = CryptoJS.SHA256(msg);
return hash.toString(CryptoJS.enc.Hex);
};
p4.getSignatureKey = function(key, dateStamp, regionName, serviceName) {
const kDate = CryptoJS.HmacSHA256(dateStamp, 'AWS4' + key);
const kRegion = CryptoJS.HmacSHA256(regionName, kDate);
const kService = CryptoJS.HmacSHA256(serviceName, kRegion);
const kSigning = CryptoJS.HmacSHA256('aws4_request', kService);
return kSigning;
};
function getEndpoint() {
const REGION = "eu-west-1";
const IOT_ENDPOINT = "blablablabla-ats.iot.eu-west-1.amazonaws.com";
// your AWS access key ID
const KEY_ID = "My-key";
// your AWS secret access key
const SECRET_KEY = "my-access-token";
// date & time
const dt = (new Date()).toISOString().replace(/[^0-9]/g, "");
const ymd = dt.slice(0,8);
const fdt = `${ymd}T${dt.slice(8,14)}Z`
const scope = `${ymd}/${REGION}/iotdevicegateway/aws4_request`;
const ks = encodeURIComponent(`${KEY_ID}/${scope}`);
let qs = `X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=${ks}&X-Amz-Date=${fdt}&X-Amz-SignedHeaders=ho
const req = `GET\n/mqtt\n${qs}\nhost:${IOT_ENDPOINT}\n\nhost\n${p4.sha256('')}`;
qs += '&X-Amz-Signature=' + p4.sign(
p4.getSignatureKey( SECRET_KEY, ymd, REGION, 'iotdevicegateway'),
`AWS4-HMAC-SHA256\n${fdt}\n${scope}\n${p4.sha256(req)}`
);
return `wss://${IOT_ENDPOINT}/mqtt?${qs}`;
}
// gets MQTT client
function initClient() {
const clientId = Math.random().toString(36).substring(7);
const _client = new Paho.MQTT.Client(getEndpoint(), clientId);
// publish method added to simplify messaging
_client.publish = function(topic, payload) {
let payloadText = JSON.stringify(payload);
let message = new Paho.MQTT.Message(payloadText);
message.destinationName = topic;
message.qos = 0;
_client.send(message);
}
return _client;
}
function getClient(success) {
if (!success) success = ()=> console.log("connected");
const _client = initClient();
const connectOptions = {
useSSL: true,
timeout: 3,
mqttVersion: 4,
onSuccess: success
};
_client.connect(connectOptions);
return _client;
}
let client = {};
function init() {
client = getClient();
client.onMessageArrived = processMessage;
client.onConnectionLost = function(e) {
console.log(e);
}
}
function processMessage(message) {
let info = JSON.parse(message.payloadString);
const publishData = {
retailer: retailData,
order: info.order
};
client.publish("sc/delivery", publishData);
}
$(document).ready(() => {
init();
client.subscribe("sc/orders/");
});
I keep getting the same error; AMQJS0011E Invalid state not connected., but I do see in the requests pane of chrome that there is an connection or so... What am I doing wrong here?
I don't see any of the logs anyhow...

Authentication Failed while making REST API call to GET container details in AZURE STORAGE BLOB

I am trying to obtain Container details in Azure Storage Blob. But it throws Auth Failed, I think there might be problems with my resource string formulation.
Here's the code:
const CryptoJS = require("crypto-js");
const request = require("request");
const parser = require("xml2json");
require("dotenv").config();
const account = process.env.ACCOUNT_NAME || "";
const key = process.env.ACCOUNT_KEY || "";
var strTime = new Date().toUTCString();
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo?restype:container`;
var secret = CryptoJS.enc.Base64.parse(key);
var hash = CryptoJS.HmacSHA256(strToSign, secret);
var hashInBase64 = CryptoJS.enc.Base64.stringify(hash);
var auth = `SharedKey ${account}:${hashInBase64}`;
const options = {
url: `https://${account}.blob.core.windows.net/demo?restype=container`,
headers: {
Authorization: auth,
"x-ms-date": strTime,
"x-ms-version": "2018-03-28",
},
};
function callback(error, response, body) {
console.log(error);
console.log(response.headers["Last-Modified"]);
console.log(response);
}
request(options, callback);
In the above example demo is a private container in my account.

Please try by changing the following line of code:
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo?restype:container`;
to
var strToSign =
"GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" +
strTime +
`\nx-ms-version:2018-03-28\n/${account}/demo\nrestype:container`;

Node.js http.request Express multiple requests with single res.render

I have successfully figured out node.js/Express code for making a single http.request to my server. However, the next step is to make multiple requests which use the same res.render statement at the end.
Here is my successful working code:
module.exports = function (app) {
// MODULES - INCLUDES
var xml2js = require('xml2js');
var parser = new xml2js.Parser();
// FORM - SUBMIT - CUCMMAPPER
app.post('/cucmmapper/submit', function (req, res) {
// FORM - DATA COLLECTION
var cucmpub = req.body.cucmpub;
var cucmversion = req.body.cucmversion;
var username = req.body.username;
var password = req.body.password;
// JS - VARIABLE DEFINITION
var authentication = username + ":" + password;
var soapreplyx = '';
var cssx = '';
var spacer = '-----';
var rmline1 = '';
var rmline2 = '';
var rmline3 = '';
var rmline4 = '';
var rmbottomup1 = '';
var rmbottomup2 = '';
var rmbottomup3 = '';
// HTTP.REQUEST - BUILD CALL
var https = require("https");
var headers = {
'SoapAction': 'CUCM:DB ver=' + cucmversion + ' listCss',
'Authorization': 'Basic ' + new Buffer(authentication).toString('base64'),
'Content-Type': 'text/xml; charset=utf-8'
};
// SOAP - AXL CALL
var soapBody = new Buffer('<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://www.cisco.com/AXL/API/11.5">' +
'<soapenv:Header/>' +
'<soapenv:Body>' +
'<ns:listCss sequence="?">' +
'<searchCriteria>' +
'<name>%</name>' +
'</searchCriteria>' +
'<returnedTags uuid="?">' +
'<name>?</name>' +
'<description>?</description>' +
'<clause>?</clause>' +
'</returnedTags>' +
'</ns:listCss>' +
'</soapenv:Body>' +
'</soapenv:Envelope>');
// HTTP.REQUEST - OPTIONS
var options = {
host: cucmpub, // IP ADDRESS OF CUCM PUBLISHER
port: 8443, // DEFAULT CISCO SSL PORT
path: '/axl/', // AXL URL
method: 'POST', // AXL REQUIREMENT OF POST
headers: headers, // HEADER VAR
rejectUnauthorized: false // REQUIRED TO ACCEPT SELF-SIGNED CERTS
};
// HTTP.REQUEST - Doesn't seem to need this line, but it might be useful anyway for pooling?
options.agent = new https.Agent(options);
// HTTP.REQUEST - OPEN SESSION
let soapRequest = https.request(options, soapResponse => {
soapResponse.setEncoding('utf8');
soapResponse.on('data', chunk => {
soapreplyx += chunk
});
// HTTP.REQUEST - RESULTS + RENDER
soapResponse.on('end', () => {
// EDIT - SCRUB XML OUTPUT
var rmline1 = soapreplyx.replace(/<\?xml\sversion='1\.0'\sencoding='utf-8'\?>/g, '');
var rmline2 = rmline1.replace(/<soapenv:Envelope\sxmlns:soapenv="http:\/\/schemas.xmlsoap.org\/soap\/envelope\/">/g, '');
var rmline3 = rmline2.replace(/<soapenv:Body>/g, '');
var rmline4 = rmline3.replace(/<ns:listCssResponse\sxmlns:ns="http:\/\/www\.cisco\.com\/AXL\/API\/[0-9]*\.[0-9]">/g, '');
var rmbottomup1 = rmline4.replace(/<\/soapenv:Envelope>/g, '');
var rmbottomup2 = rmbottomup1.replace(/<\/soapenv:Body>/g, '');
var xmlscrubbed = rmbottomup2.replace(/<\/ns:listCssResponse>/g, '');
// console.log(xmlscrubbed);
// console.log(spacer);
// XML2JS - TESTING
parser.parseString(xmlscrubbed, function (err, result) {
var cssx = result['return']['css'];
// console.log(cssx);
// console.log(spacer);
res.render('cucmmapper-results.html', {
title: 'CUCM Toolbox',
cucmpub: cucmpub,
cssx: cssx,
soapreply: soapreplyx,
xmlscrubbed: xmlscrubbed
});
});
});
});
// SOAP - SEND AXL CALL
soapRequest.write(soapBody);
soapRequest.end();
});
}
My guess is that I have to setup several things to make this work:
Another "var soapBody" with my new request (I can do this).
Another "let soapRequest" (I'm good with this too).
Another "soapRequest.write" statement (Again, easy enough).
Split the "res.render" statement out of the specific "let soapRequest" statement and gather all the variable (this is where I'm stuck).
My guess is that I need to use async. However, I can't for the life of me wrap my head around how to get that "res.render" to work with async.
Here is the closest I can come to an answer. However, the "cssx" and "partitionsx" variable are not translated over to the "function complete". They both still show up as null.
module.exports = function (app) {
// MODULES - INCLUDES
var xml2js = require('xml2js');
var parser = new xml2js.Parser();
// FORM - SUBMIT - CUCMMAPPER
app.post('/cucmmapper/submit', function (req, res) {
// FORM - DATA COLLECTION
var cucmpub = req.body.cucmpub;
var cucmversion = req.body.cucmversion;
var username = req.body.username;
var password = req.body.password;
// JS - VARIABLE DEFINITION - GLOBAL
var authentication = username + ":" + password;
var soapreplyx = '';
var cssx = null;
var spacer = '-----';
var rmline1 = '';
var rmline2 = '';
var rmline3 = '';
var rmline4 = '';
var rmbottomup1 = '';
var rmbottomup2 = '';
var rmbottomup3 = '';
var soapreplyp = '';
var partitionsx = null;
var rmline1p = '';
var rmline2p = '';
var rmline3p = '';
var rmline4p = '';
var rmbottomup1p = '';
var rmbottomup2p = '';
var rmbottomup3p = '';
// HTTP.REQUEST - BUILD CALL - GLOBAL
var https = require("https");
var headers = {
'SoapAction': 'CUCM:DB ver=' + cucmversion + ' listCss',
'Authorization': 'Basic ' + new Buffer(authentication).toString('base64'),
'Content-Type': 'text/xml; charset=utf-8'
};
// SOAP - AXL CALL - CSS
var soapBody = new Buffer('<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://www.cisco.com/AXL/API/11.5">' +
'<soapenv:Header/>' +
'<soapenv:Body>' +
'<ns:listCss sequence="?">' +
'<searchCriteria>' +
'<name>%</name>' +
'</searchCriteria>' +
'<returnedTags uuid="?">' +
'<name>?</name>' +
'<description>?</description>' +
'<clause>?</clause>' +
'</returnedTags>' +
'</ns:listCss>' +
'</soapenv:Body>' +
'</soapenv:Envelope>');
// SOAP - AXL CALL - PARTITIONS
var soapBody2 = new Buffer('<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://www.cisco.com/AXL/API/11.5">' +
'<soapenv:Header/>' +
'<soapenv:Body>' +
'<ns:listRpite{artotopm} sequence="?">' +
'<searchCriteria>' +
'<name>%</name>' +
'</searchCriteria>' +
'<returnedTags uuid="?">' +
'<name>?</name>' +
'</returnedTags>' +
'</ns:listRoutePartition>' +
'</soapenv:Body>' +
'</soapenv:Envelope>');
// HTTP.REQUEST - OPTIONS - GLOBAL
var options = {
host: cucmpub, // IP ADDRESS OF CUCM PUBLISHER
port: 8443, // DEFAULT CISCO SSL PORT
path: '/axl/', // AXL URL
method: 'POST', // AXL REQUIREMENT OF POST
headers: headers, // HEADER VAR
rejectUnauthorized: false // REQUIRED TO ACCEPT SELF-SIGNED CERTS
};
// HTTP.REQUEST - GLOBAL (Doesn't seem to need this line, but it might be useful anyway for pooling?)
options.agent = new https.Agent(options);
// HTTP.REQUEST - OPEN SESSION - CSS
var soapRequest = https.request(options, soapResponse => {
soapResponse.setEncoding('utf8');
soapResponse.on('data', chunk => {
soapreplyx += chunk
});
// HTTP.REQUEST - RESULTS + RENDER
soapResponse.on('end', () => {
// EDIT - SCRUB XML OUTPUT
var rmline1 = soapreplyx.replace(/<\?xml\sversion='1\.0'\sencoding='utf-8'\?>/g, '');
var rmline2 = rmline1.replace(/<soapenv:Envelope\sxmlns:soapenv="http:\/\/schemas.xmlsoap.org\/soap\/envelope\/">/g, '');
var rmline3 = rmline2.replace(/<soapenv:Body>/g, '');
var rmline4 = rmline3.replace(/<ns:listCssResponse\sxmlns:ns="http:\/\/www\.cisco\.com\/AXL\/API\/[0-9]*\.[0-9]">/g, '');
var rmbottomup1 = rmline4.replace(/<\/soapenv:Envelope>/g, '');
var rmbottomup2 = rmbottomup1.replace(/<\/soapenv:Body>/g, '');
var xmlscrubbed = rmbottomup2.replace(/<\/ns:listCssResponse>/g, '');
// console.log(xmlscrubbed);
// console.log(spacer);
// XML2JS - TESTING
parser.parseString(xmlscrubbed, function (err, result) {
var cssx = result['return']['css'];
// console.log(cssx);
// console.log(spacer);
complete();
});
});
});
// SOAP - SEND AXL CALL - CSS
soapRequest.write(soapBody);
soapRequest.end();
// SOAP - SEND AXL CALL - PARTITIONS
var soapRequest2 = https.request(options, soapResponse2 => {
soapResponse2.setEncoding('utf8');
soapResponse2.on('data', chunk => {
soapreplyp += chunk
});
// HTTP.REQUEST - RESULTS + RENDER
soapResponse2.on('end', () => {
// EDIT - SCRUB XML OUTPUT
var rmline1p = soapreplyy.replace(/<\?xml\sversion='1\.0'\sencoding='utf-8'\?>/g, '');
var rmline2p = rmline1.replace(/<soapenv:Envelope\sxmlns:soapenv="http:\/\/schemas.xmlsoap.org\/soap\/envelope\/">/g, '');
var rmline3p = rmline2.replace(/<soapenv:Body>/g, '');
var rmline4p = rmline3.replace(/<ns:listCssResponse\sxmlns:ns="http:\/\/www\.cisco\.com\/AXL\/API\/[0-9]*\.[0-9]">/g, '');
var rmbottomup1p = rmline4.replace(/<\/soapenv:Envelope>/g, '');
var rmbottomup2p = rmbottomup1.replace(/<\/soapenv:Body>/g, '');
var xmlscrubbedp = rmbottomup2.replace(/<\/ns:listCssResponse>/g, '');
console.log(xmlscrubbedp);
console.log(spacer);
// XML2JS - TESTING
parser.parseString(xmlscrubbedp, function (err, result) {
var partitionsx = result['return']['css'];
// console.log(partitionsx);
// console.log(spacer);
complete();
});
});
});
// SOAP - SEND AXL CALL - PARTITIONS
soapRequest2.write(soapBody2);
soapRequest2.end();
// PAGE - RENDER
function complete() {
if (cssx !== null && partitionsx !== null) {
res.render('cucmmapper-results.html', {
title: 'CUCM Toolbox',
cucmpub: cucmpub,
cssx: cssx,
partitionsx: partitionsx,
})
} else {
res.render('cucmerror.html', {
title: 'CUCM Toolbox',
})
}
};
});
}
Any help or suggestions would be greatly appreciated.

OK, so the thing to remember is that there is always one request mapped to one response in HTTP. So you can't send multiple requests and expect to get just one response from that.
Instead, you need to have the server keep track of what's been posted (perhaps in a database on a production app), and respond to each request in turn. One way might be to respond with partial documents, or respond with other codes that indicate the submission was accepted but that you need to send another request to push more info, that sort of thing.
But again, you can't strictly accept multiple requests without responding and then respond only after all requests are given.

Kraken.com API: Generate the message signature in Google App Script

I would like to retrieve data from the Kraken.com API. I am trying to call the "Private" methods. (Those one need to be authenticated)
As precised here: https://www.kraken.com/help/api
The expected signature is:
API-Sign = Message signature using HMAC-SHA512 of (URI path + SHA256(nonce + POST data)) and base64 decoded secret API key
I've found here a function from their node.js library that should do the job, but I can't seem to get it.
/**
* This method returns a signature for a request as a Base64-encoded string
* #param {String} path The relative URL path for the request
* #param {Object} request The POST body
* #param {Integer} nonce A unique, incrementing integer
* #return {String} The request signature
*/
function getMessageSignature(path, request, nonce) {
var message = querystring.stringify(request);
var secret = new Buffer(config.secret, 'base64');
var hash = new crypto.createHash('sha256');
var hmac = new crypto.createHmac('sha512', secret);
var hash_digest = hash.update(nonce + message).digest('binary');
var hmac_digest = hmac.update(path + hash_digest, 'binary').digest('base64');
return hmac_digest;
}
Here is my full code
function main() {
var apiKey = "API-KEY";
var apiSecret = "API-SECRET";
var url = "https://api.kraken.com/0/private/Balance";
var path = "/0/private/Balance";
const nonce = new Date() * 1000;
const payload = {
'nonce': nonce
};
const postData = 'nonce=' + nonce;
const signature = getMessageSignature(path, apiSecret, postData, nonce);
var httpOptions = {
'method': 'post',
'headers': {
"API-Key": apiKey,
"API-Sign": signature
},
'payload': postData
};
var response = UrlFetchApp.fetch(url, httpOptions);
Logger.log(response.getContentText());
}
function getMessageSignature(url, secret, data, nonce) {
const hash = Utilities.computeDigest(Utilities.DigestAlgorithm.SHA_256, nonce + data);
const hmac_digest = Utilities.computeHmacSignature(Utilities.MacAlgorithm.HMAC_SHA_512, url + hash, Utilities.base64Decode(secret));
return Utilities.base64Encode(hmac_digest);
}
But i end up getting error
{"error":["EAPI:Invalid key"]}
Thanks in anticipation.

I finally got the code to work using jsSHA
function getKrakenSignature (path, postdata, nonce) {
var sha256obj = new jsSHA ("SHA-256", "BYTES");
sha256obj.update (nonce + postdata);
var hash_digest = sha256obj.getHash ("BYTES");
var sha512obj = new jsSHA ("SHA-512", "BYTES");
sha512obj.setHMACKey ("HMACKEY", "B64");
sha512obj.update (path);
sha512obj.update (hash_digest);
return sha512obj.getHMAC ("B64");
}
function getKrakenBalance () {
var path = "/0/private/Balance";
var nonce = new Date () * 1000;
var postdata = "nonce=" + nonce;
var signature = getKrakenSignature (path, postdata, nonce);
var url = "https://api.kraken.com" + path;
var options = {
method: 'post',
headers: {
'API-Key': "<API-KEY>",
'API-Sign': signature
},
payload: postdata
};
var response = UrlFetchApp.fetch (url, options);
return response.getContentText ();
}
function main() {
Logger.log(getKrakenBalance());
}