We Keep Coding

Issues with JSON.parse on input object form S3 - Node.js

Node newbie here I have a problem that I have been banging my head against for a few days now and I'm sure I am missing something small but my google too has failed me so far.
I have an AWS Firehose that is writing CloudWatch logs to S3 that are then picked up by a lambda function parsed and ingested into Elasticsearch. Or at least that is the plan. I reason I am using Firehose is that the data is coming from remote AWS accounts and sending all the logs to one place s3. I am able to pull the logs once in S3. Cloud watch logs by default are gzipped so the firehose is simply writing the files to S3 no transforming or anything. Now the issue :). Once I read the file from S3 I attempt to send it to a function transform to parse the data but it fails when trying to loop through the log which is JSON. I did a console.log(payload) and it appears to be in JSON format but running it through JSON.stringify the back to parsing does nothing. JSON.parse will work but the JSON will not be valid.
I am attaching the code below. I am not the original author someone in my company found it on GitHub. I just added the S3 parts to it.
I added a file that was one of the logs as an example it's in a public s3 bucket if anyone wants to grab it. No worries its all test data nothing special.
https://s3.amazonaws.com/node-issue-stackoverflow/cwl-test-11-2018-08-26-00-45-34-84a4c3de-179a-4bf2-9376-895cdc063e6b+(1)
Pastebin Link
// v1.1.2
var https = require('https');
var zlib = require('zlib');
var crypto = require('crypto');
var AWS = require('aws-sdk');
var endpoint = process.env.es_endpoint;
var s3 = new AWS.S3();
var params;
exports.handler = function(input, context) {
// Get the event from S3 based on input
params = {Bucket: input.Records[0].s3.bucket.name, Key: input.Records[0].s3.object.key};
console.log(input.Records[0].s3);
s3.getObject(params, function(error, event){
if (error) { context.fail(error); return; }
console.log(event);
console.log(event.Body);
// decode input from base64
var zippedInput = new Buffer(event.Body, 'base64');
// decompress the input
zlib.gunzip(zippedInput, function(error, buffer) {
if (error) { context.fail(error); return; }
// console.log(buffer.toString());
// parse the input from JSON
var awslogsData = buffer.toString('utf8');
// transform the input to Elasticsearch documents
var elasticsearchBulkData = transform(awslogsData);
// skip control messages
if (!elasticsearchBulkData) {
console.log('Received a control message');
context.succeed('Control message handled successfully');
return;
}
// post documents to the Amazon Elasticsearch Service
post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
console.log('Response: ' + JSON.stringify({
"statusCode": statusCode
}));
if (error) {
console.log('Error: ' + JSON.stringify(error, null, 2));
if (failedItems && failedItems.length > 0) {
console.log("Failed Items: " +
JSON.stringify(failedItems, null, 2));
}
context.fail(JSON.stringify(error));
} else {
console.log('Success: ' + JSON.stringify(success));
context.succeed('Success');
}
});
});
});
};
function transform(payload) {
if (payload.messageType === 'CONTROL_MESSAGE') {
return null;
}
var bulkRequestBody = '';
payload.logEvents.forEach(function(logEvent) {
var timestamp = new Date(1 * logEvent.timestamp);
// index name format: cwl-YYYY.MM.DD
var indexName = [
'cwl-' + timestamp.getUTCFullYear(), // year
('0' + (timestamp.getUTCMonth() + 1)).slice(-2), // month
('0' + timestamp.getUTCDate()).slice(-2) // day
].join('.');
var source = buildSource(logEvent.message, logEvent.extractedFields);
source['#id'] = logEvent.id;
source['#timestamp'] = new Date(1 * logEvent.timestamp).toISOString();
source['#message'] = logEvent.message;
source['#owner'] = payload.owner;
source['#log_group'] = payload.logGroup;
source['#log_stream'] = payload.logStream;
var action = { "index": {} };
action.index._index = indexName;
action.index._type = payload.logGroup;
action.index._id = logEvent.id;
bulkRequestBody += [
JSON.stringify(action),
JSON.stringify(source),
].join('\n') + '\n';
});
return bulkRequestBody;
}
function buildSource(message, extractedFields) {
if (extractedFields) {
var source = {};
for (var key in extractedFields) {
if (extractedFields.hasOwnProperty(key) && extractedFields[key]) {
var value = extractedFields[key];
if (isNumeric(value)) {
source[key] = 1 * value;
continue;
}
jsonSubString = extractJson(value);
if (jsonSubString !== null) {
source['$' + key] = JSON.parse(jsonSubString);
}
source[key] = value;
}
}
return source;
}
jsonSubString = extractJson(message);
if (jsonSubString !== null) {
return JSON.parse(jsonSubString);
}
return {};
}
function extractJson(message) {
var jsonStart = message.indexOf('{');
if (jsonStart < 0) return null;
var jsonSubString = message.substring(jsonStart);
return isValidJson(jsonSubString) ? jsonSubString : null;
}
function isValidJson(message) {
try {
JSON.parse(message);
} catch (e) { return false; }
return true;
}
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}
function post(body, callback) {
var requestParams = buildRequest(endpoint, body);
var request = https.request(requestParams, function(response) {
var responseBody = '';
response.on('data', function(chunk) {
responseBody += chunk;
});
response.on('end', function() {
var info = JSON.parse(responseBody);
var failedItems;
var success;
if (response.statusCode >= 200 && response.statusCode < 299) {
failedItems = info.items.filter(function(x) {
return x.index.status >= 300;
});
success = {
"attemptedItems": info.items.length,
"successfulItems": info.items.length - failedItems.length,
"failedItems": failedItems.length
};
}
var error = response.statusCode !== 200 || info.errors === true ? {
"statusCode": response.statusCode,
"responseBody": responseBody
} : null;
callback(error, success, response.statusCode, failedItems);
});
}).on('error', function(e) {
callback(e);
});
request.end(requestParams.body);
}
function buildRequest(endpoint, body) {
var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/);
var region = endpointParts[2];
var service = endpointParts[3];
var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
var date = datetime.substr(0, 8);
var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
var kRegion = hmac(kDate, region);
var kService = hmac(kRegion, service);
var kSigning = hmac(kService, 'aws4_request');
var request = {
host: endpoint,
method: 'POST',
path: '/_bulk',
body: body,
headers: {
'Content-Type': 'application/json',
'Host': endpoint,
'Content-Length': Buffer.byteLength(body),
'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
'X-Amz-Date': datetime
}
};
var canonicalHeaders = Object.keys(request.headers)
.sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
.map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
.join('\n');
var signedHeaders = Object.keys(request.headers)
.map(function(k) { return k.toLowerCase(); })
.sort()
.join(';');
var canonicalString = [
request.method,
request.path, '',
canonicalHeaders, '',
signedHeaders,
hash(request.body, 'hex'),
].join('\n');
var credentialString = [ date, region, service, 'aws4_request' ].join('/');
var stringToSign = [
'AWS4-HMAC-SHA256',
datetime,
credentialString,
hash(canonicalString, 'hex')
] .join('\n');
request.headers.Authorization = [
'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
'SignedHeaders=' + signedHeaders,
'Signature=' + hmac(kSigning, stringToSign, 'hex')
].join(', ');
return request;
}
function hmac(key, str, encoding) {
return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
}
function hash(str, encoding) {
return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
}

Node js async convert to sync

I need to push data to array synchronously. First API request get image key base one that need to get image data within loop.
var deasync = require('deasync');
router.get('/a', function(req, res) {
var username="user";
var passw ="pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
//this is first api request
client.get(global.apiUrl+"V1/ProductItem", args,
function (data, response) {
//this is second api request
data.forEach(function(img) {client.get(global.apiUrl+"V1/ImagePreview/"+img.AvatarKey,args,
function (data2, response){
img['data']=data2.Image;
deasync(pushData(img));
});
});
});
function pushData(img){
op.push(img);//array push
}
res.render('test1', { "out":JSON.stringify(op) });
});

As much as I think deasync is a poor choice of solving your particular issue, the key to using it, is to "deasync" asynchronous functions. As Array.push is synchronous, deasync'ing Array.push makes no sense
having read the documentation for deasync, it's fairly simple to use
var deasync = require('deasync');
// create a sync client.get
function syncClientGet(client, url, args) {
var inflight = true;
var ret;
client.get(url, args, function(data, response) {
// as your original code ignores response, ignore it here as well
ret = data;
inflight = false;
});
deasync.loopWhile(() => inflight);
return ret;
}
router.get('/a', function(req, res) {
var username = "user";
var passw = "pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
let data = syncClientGet(client, global.apiUrl + "V1/ProductItem", args);
data.forEach(function(img) {
let data2 = syncClientGet(client, global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args);
img['data'] = data2.Image;
op.push(img);
});
res.render('test1', {
"out": JSON.stringify(op)
});
});
However, embracing asynchronicity, the code you posted could be easily written as
router.get('/a', function (req, res) {
var username = "user";
var passw = "pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
client.get(global.apiUrl + "V1/ProductItem", args, function (data, response) {
data.forEach(function (img) {
client.get(global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args, function (data2, response) {
img['data'] = data2.Image;
op.push(img);
if (img.length == data.length) {
res.render('test1', {
"out": JSON.stringify(op)
});
}
});
});
});
});
or, using Promises
router.get('/a', function (req, res) {
var username = "user";
var passw = "pass";
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
// create a Promisified client get
var clientGetPromise = function clientGetPromise(client, url, args) {
return new Promise(function (resolve, reject) {
return client.get(url, args, function (data, response) {
return resolve(data);
});
});
};
clientGetPromise(client, global.apiUrl + "V1/ProductItem", args).then(function (data) {
return Promise.all(data.map(function (img) {
return clientGetPromise(client, global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args).then(function (data2) {
img['data'] = data2.Image;
return img;
});
}));
}).then(function (op) { // op is an Array of img because that's how Promise.all rolls
return res.render('test1', { "out": JSON.stringify(op) });
});
});

Process javascript object from html lists

I have a html structure as,
<ul>
<li>data</li>
<li><strong>data</strong></li>
<li><span>data.name</span></li>
</ul>
I want to process it to javascript object something as below,
[
{
'index': 0,
'render': function (data) {
return data;
}
},
{
'index': 1,
'render': function (data) {
return '<strong>' + data + '</strong>';
}
},
{
'index': 2,
'render': function (data) {
return '' + data.name + '';
}
}
]
I tried this code but the data parameter which is a json object are not being resolved in render field content.
var obj = [];
$('ul li').each(function(i, content){
var row = {};
row.index = i;
row.render = function(data) {
return content;
};
obj.push(row);
});
What I am looking is this code should work,
var data = {};
data.name = 'Example';
data.link = 'http://example.com';
console.log(obj[2].render(data));
It must return Example as string.

Are you looking for a string concatenation?
https://jsfiddle.net/oeavo45w/
var obj = [];
$('ul li').each(function (i, content) {
var row = {};
row.index = i;
row.render = function (data) {
console.log(111, data)
return '' + data.name + ''
};
obj.push(row);
});
var data = {};
data.name = 'Example';
data.link = 'http://example.com';
console.log(obj[2].render(data));

Solution: Parse an HTML Object and Evaluate Text as data Variable
Grab content and split it on the variable name data, making sure to accommodate key names. Provide the resulting array to a render function that checks for key names and replaces the placeholders with data supplied in the render parameter.
var obj = [];
$('ul li').each(function(i, content){
var content_arr = content.innerHTML.split(/data\.?([^"<]+)?/);
var row = {};
row.index = i;
row.render = function(data) {
var return_string = '';
for ( ii in content_arr ) {
if ( ii % 2 ) {
if ( content_arr[ii] ) {
return_string += data[content_arr[ii]];
}
else if ( 'string' === typeof data ) {
return_string += data;
}
}
else { return_string += content_arr[ii]; }
}
return return_string;
};
obj.push(row);
});
// data object
var data = {};
data.name = 'EXAMPLE';
data.link = 'http://example.com';
// <span>EXAMPLE</span>
// or data string
data = 'testme';
console.log(obj[1].render(data));
// <strong>testme</strong>
http://jsfiddle.net/heksh7Lr/6/
(Ya, I'm guessing HTML template libraries will be more powerful.)

How to feed a class a list of options through a function?

I'm trying out classes in javascript also using extend.js to create them. I am having some trouble with classes and am not really sure if I need them, but I think this one will be useful. It's a class to just get some database table data.
var TableData = Class.extend(function(){
this.rowId;
this.action;
this.returnWhat;
this.tableType;
this.php;
this.query = function(){
if (this.returnWhat) this.action += '_' + this.returnWhat;
var postData = { func : this.action };
if (this.rowId) $.extend(postData, { rowId : this.rowId });
$.post(this.php, postData, function(data){
if(data){this.data = data;}
});
return this.data;
};
});
I will have only tables that I know of, so I just store them as vars/options, but I don't know how to set the class vars up properly. This for example does not work but maybe demonstrates what I'm trying to do.
var options = {
rowId : '*',
action : 'get',
returnWhat : 'all'
}
var league = function (options){
this.tableType = 'league';
this.rowId = options.rowId;
this.action = options.action;
this.returnWhat = options.returnWhat;
this.php = 'php/' + options.tableType + '.php';
}
var tableData = new TableData(assoc_league);

I always like this idea
(function(my, $){
var cfg = {
rowId: null,
action: null,
returnWhat: null,
tableType: null,
url: null
};
my.init = function(options) {
if (options != undefined) {
for (var k in options) {
if (!options.hasOwnProperty(k)) continue;
cfg[k] = options[k];
}
}
return this;
};
my.fetchData = function(data, callback) {
$.post(cfg.url, data, function(resp) {
callback(resp);
});
}
})(window.myapp, window.jQuery);
//---------------
myapp.init({
url: '/blah/blah'
})
.fetchData({animal1: 'monkey', animal2: 'zebra'}, function(data){
});

Issue with Async Operation Setting Object Property in JS

I'm developing a client-side library for a web-based service and I'm having some issues with setting an object variable and later retrieving it.
Here is the start of the library
var QuickBase = function(username, password, apptoken, realm) {
this.username = username;
this.password = password;
this.apptoken = (typeof apptoken === "undefined") ? '' : apptoken;
this.realm = (typeof realm === "undefined") ? 'www' : realm;
this.ticket = '';
this.dbid = '';
this.payload = '<qdbapi>';
this.init = function() {
var self = this;
this.authenticate(this.username, this.password, null, null, function(data) {
var errcode = $(data).find('errcode').text();
if(errcode > 0)
throw new Error($(data).find('errtext').text());
self.ticket = $(data).find('ticket').text();
});
}
this.setBaseUrl = function() {
var endpoint = this.dbid == '' ? 'main' : this.dbid;
this.baseUrl = 'https://' + this.realm + '.quickbase.com/db/' + endpoint;
}
this.transmit = function(method, callback) {
this.setBaseUrl();
if(this.apptoken)
this.payload += '<apptoken>' + this.apptoken + '</apptoken>';
if(this.ticket)
this.payload += '<ticket>' + this.ticket + '</ticket>';
this.payload += '</qdbapi>';
console.log(this.payload);
$.ajax({
url: this.baseUrl,
type: 'POST',
data: this.payload,
dataType: 'xml',
headers: {
'Content-Type': 'application/xml',
'QUICKBASE-ACTION': method
},
success: callback
});
this.payload = '<qdbapi>';
}
this.addSettingsToPayload = function(settings) {
for(var key in settings) {
this.payload += '<' + key + '>' + settings[key] + '</' + key + '>';
}
}
this.authenticate = function(username, password, hours, udata, callback) {
this.payload += '<username>' + username + '</username>';
this.payload += '<password>' + password + '</password>';
this.payload += (typeof hours === "undefined") ? '' : '<hours>' + hours + '</hours>';
this.payload += (typeof udata === "undefined") ? '' : '<udata>' + udata + '</udata>';
this.transmit('API_Authenticate', callback);
}
And here's the use case:
var username = 'foo',
password = 'bar',
token = 'footoken',
realm = 'foorealm';
window.qb = new QuickBase(username, password, token, realm);
$.when(qb.init()).then(function(){
console.log(qb); // shows the object with ticket set
console.log(qb.ticket); // empty
qb.doQuery(); // breaks because internal this.ticket is empty
});
So my question is why is qb.ticket not being set and not available in future function calls?
In addition, is there a way that I don't have to wrap .init() in .when?
Basically, init sets the ticket that all future API methods will need. If I just call qb.init() and then qb.doQuery(), there is no guarantee init() will have finished - but if I use .when, won't that mean all future method calls would need to be inside of the .then callback? That seems ugly.

$.when expects a promise object. If it doesn't get one it will execute the callback immediately. You have to return the promise from the $.ajax call, or create your own.
But since you are only working with one promise you don't even need when. You could do:
qb.init().done(qb.doQuery.bind(qb));
Example:
var QuickBase = function(username, password, apptoken, realm) {
// ...
this.init = function() {
var self = this;
return this.authenticate(this.username, this.password, null, null).done(function(data) {
var errcode = $(data).find('errcode').text();
if(errcode > 0)
throw new Error($(data).find('errtext').text());
self.ticket = $(data).find('ticket').text();
});
}
this.transmit = function(method) {
// ...
var promise = $.ajax({
url: this.baseUrl,
type: 'POST',
data: this.payload,
dataType: 'xml',
headers: {
'Content-Type': 'application/xml',
'QUICKBASE-ACTION': method
}
});
this.payload = '<qdbapi>';
return promise;
}
this.authenticate = function(username, password, hours, udata) {
// ..
return this.transmit('API_Authenticate');
}
}
As alternative, you can just make .init accept a callback:
this.init = function(callback) {
var self = this;
this.authenticate(this.username, this.password, null, null, function(data) {
var errcode = $(data).find('errcode').text();
if(errcode > 0)
throw new Error($(data).find('errtext').text());
self.ticket = $(data).find('ticket').text();
callback();
});
};
and then:
qb.init(function(){
qb.doQuery();
});

Easiest approach is probably to return a promise from init(), as follows :
this.init = function() {
var self = this;
var dfrd = $.Deferred();
this.authenticate(this.username, this.password, null, null, function(data) {
var errcode = $(data).find('errcode').text();
if(errcode > 0)
throw new Error($(data).find('errtext').text());
self.ticket = $(data).find('ticket').text();
dfrd.resolve();
});
return dfrd.promise();
}
Then :
window.qb = new QuickBase(username, password, token, realm);
qb.init().then(function() {
console.log(qb);
console.log(qb.ticket);
qb.doQuery();
});