Node newbie here I have a problem that I have been banging my head against for a few days now and I'm sure I am missing something small but my google too has failed me so far.
I have an AWS Firehose that is writing CloudWatch logs to S3 that are then picked up by a lambda function parsed and ingested into Elasticsearch. Or at least that is the plan. I reason I am using Firehose is that the data is coming from remote AWS accounts and sending all the logs to one place s3. I am able to pull the logs once in S3. Cloud watch logs by default are gzipped so the firehose is simply writing the files to S3 no transforming or anything. Now the issue :). Once I read the file from S3 I attempt to send it to a function transform to parse the data but it fails when trying to loop through the log which is JSON. I did a console.log(payload) and it appears to be in JSON format but running it through JSON.stringify the back to parsing does nothing. JSON.parse will work but the JSON will not be valid.
I am attaching the code below. I am not the original author someone in my company found it on GitHub. I just added the S3 parts to it.
I added a file that was one of the logs as an example it's in a public s3 bucket if anyone wants to grab it. No worries its all test data nothing special.
https://s3.amazonaws.com/node-issue-stackoverflow/cwl-test-11-2018-08-26-00-45-34-84a4c3de-179a-4bf2-9376-895cdc063e6b+(1)
Pastebin Link
// v1.1.2
var https = require('https');
var zlib = require('zlib');
var crypto = require('crypto');
var AWS = require('aws-sdk');
var endpoint = process.env.es_endpoint;
var s3 = new AWS.S3();
var params;
exports.handler = function(input, context) {
// Get the event from S3 based on input
params = {Bucket: input.Records[0].s3.bucket.name, Key: input.Records[0].s3.object.key};
console.log(input.Records[0].s3);
s3.getObject(params, function(error, event){
if (error) { context.fail(error); return; }
console.log(event);
console.log(event.Body);
// decode input from base64
var zippedInput = new Buffer(event.Body, 'base64');
// decompress the input
zlib.gunzip(zippedInput, function(error, buffer) {
if (error) { context.fail(error); return; }
// console.log(buffer.toString());
// parse the input from JSON
var awslogsData = buffer.toString('utf8');
// transform the input to Elasticsearch documents
var elasticsearchBulkData = transform(awslogsData);
// skip control messages
if (!elasticsearchBulkData) {
console.log('Received a control message');
context.succeed('Control message handled successfully');
return;
}
// post documents to the Amazon Elasticsearch Service
post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
console.log('Response: ' + JSON.stringify({
"statusCode": statusCode
}));
if (error) {
console.log('Error: ' + JSON.stringify(error, null, 2));
if (failedItems && failedItems.length > 0) {
console.log("Failed Items: " +
JSON.stringify(failedItems, null, 2));
}
context.fail(JSON.stringify(error));
} else {
console.log('Success: ' + JSON.stringify(success));
context.succeed('Success');
}
});
});
});
};
function transform(payload) {
if (payload.messageType === 'CONTROL_MESSAGE') {
return null;
}
var bulkRequestBody = '';
payload.logEvents.forEach(function(logEvent) {
var timestamp = new Date(1 * logEvent.timestamp);
// index name format: cwl-YYYY.MM.DD
var indexName = [
'cwl-' + timestamp.getUTCFullYear(), // year
('0' + (timestamp.getUTCMonth() + 1)).slice(-2), // month
('0' + timestamp.getUTCDate()).slice(-2) // day
].join('.');
var source = buildSource(logEvent.message, logEvent.extractedFields);
source['#id'] = logEvent.id;
source['#timestamp'] = new Date(1 * logEvent.timestamp).toISOString();
source['#message'] = logEvent.message;
source['#owner'] = payload.owner;
source['#log_group'] = payload.logGroup;
source['#log_stream'] = payload.logStream;
var action = { "index": {} };
action.index._index = indexName;
action.index._type = payload.logGroup;
action.index._id = logEvent.id;
bulkRequestBody += [
JSON.stringify(action),
JSON.stringify(source),
].join('\n') + '\n';
});
return bulkRequestBody;
}
function buildSource(message, extractedFields) {
if (extractedFields) {
var source = {};
for (var key in extractedFields) {
if (extractedFields.hasOwnProperty(key) && extractedFields[key]) {
var value = extractedFields[key];
if (isNumeric(value)) {
source[key] = 1 * value;
continue;
}
jsonSubString = extractJson(value);
if (jsonSubString !== null) {
source['$' + key] = JSON.parse(jsonSubString);
}
source[key] = value;
}
}
return source;
}
jsonSubString = extractJson(message);
if (jsonSubString !== null) {
return JSON.parse(jsonSubString);
}
return {};
}
function extractJson(message) {
var jsonStart = message.indexOf('{');
if (jsonStart < 0) return null;
var jsonSubString = message.substring(jsonStart);
return isValidJson(jsonSubString) ? jsonSubString : null;
}
function isValidJson(message) {
try {
JSON.parse(message);
} catch (e) { return false; }
return true;
}
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}
function post(body, callback) {
var requestParams = buildRequest(endpoint, body);
var request = https.request(requestParams, function(response) {
var responseBody = '';
response.on('data', function(chunk) {
responseBody += chunk;
});
response.on('end', function() {
var info = JSON.parse(responseBody);
var failedItems;
var success;
if (response.statusCode >= 200 && response.statusCode < 299) {
failedItems = info.items.filter(function(x) {
return x.index.status >= 300;
});
success = {
"attemptedItems": info.items.length,
"successfulItems": info.items.length - failedItems.length,
"failedItems": failedItems.length
};
}
var error = response.statusCode !== 200 || info.errors === true ? {
"statusCode": response.statusCode,
"responseBody": responseBody
} : null;
callback(error, success, response.statusCode, failedItems);
});
}).on('error', function(e) {
callback(e);
});
request.end(requestParams.body);
}
function buildRequest(endpoint, body) {
var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/);
var region = endpointParts[2];
var service = endpointParts[3];
var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
var date = datetime.substr(0, 8);
var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
var kRegion = hmac(kDate, region);
var kService = hmac(kRegion, service);
var kSigning = hmac(kService, 'aws4_request');
var request = {
host: endpoint,
method: 'POST',
path: '/_bulk',
body: body,
headers: {
'Content-Type': 'application/json',
'Host': endpoint,
'Content-Length': Buffer.byteLength(body),
'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
'X-Amz-Date': datetime
}
};
var canonicalHeaders = Object.keys(request.headers)
.sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
.map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
.join('\n');
var signedHeaders = Object.keys(request.headers)
.map(function(k) { return k.toLowerCase(); })
.sort()
.join(';');
var canonicalString = [
request.method,
request.path, '',
canonicalHeaders, '',
signedHeaders,
hash(request.body, 'hex'),
].join('\n');
var credentialString = [ date, region, service, 'aws4_request' ].join('/');
var stringToSign = [
'AWS4-HMAC-SHA256',
datetime,
credentialString,
hash(canonicalString, 'hex')
] .join('\n');
request.headers.Authorization = [
'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
'SignedHeaders=' + signedHeaders,
'Signature=' + hmac(kSigning, stringToSign, 'hex')
].join(', ');
return request;
}
function hmac(key, str, encoding) {
return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
}
function hash(str, encoding) {
return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
}
Related
I'm currently using AWS Lambda JavaScript code to try and search a DynamoDB table this is then implemented into an Amazon Alexa application, but that isn't really important for what I'm asking. Here is the code I'm struggling with:
function readDynamoItem(params2, callback) {
var AWS = require('aws-sdk');
AWS.config.update({region: AWSregion});
var dynamodb = new AWS.DynamoDB();
console.log('reading item from DynamoDB table');
dynamodb.scan(params2, function (err, data){
if (err) {
callback("error");
//console.log(err, err.stack); // an error occurred
}
else{
callback(data);
}
});
}
So when an error occurs I want it to callback the message "error" and then use it here:
const params2 = {
TableName: 'Fixtures',
FilterExpression: 'team1 = :value',
ExpressionAttributeValues: {':value': {"S": MyQuestion.toLowerCase()}}
};
readDynamoItem(params2, myResult=>{
say = myResult;
this.response.speak(say).listen('try again');
this.emit(':responseReady');
});
All I'm getting at the moment is this response when I test, I think due to err just ending the program instead of calling the error back to use in the implementation:
Response:
{
"errorMessage": "RequestId: 0f586880-2ddb-11e8-bdf7-07b4c224b25d Process exited before completing request"
}
Any help would be greatly appreciated.
Here's the full code for my project for further reference:
const AWSregion = 'eu-west-1';
const Alexa = require('alexa-sdk');
const AWS = require('aws-sdk');
AWS.config.update({
region: AWSregion
});
exports.handler = function(event, context, callback) {
var alexa = Alexa.handler(event, context);
// alexa.appId = 'amzn1.echo-sdk-ams.app.1234';
// alexa.dynamoDBTableName = 'YourTableName'; // creates new table for session.attributes
alexa.registerHandlers(handlers);
alexa.execute();
};
const handlers = {
'LaunchRequest': function () {
this.response.speak('welcome to magic answers. ask me a yes or no question.').listen('try again');
this.emit(':responseReady');
},
'MyIntent': function () {
var MyQuestion = this.event.request.intent.slots.MyQuestion.value;
console.log('MyQuestion : ' + MyQuestion);
const params2 = {
TableName: 'Fixtures',
FilterExpression: 'team1 = :value',
ExpressionAttributeValues: {':value': {"S": MyQuestion.toLowerCase()}}
};
const params3 = {
TableName: 'Fixtures',
FilterExpression: 'team2 = :value',
ExpressionAttributeValues: {':value': {"S": MyQuestion.toLowerCase()}}
};
readDynamoItem(params2, myResult=>{
var say = MyQuestion;
//if nothing is found when scanning for team1, scan team2
if (myResult == "error"){
readDynamoItem(params3, myResult2=>{
say = myResult2;
say = 'The top scorer for ' + MyQuestion + ' is ' + myResult2;
this.response.speak(say).listen('try again');
this.emit(':responseReady');
});
}
else{
say = myResult;
say = 'The top scorer for ' + MyQuestion + ' is ' + myResult;
this.response.speak(say).listen('try again');
this.emit(':responseReady');
}
});
},
'AMAZON.HelpIntent': function () {
this.response.speak('ask me a yes or no question.').listen('try again');
this.emit(':responseReady');
},
'AMAZON.CancelIntent': function () {
this.response.speak('Goodbye!');
this.emit(':responseReady');
},
'AMAZON.StopIntent': function () {
this.response.speak('Goodbye!');
this.emit(':responseReady');
}
};
// END of Intent Handlers {} ========================================================================================
// Helper Function =================================================================================================
//reading the Fixtures table
function readDynamoItem(params2, callback) {
var AWS = require('aws-sdk');
AWS.config.update({region: AWSregion});
var dynamodb = new AWS.DynamoDB();
var team1;
var team2;
console.log('reading item from DynamoDB table');
dynamodb.scan(params2, function (err, data){
if (err) {
callback("error");
//callback("error");
//console.log(err, err.stack); // an error occurred
}
else{
console.log(data); // successful response
team1 = jsonToString(data.Items[0].team1);
team2 = jsonToString(data.Items[0].team2);
var t1goals = jsonToString(data.Items[0].t1goals);
var t2goals = jsonToString(data.Items[0].t2goals);
t1goals = parseInt(t1goals);
t2goals = parseInt(t2goals);
var search;
var chosenValue = Math.random() < 0.5 ? team1 : team2;
// if goals are equal in a match then it is random which team will score next
if(t1goals == t2goals){
search = chosenValue;
}
//if a team has 1 goal more than the other then it is a 3rd more likely they will score next
else if(t1goals > t2goals && t1goals == 1){
if(randomInt(1, 3) == 1){
search = team2;
}
else{
search = team1;
}
}
else if(t2goals > t1goals && t2goals == 1){
if(randomInt(1, 3) == 1){
search = team1;
}
else{
search = team2;
}
}
//if a team has more than 1 goal more than the other then it is a 5th more likely they will score next
else if(t1goals > t2goals && t1goals > 1){
if(randomInt(1, 5) == 1){
search = team2;
}
else{
search = team1;
}
}
else if(t2goals > t1goals && t2goals > 1){
if(randomInt(1, 5) == 1){
search = team1;
}
else{
search = team2;
}
}
var params = {
TableName: 'yesno',
FilterExpression: 'team = :value',
ExpressionAttributeValues: {':value': {"S": search}}
};
readDynamoFixtures(params, myResult=>{
callback(myResult);
});
}
});
}
//read player details from the the yesno table
function readDynamoFixtures(params, callback) {
var goals = new Array();
var playing = new Array();
var messages = new Array();
var most = 0;
var mostMessage;
var dynamodb = new AWS.DynamoDB();
dynamodb.scan(params, function (err, data) {
if (err) console.log(err, err.stack); // an error occurred
else{
for(var i = 0; i <= (data.Count - 1); i++){
console.log(data); // successful response
var temp = jsonToString(data.Items[i].playername);
messages[i] = temp;
temp = jsonToString(data.Items[i].goals);
temp = parseInt(temp);
goals[i] = temp;
temp = jsonToString(data.Items[i].playing);
playing[i] = temp;
//compare each players goals
if (goals[i] > most && playing[i] == "true"){
most = goals[i];
mostMessage = messages[i];
}
}
}
callback(mostMessage);
});
}
//convert database items from json format to string
function jsonToString(str){
str = JSON.stringify(str);
str = str.replace('{\"S\":\"', '');
str = str.replace('\"}', '');
return str;
}
//get a random int between min and max
function randomInt(min,max)
{
return Math.floor(Math.random()*(max-min+1)+min);
}
Edit:
I have tried testing this code with .query instead of .scan and the error callback works perfectly which is strange but obviously for this implementation I need to use .scan
When you get the "Process exited" response from the Lambda it is helpful to log heavily to see where the Lambda is getting stuck and then check the Cloudwatch Logs to get to the detail.
Then you can pinpoint the exception and focus on it. At least for me, the root cause was many times unexpected as Lambdas force a different way of thinking.
I am still pretty new to this, so forgive me if I dont' say this correctly. We have an array.reduce that calls a method with a returning promise that iterates through a list of files and post results to the db. Everything was working great, until it ran into a field that had an apostrophe in it and then the db insert fails. This is the field value. 'Expected 100002822' to be 100002822.'
I tried adding a replaceAll on the field and now get an error in the array.reduce.
Here is the .reduce
console.log('Found test results in ' + matches.length + ' files. Parsing and posting to the database now...');
var startTime = moment();
var parser = new Parser();
matches.reduce(function (p, val) {
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
var endTime = moment();
var testDuration = moment.duration(endTime.diff(startTime));
console.log(chalk.blue('*** File parsing time: ' + testDuration.humanize() + ' ***'));
if (finalResult.insertSuccess == matches.length) {
var publishOut = {
totalFiles: matches.length,
totalTests: 0,
totalTestsSuccess: 0,
totalTestsFailed: 0
}
publishOut.totalTests += finalResult.totalTests;
publishOut.totalTestsSuccess += finalResult.testPassedCount;
publishOut.totalTestsFailed += finalResult.testFailedCount;
console.log(`Successfully inserted ${finalResult.insertSuccess} of ${publishOut.totalTests} test results.`);
// for (var i = 0; i < matches.length; i++) {
// var currentFile = `./testing/results/${matches[i]}`;
// fs.unlinkSync(currentFile);
// }
resolve(publishOut);
} else {
reject('Only ' + finalResult.insertSuccess + ' of ' + matches.length + ' successfully posted to the database');
}
}, function (err) {
reject('error in reduce', err);
});
I have tried several different ways of using the replaceAll with the same failure. It hits this code from the array.reduce
}, function (err) {
reject('error in reduce', err);
});
And this is the called method. The added code causing the failure in the .reduce is this Message = expectation.message.replaceAll("'", "");
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
//console.log('In parseresults', fileName);
var currentFile = './testing/results/' + fileName
json.readFile(currentFile, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function (obj) {
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
testResults.totalTests++;
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length - 1];
Passed = 0;
Message = expectation.message.replaceAll("'", "");
Stack = expectation.stack.split('\n')[1].trim();
testResults.testFailedCount++
} else {
testResults.testPassedCount++
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.then(() => {
//fs.unlinkSync(currentFile);
testResults.insertSuccess++;
//console.log('insertSuccess', testResults.insertSuccess);
},
err => { console.log('… failed', err); throw err; }
));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results).then(() => testResults);
});
};
I'm quite new to the topic and i'm still having some issues with my mailparser. Though searching and finding emails in the email header (mail.from) does work, it doesn't work in the email body. Does anybody have some experience with that and is willing to help? You can find the function i'm talking about under the "// Check for other addresses in Mail-Body (Doesn't work yet)"-comment. I think, that my Regex is correct. Also if the matchAll-Function give back an array and it can't be saved in the the subscriber.email-object, it shall be at least logged to the console. Also i checked manually in the inbox if there are mails with email adresses in the mail body. There are at least two, which shall be found..
The part of the App.js, that does the mailparsing:
const simpleParser = require('mailparser').simpleParser;
//const htmlparser = require("htmlparser2");
var fs = require('fs');
var config = require('./config');
var Imap = require('imap');
var imap = new Imap(config.imap);
var blacklistString = '';
String.prototype.matchAll = function(regexp) {
var matches = [];
this.replace(regexp, function() {
var arr = ([]).slice.call(arguments, 0);
var extras = arr.splice(-2);
arr.index = extras[0];
arr.input = extras[1];
matches.push(arr);
});
return matches.length ? matches : null;
};
function openInbox(subbox,cb) {
imap.openBox('INBOX.'+subbox, true, cb);
}
function getBoxes(cb) {
imap.getBoxes(cb);
}
function showBoxes(boxes) {
imap.end();
}
function logArrayElements(element) {
if(element[1].indexOf('placeholder.de')==-1){
addToBlacklistString(element[1]);
}
}
function addToBlacklistString(str) {
blacklistString += str+"\n";
}
function writeBlacklistFile() {
fs.appendFile('data/data.csv', blacklistString, function (err) {
if (err) throw err;
console.log('Saved!');
});
}
function search(searchArray, regex){
imap.search(searchArray, function(err, results) {
if (err) throw err;
var temp = 0;
var mailtemp = [];
var f = imap.fetch(results, { bodies: '' });
f.on('message', function(msg, seqno) {
console.log('Message #%d', seqno);
var prefix = '(#' + seqno + ') ';
msg.on('body', function(stream, info) {
simpleParser(stream, (err, mail)=>{
//console.log(temp);
//console.log(mail.subject);
/*fs.writeFile('data/'+seqno+'.txt',mail.text, function(err){
console.log(err);
});*/
//var text = mail.text;
// New Subscriber Object
var subscr = new Subscriber({nr: '', mailIdent: '', from: '', emails: '', text:'', uLink: '', anwalt: false });
subscr.nr = seqno;
//Check for From-Address
if(!!mail.from) {
//console.log(mail.from.value);
for(var i = 0; i < mail.from.value.length; i++) {
mailtemp = mail.from.value[i].address.matchAll(regex);
mailtemp.forEach(function(element){
/*fs.appendFile('data/data.csv', element[0] + "\n", function(error){
console.log(error);
});*/
subscr.from = element[0];
});
if(!!mailtemp) {
mailtemp.forEach(logArrayElements);
}
}
}else{
//console.log(mail.text);
}
// Message-ID
if(!!mail.messageId) {
subscr.mailIdent = mail.messageId;
}
console.log(mail.messageId);
// Check for other addresses in Mail-Body (Doesn't work yet)
var regexEmails = new RegExp('/([\w\.\-\_\#\+]+#[\w\.\-\_äüö]+\.[a-zA-Z]+)/g');
if(!!mail.text){
if(mail.text.matchAll(regexEmails)!=null) {
subscr.emails = mail.text.matchAll(regexEmails);
console.log(subscr.emails);
}
}
/* Split mail.text at substrings in substr-array. Extend if necessary..
*
* Also check for 'Anwalt'-Expression in splitted Substring
*
* If mail.text doesn't exist -> Check for html body and convert it to text-format
*/
//var regexLink = new RegExp('\.de\/(unsubscribe|austragen)\/([^\"]+)');
var regexAnwalt = new RegExp('nwalt|echtsanwalt|rechtlicher');
if(!!mail.text) {
var substr = ["schrieb pplaceholder.de", "Von: \"placeholder.de", "Von: pplaceholder.de", "From: placeholder.de", "Ursprüngliche Nachricht"];
for (var i = 0; i<substr.length; i++) {
if(mail.text.indexOf(substr[i]) > -1) {
var textTemp = mail.text;
var arr = textTemp.split(substr[i]);
if(arr[0].matchAll(regexAnwalt)!=null) {
subscr.anwalt = true;
};
subscr.text = arr[0];
break;
} else {
subscr.text = mail.text;
}
}
//console.log(arr);
}
else
{
var html = mail.html;
var text = htmlToText.fromString(html, {
noLinkBrackets: true,
ignoreImage: true,
uppercaseHeadings: false,
preserveNewlines: false,
wordwrap:130,
format: {
heading: function (node, fn, options) {
var h = fn(node.children, options);
return '\n==== ' + h + ' ====\n\n';
}
}
});
subscr.text = text;
}
mail.headers.forEach(function(value, key) {
//console.log(value);
});
subscr.save();
//console.log(subscr);
temp++;
});
});
msg.once('end', function() {
console.log(prefix + 'Finished');
});
});
f.once('error', function(err) {
console.log('Fetch error: ' + err);
});
f.once('end', function() {
console.log('Done fetching all messages!');
//writeBlacklistFile();
imap.end();
});
});
}
imap.once('ready', function() {
openInbox('Test',function(err, box) {
var searchArray = [['FROM', '#']];
search(searchArray,/([\w\.\-\_\#\+]+#[\w\.\-\_äüö]+\.[a-zA-Z]+)/g);
});
});
imap.once('error', function(err) {
console.log(err);
});
imap.once('end', function() {
console.log('Connection ended');
});
imap.connect();
app.listen(2700, function(){
console.log("Listening on Port 2700")
});
module.exports = app;
subscriber.js
const mongoose = require('mongoose');
var subscriberSchema = mongoose.Schema({
nr: Number,
mailIdent: String,
from: String,
emails: String,
text: String,
uLink: String,
anwalt: Boolean
});
var Subscriber = module.exports = mongoose.model('Subscriber', subscriberSchema);
//get Subscriber
module.exports.getSubscribers = function(callback, limit){
Subscriber.find(callback).limit(limit);
};
module.exports.getSubscriberByID = function(_id, callback){
Subscriber.findById(_id, callback);
};
The Regex for the Emails was a little bit wrong.
Also i didn't noticed that the matchAll-Fct. is giving back a two-dimensional Array. Here is the changed part of the code:
var regexEmails = new RegExp("([\\w\\.\\-\\_\\#\\+]+#[\\w\\.\\-\\_äüö]+\\.[a-zA-Z]+)");
var temp1 = mail.text.matchAll(regexEmails);
if(!!temp1){
//console.log(temp1);
for(var i =0; i<temp1.length; i++) {
if(temp1[0][i]!=='info#service.placeholder.de' && temp1[0][i] !== "info#placeholder.de"){
subscr.emails += temp1[0][i];
}
}
}
I need to push data to array synchronously. First API request get image key base one that need to get image data within loop.
var deasync = require('deasync');
router.get('/a', function(req, res) {
var username="user";
var passw ="pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
//this is first api request
client.get(global.apiUrl+"V1/ProductItem", args,
function (data, response) {
//this is second api request
data.forEach(function(img) {client.get(global.apiUrl+"V1/ImagePreview/"+img.AvatarKey,args,
function (data2, response){
img['data']=data2.Image;
deasync(pushData(img));
});
});
});
function pushData(img){
op.push(img);//array push
}
res.render('test1', { "out":JSON.stringify(op) });
});
As much as I think deasync is a poor choice of solving your particular issue, the key to using it, is to "deasync" asynchronous functions. As Array.push is synchronous, deasync'ing Array.push makes no sense
having read the documentation for deasync, it's fairly simple to use
var deasync = require('deasync');
// create a sync client.get
function syncClientGet(client, url, args) {
var inflight = true;
var ret;
client.get(url, args, function(data, response) {
// as your original code ignores response, ignore it here as well
ret = data;
inflight = false;
});
deasync.loopWhile(() => inflight);
return ret;
}
router.get('/a', function(req, res) {
var username = "user";
var passw = "pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
let data = syncClientGet(client, global.apiUrl + "V1/ProductItem", args);
data.forEach(function(img) {
let data2 = syncClientGet(client, global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args);
img['data'] = data2.Image;
op.push(img);
});
res.render('test1', {
"out": JSON.stringify(op)
});
});
However, embracing asynchronicity, the code you posted could be easily written as
router.get('/a', function (req, res) {
var username = "user";
var passw = "pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
client.get(global.apiUrl + "V1/ProductItem", args, function (data, response) {
data.forEach(function (img) {
client.get(global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args, function (data2, response) {
img['data'] = data2.Image;
op.push(img);
if (img.length == data.length) {
res.render('test1', {
"out": JSON.stringify(op)
});
}
});
});
});
});
or, using Promises
router.get('/a', function (req, res) {
var username = "user";
var passw = "pass";
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
// create a Promisified client get
var clientGetPromise = function clientGetPromise(client, url, args) {
return new Promise(function (resolve, reject) {
return client.get(url, args, function (data, response) {
return resolve(data);
});
});
};
clientGetPromise(client, global.apiUrl + "V1/ProductItem", args).then(function (data) {
return Promise.all(data.map(function (img) {
return clientGetPromise(client, global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args).then(function (data2) {
img['data'] = data2.Image;
return img;
});
}));
}).then(function (op) { // op is an Array of img because that's how Promise.all rolls
return res.render('test1', { "out": JSON.stringify(op) });
});
});
I looked at various answers but I can't not find a way to get this going for myself.
I have a function (in node.js) that selects a winner out of a pool, when it selects the winner though, I need it to send a event to the client where it runs a function with data. The data would be the array index of the winner.
Selecting a winner:
var endRound = function() {
ref.child('currentJackpot').once('value', function(data) {
var currentJackpot = data.val();
var winnerArray = [];
var winnerObj = {};
winnerObj.items = [];
for (var i = 0; i < currentJackpot.players.length; i++) {
winnerObj.items = winnerObj.items.concat(currentJackpot.players[i].items);
var playerValue = currentJackpot.players[i].itemsValue * 100;
currentJackpot.players[i].chance = ((currentJackpot.players[i].itemsValue / currentJackpot.jackpotValue) * 100).toFixed(2);
for (var j = 0; j < playerValue; j++) {
winnerArray.push(i);
}
}
var formatted = currentJackpot.roundHash.replace(/[.#$/]/g, "");
sgRef.child(formatted).once('value', function(data) {
var sgData = data.val();
salt = sgData.salt;
rngStr = sgData.rngStr;
console.log('ROUND ENDED! hash: ', hash, ' salt: ', salt, ' rngStr: ', rngStr);
currentJackpot.tickets = currentJackpot.jackpotValue * 100;
currentJackpot.winningTicket = Math.floor((parseFloat(rngStr, 2) * currentJackpot.tickets));
currentJackpot.winningNumber = (parseFloat(rngStr, 2) * 100).toFixed(2) + "%";
currentJackpot.winner = currentJackpot.players[winnerArray[currentJackpot.winningTicket]];
currentJackpot.salt = salt;
currentJackpot.rngStr = rngStr;
winnerObj.jackpotValue = currentJackpot.jackpotValue;
currentJackpot.jackpotValue = currentJackpot.jackpotValue.toFixed(2);
winnerObj.winner = currentJackpot.winner;
winnerObj.tradeToken = currentJackpot.winner.tradeToken;
ref.child('endedJackpots').push(currentJackpot);
bcrypt.genSalt(10, function(err, data) {
salt = data;
rngStr = JSON.stringify(rng());
bcrypt.hash(rngStr, salt, function(err, data) {
hash = data;
ref.child('currentJackpot').set({
itemsCount: 0,
jackpotValue: 0,
roundHash: hash,
}, function() {
console.log('NEW ROUND! hash: ', hash, 'salt: ', salt, 'rngStr: ', rngStr);
var formatted = hash.replace(/[.#$/]/g, "");
var sgJackpotRef = sgRef.child(formatted);
sgRef.set({}, function() {
sgJackpotRef.set({
salt: salt,
rngStr: rngStr,
}, function() {
request.post({
url: '*******',
body: winnerObj,
json: true,
}, function(error, response, body) {
if (error) {
console.log(error);
setPollTimer(10000);
} else {
usersRef.child(winnerObj.winner.id).once('value', function(data) {
var userData = data.val();
if (data.child('won').exists()) {
userData.won = (Math.floor(parseFloat(userData.won, 2)) + Math.floor(parseFloat(winnerObj.jackpotValue, 2))).toFixed(2);
} else {
userData.won = (Math.floor(parseFloat(winnerObj.jackpotValue, 2))).toFixed(2);
}
usersRef.child(winnerObj.winner.id).update({
won: userData.won
}, function() {
console.log('Added winnings to user data');
});
});
console.log('Making a withdraw request now to bot');
setPollTimer(10000);
}
});
});
});
});
});
});
});
});
};
And the function it should run client side:
function slotMachine(winnerIndex) {
var params = {
active: 3,
randomize: function(activeElementIndex){
return 1;
}
};
var machine = $('.slot').slotMachine( params );
$("#slotMachineButton").click(function(){
machine.shuffle(3, function(){
$(this).text("Index: " + this.active);
});
});
}
I'm completely stuck and stressed though, this isn't my code and It's a lot of code I can't grasp where to begin to do this. Any help?