We Keep Coding

Require in forloop not working as expected

I have the following code:
var tableRequiredList = [];
var requireListPath = [
'./slimShady.js',
'./chickaChicka.js'
];
var getRequires = function() {
for (var i = 0; i < requireListPath.length; i++) {
((requireNamePath) => {
try {
console.log("INSIDE LOOP RESULT", i, require(requireNamePath)().getName()); // Outputs correct result for the index ("ChickaChicka")
tableRequiredList.push({ "name": requireNamePath, "theReq": require(requireNamePath)() });
// tableRequiredList.push({ "name": requireNamePath, "theReq": ((thePath) => { return require(thePath)(); })(requireNamePath) }); // This also doesn't seem to work.
} catch(err) {
console.log("Error importing: ", requireNamePath, " Error reported: ", err);
}
})(requireListPath[i]);
};
console.log("NAME", tableRequiredList[0].name); // Outputs the correct result ("slimShady.js")
console.log("FUNC NAME", tableRequiredList[0].theReq.getName()); // Always outputs the last item in requireListPath list ("ChickaChicka")
};
getRequires();
Example Module 1 - slimShady.js
((module) => {
module.exports = {};
var exampleModuleName1 = function() {
this.getName = function() {
return 'myNameIsSlimShady';
};
return this;
};
module.exports = exampleModuleName1;
})(module);
Example Module 2 - chickaChicka.js
((module) => {
module.exports = {};
var exampleModuleName2 = function() {
this.getName = function() {
return 'ChickaChicka';
};
return this;
};
module.exports = exampleModuleName2;
})(module);
Why does it output:
INSIDE LOOP RESULT 0 myNameIsSlimShady
INSIDE LOOP RESULT 1 ChickaChicka
NAME ./slimShady.js
FUNC NAME ChickaChicka
When it should be outputting the first index of the tableRequiredList array? This seems to only happen with require(). I have tried using map and forEach, along with the closure example above. All have the same results.

Thanks to #liliscent, I figured it out.
Just needed to change modules to this:
((module) => {
module.exports = {};
var exampleModuleName2 = function() {
var retr = {};
retr.getName = function() {
return 'ChickaChicka';
};
return retr;
};
module.exports = exampleModuleName2;
})(module);

Confirmed populated array of objects returns empty

I have a method that is failing when returning an array of objects. As mentioned in the title - the array is confirmed to be populated but is empty in the response.
Here is the full flow:
The Url:
http://localhost:53000/api/v1/landmarks?lat=40.76959&lng=-73.95136&radius=160
Is routed to the corresponding index:
api.route('/api/v1/landmarks').get(Landmark.list);
The Index Calls a Service:
exports.list = (req, res) => {
const landmark = new LandmarkService();
landmark.getLandmarks(req)
.then(landmarks => {
var response = new Object();
response.startindex = req.query.page;
response.limit = req.query.per_page;
response.landmarks = landmarks;
res.json(response);
})
.catch(err => {
logger.error(err);
res.status(422).send(err.errors);
});
};
The Service Method Uses a Data Access Class to Return the Promise
getLandmarks(req) {
const params = req.params || {};
const query = req.query || {};
const page = parseInt(query.page, 10) || 1;
const perPage = parseInt(query.per_page, 10);
const userLatitude = parseFloat(query.lat);
const userLongitude = parseFloat(query.lng);
const userRadius = parseFloat(query.radius) || 10;
const utils = new Utils();
const data = new DataService();
const landmarkProperties = ['key','building','street','category','closing',
'email','name','opening','phone','postal','timestamp','type','web'];
return data.db_GetAllByLocation(landmarksRef, landmarkLocationsRef,
landmarkProperties, userLatitude, userLongitude, userRadius);
} // getLandmarks
However, the response is always empty.
I am building an array in the called method and populating it with JSON objects. That is what is supposed to be sent back in the response. I can confirm that the attributes array is correctly populated before I hit the return statement. I can log it to the console. I can also send back a test array filled with stub values successfully.
I have a feeling it is how I am setting things up inside the Promise?
Data Access Method That Should Return Array of Objects:
db_GetAllByLocation(ref, ref_locations, properties, user_latitude, user_longitude, user_radius)
{
const landmarkGeoFire = new GeoFire(ref_locations);
var geoQuery = landmarkGeoFire.query({
center: [user_latitude, user_longitude],
radius: user_radius
});
var locations = [];
var onKeyEnteredRegistration = geoQuery.on("key_entered", function (key, coordinates, distance) {
var location = {};
location.key = key;
location.latitude = coordinates[0];
location.longitude = coordinates[1];
location.distance = distance;
locations.push(location);
});
var attributes = [];
var onReadyRegistration = geoQuery.on("ready", function() {
ref.on('value', function (refsSnap) {
refsSnap.forEach((refSnap) => {
var list = refSnap;
locations.forEach(function(locationSnap)
{
//console.log(refSnap.key, '==', locationSnap.key);
if (refSnap.key == locationSnap.key)
{
var attribute = {};
for(var i=0; i<=properties.length-1; i++)
{
if(properties[i] == 'key') {
attribute[properties[i]] = refSnap.key;
continue;
}
attribute[properties[i]] = list.child(properties[i]).val();
}
attribute['latitude'] = locationSnap.latitude;
attribute['longitude'] = locationSnap.longitude;
attribute['distance'] = locationSnap.distance;
attributes.push(attribute);
} // refSnap.key == locationSnap.key
}); // locations.forEach
}); // refsSnap.forEach
return Promise.resolve(attributes); <-- does not resolve (throws 'cannot read property .then')
//geoQuery.cancel();
}); // ref.on
}); // onreadyregistration
return Promise.resolve(attributes); <-- comes back empty
}

It seems that data.db_GetAllByLocation is an asynchronous function, therefore the call resolve(landmarks); is getting called before the execution of the async function is finished. If the data.db_GetAllByLocation returns a promise then call the resolve(landmarks) inside the promise.
data.db_GetAllByLocation().then(function() {
resolve();
})
Also try the following modified db_GetAllByLocation()
db_GetAllByLocation(ref, ref_locations, properties, user_latitude, user_longitude, user_radius)
{
return new Promise(function(resolve, reject){
const landmarkGeoFire = new GeoFire(ref_locations);
var geoQuery = landmarkGeoFire.query({
center: [user_latitude, user_longitude],
radius: user_radius
});
var locations = [{}];
var onKeyEnteredRegistration = geoQuery.on("key_entered", function (key, coordinates, distance) {
var location = {};
location.key = key;
location.latitude = coordinates[0];
location.longitude = coordinates[1];
location.distance = distance;
locations.push(location);
});
var attributes = [{}];
var onReadyRegistration = geoQuery.on("ready", function() {
ref.on('value', function (refsSnap) {
refsSnap.forEach((refSnap) => {
var list = refSnap;
locations.forEach(function(locationSnap)
{
if (refSnap.key == locationSnap.key)
{
var attribute = {};
for(var i=0; i<=properties.length-1; i++)
{
if(properties[i] == 'key') {
attribute[properties[i]] = refSnap.key;
continue;
}
attribute[properties[i]] = list.child(properties[i]).val();
}
attribute['latitude'] = locationSnap.latitude;
attribute['longitude'] = locationSnap.longitude;
attribute['distance'] = locationSnap.distance;
attributes.push(attribute);
} // refSnap.key == locationSnap.key
}); // locations.forEach
}); // refsSnap.forEach
// return JSON.stringify(attributes);
return resolve(attributes);
}); // ref.on
}); // onreadyregistration
});
}

OK, I sorted this by removing all my code and writing some test logic (I should have done this before I posted my question).
The below flow works for me, and, applied back to my code, gave me the results I was looking for. No need to re-post the code, but maybe the below flow will be helpful to somebody.
route
api.route('/api/v1/landmarks').get(Landmark.test);
index
exports.test = (req, res) => {
const landmark = new LandmarkService();
landmark.getLandmarksTest(req)
.then(landmarks => {
var final = {};
final.attr1 = 'attr1';
final.attr2 = 'attr2';
final.landmarks = landmarks;
res.json(final);
})
.catch(err => {
logger.error(err);
res.status(422).send(err.errors);
});
};
service method
getLandmarksTest(req)
{
const data = new DataService();
data.db_PromiseTest().then(results => {
return Promise.resolve(results);
}).catch(err => {
return Promise.reject(err.errors);
});
}
data layer method
db_PromiseTest()
{
var stub = {
"name": "Madame Uppercut",
"age": 39,
"secretIdentity": "Jane Wilson",
"powers": [
"Million tonne punch",
"Damage resistance",
"Superhuman reflexes"
]
};
return Promise.resolve(stub);
}

Node.js readStream for large data processing

I'm having trouble creating a line by line node.js method of processing large nessus xml files without high RAM usage. In its current form, it is saving data in MongoDB correctly, however the RAM usage keeps increasing, and errors out with files over ~1.5GB.
I've tried using .pause() on the readStream, however, I must have implemented it incorrectly, because it never seemed to actually pause the stream.
Here is the code:
// LR.JS Imports
var fs = require('fs');
var readline = require('readline');
var stream = require('stream');
var instream = fs.createReadStream('test.nessus');
var outstream = new stream;
var rl = readline.createInterface(instream, outstream);
var buff = false;
var parseString = require('xml2js').parseString;
var buffStream = '';
//Mongoose Imports
var mongoose = require('mongoose');
var ReportHostDoc = require('./schemas/report-host.model.js');
var ReportItemDoc = require('./schemas/report-item.model.js');
var PluginDetailDoc = require('./schemas/plugin-detail.model.js');
mongoose.Promise = require('bluebird');
// Mongoose Connect
mongoose.connect('mongodb://localhost/test');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', () => {
// Create counters for _taskCheck
var reportHostCounter = 0;
var reportHostSaveCounter = 0;
rl.on('line', (line) => {
// process line here
if (/[<]ReportHost/.test(line)) {
buff = true;
reportHostCounter++;
}
if (buff == true) {
buffStream += line + '\n';
}
if (/[<][/]ReportHost/i.test(line)) {
buff = false; // changed to = not == 9/6
// XML2JS Parse ReportHost Buffstream
parseString(buffStream, (err, result) => {
// Loop through ReportHost properties to reliably find IP
var reportHostIP = '';
var reportHostOS = '';
result.ReportHost.HostProperties[0].tag.forEach((entry) => {
if (entry.$.name === 'host-ip') {
reportHostIP = entry._;
}
if (entry.$.name === 'operating-system') {
reportHostOS = entry._;
}
});
// Save Report Host Document
var host = new ReportHostDoc({
hostname: result.ReportHost.$.name,
ip: reportHostIP,
os: reportHostOS,
high: 0,
critical: 0
});
// Process Each Report Item
result.ReportHost.ReportItem.forEach((entry) => {
var cvssScore = '';
if (entry.cvss_base_score) {
cvssScore = JSON.stringify(entry.cvss_base_score).slice(2, 5)
} else {
cvssScore = 0;
}
var item = new ReportItemDoc({
itemName: entry.$.pluginName,
pluginID: entry.$.pluginID,
ipAddress: reportHostIP,
exploitAvailable: entry.exploit_available,
cvssBaseScore: cvssScore,
pluginPublishedDate: entry.plugin_publication_date,
pluginModifiedDate: entry.plugin_modification_date,
description: entry.description
})
if (item.cvssBaseScore >= 7 && item.cvssBaseScore < 10) {
host.high++;
}
if (item.cvssBaseScore == 10) {
host.critical++;
}
item.save((err, item) => {
if (err) return console.log(err);
})
});
host.save((err, host) => {
if (err) return console.log(err);
reportHostSaveCounter++;
});
})
buffStream = ''; // Empty buffer for next report host
}
});
rl.on('close', () => { // Read Stream Finished
console.log('Log Parse finished!');
var _taskCheck = setInterval(() => { // Async loop waits for all tasks to finish
if (reportHostCounter == reportHostSaveCounter) {
clearInterval(_taskCheck);
var pluginCounter = 0;
var pluginSaveCounter = 0;
ReportItemDoc.distinct('pluginID', (err, ids) => {
ids.forEach((id) => {
pluginCounter++;
ReportItemDoc.findOne({
'pluginID': id
}, (err, plugin) => {
ReportItemDoc.count({
'pluginID': id
}, (err, count) => {
var pluginSeverity = '';
var cvss = plugin.cvssBaseScore;
if (cvss >= 7 && cvss < 10) {
pluginSeverity = 'High';
}
if (cvss == 10) {
pluginSeverity = 'Critical';
}
item = new PluginDetailDoc({
pluginName: plugin.itemName,
pluginID: id,
severity: pluginSeverity,
quantity: count,
description: plugin.description
})
item.save((err, host) => {
if (err) return console.log(err);
pluginSaveCounter++;
});
})
});
})
})
var _pluginTaskCheck = setInterval(() => { // Async loop waits for all tasks to finish
if (pluginCounter == pluginSaveCounter) {
clearInterval(_pluginTaskCheck);
mongoose.connection.close();
}
}, 100);
}
}, 100);
});
});

Sinon.stub() return different values every time it's called

Here is the code I am writing tests for:
'use strict';
var internals = {};
var _ = require('lodash');
module.exports = {
initialize: function (query) {
internals.query = query;
},
createField: function (fieldId, accountId, payload) {
function callQuery (parList) {
var query = 'INSERT into fields VALUES (:uuid, :accountId, :shortcutName, :displayName, :fieldType, :widgetType, :columnOrder, :options, :required, NULL)';
return internals.query(query, parList, function () { return fieldId; });
}
var increment = 10;
var parameterList = {
'uuid': fieldId,
'accountId': accountId,
'shortcutName': payload.shortcutName,
'displayName': payload.displayName,
'fieldType': payload.fieldType,
'widgetType': payload.widgetType,
'columnOrder': payload.columnOrder,
'options': JSON.stringify(payload.options) || null,
'required': payload.required || 'f'
};
if (!payload.columnOrder) {
var columnQuery = 'SELECT MAX(column_order) from fields';
return internals.query(columnQuery, {}, function (x) {return x; })
.then(function (results) {
var highestColumnOrder = results[0]['MAX(column_order)'];
var newHighestColumnOrder = Math.ceil(highestColumnOrder / 10) * 10;
if (newHighestColumnOrder > highestColumnOrder) {
parameterList.columnOrder = newHighestColumnOrder;
} else {
parameterList.columnOrder = newHighestColumnOrder + increment;
}
return callQuery(parameterList);
});
} else {
return callQuery(parameterList);
}
},
getFieldsByAccountId: function(accountId, showDeleted) {
var callQuery = function(paramList) {
var query = 'SELECT ' + paramList.columns.join(", ") + ' FROM fields WHERE account_id = :account_id';
if (!showDeleted) {
query += ' AND archived_at IS NULL';
}
return internals.query(query, paramList, function(rows) {
return _.each(rows, function(row) {
if(row.options) {
row.options = JSON.parse(row.options);
}
row.required = !!row.required;
});
});
};
var columnList = ["uuid", "account_id", "shortcut_name", "display_name", "field_type", "required", "column_order", "options"];
var paramList = {'account_id': accountId};
if (showDeleted) {
columnList.push("archived_at");
}
_.extend(paramList, {'columns': columnList});
return callQuery(paramList);
}
};
Here is my test:
'use strict';
var assert = require('assert');
var sinon = require('sinon');
var Promise = require('bluebird');
var proxyquire = require('proxyquire');
var returnedValues = require('../../../return_values.js');
var fieldGateway = proxyquire('../../../../src/fields/lib/gateway', {});
describe('gateway', function () {
var accountId = 100;
var fieldId = 200;
var _query, sql, mockData, rows;
describe('createField', function() {
describe('is successful with a column order value', function () {
beforeEach(function() {
sql = 'INSERT into fields VALUES (:uuid, :accountId, :shortcutName, :displayName, :fieldType, :widgetType, :columnOrder, :options, :required, NULL)';
mockData = returnedValues.getFieldInputValues();
});
it("should only insert new field", function () {
_query = sinon.spy(function() { return Promise.resolve(); });
fieldGateway.initialize(_query);
fieldGateway.createField(fieldId, accountId, mockData);
mockData.accountId = accountId;
mockData.uuid = fieldId;
mockData.options = JSON.stringify(mockData.options);
assert.equal(sql, _query.getCall(0).args[0]);
assert.deepEqual(mockData, _query.getCall(0).args[1]);
});
it.only("_query should be called with the right sql statement and parameterList", function () {
_query = sinon.stub().returns(Promise.resolve(fieldId));
// _query.onCall(0).returns(Promise.resolve([{'MAX(column_order)': 10}]));
// _query.onCall(1).returns(Promise.resolve(fieldId));
fieldGateway.initialize(_query);
delete mockData.columnOrder;
fieldGateway.createField(fieldId, accountId, mockData);
console.log(_query.args);
assert.equal(sql, _query.getCall(0).args[0]);
fieldGateway.createField.restore();
});
});
});
});
The problem is that when the test runs, the only SQL query that runs is the SELECT statement. What should happen is one SQL statement runs, then an INSERT statement runs

This happens because bluebird is a true Promise/A+ compliant library. And by definition all chained promises must be run in a different execution tick. So only the first promise is executed synchronously (in same tick).
You should tell mocha to "wait" for the rest to act. You do this by specifying a done callback in your unit test and calling it accordingly when your promises finished their job
it.only("_query should be called with the right sql statement and parameterList", function (done) {
_query = sinon.stub().returns(Promise.resolve(fieldId));
fieldGateway.initialize(_query);
delete mockData.columnOrder;
fieldGateway.createField(fieldId, accountId, mockData)
.then(function(){
/// assertion code should be adjusted here
console.log(_query.args);
assert.equal(sql, _query.getCall(0).args[0]);
fieldGateway.createField.restore();
//tell Mocha we're done, it can stop waiting
done();
})
.catch(function(error) {
//in case promise chain was rejected unexpectedly
//gracefully fail the test
done(error);
};
});
Whenever you test your promise-returning functions you should always handle result in a .then

How to get dynamic HTML and Javascript values from a page using PhantomJS

How can I get the latest page data (HTML & Javascript varaibles) from PhantomJS
e.g page.refresh() or something?
I have an Interval, than checks a variable (on the page) every 200ms. However, this variable and the page content, isn't shown to have changed over time. (even though I know it has)
So I need an efficient way to check the value of a JS variable every 200ms or so,
then once I've discovered that variable has changed value, I want to request the latest page HTML.
How can I do this?
var Error = function (description) {
this.description = description;
return this;
};
var DTO = function (status, content, error) {
this.status = status;
this.content = content;
this.error = error;
return this;
};
function outputAndExit(dto) {
console.log(JSON.stringify(dto));
phantom.exit();
}
//For any uncaught exception, just log it out for .NET to capture
window.onerror = function (errorMsg, url, lineNumber) {
var description = 'window.onerror caught an error: ' +
'errorMsg: ' + errorMsg +
'url: ' + url +
'lineNumber: ' + lineNumber;
outputAndExit(new DTO(false, null, new Error(description)));
};
var GetDynamicPageResult__ = function () {
var obj = new GetDynamicPageResult();
obj.initialize();
return obj;
};
var GetDynamicPageResult = function () {
var self = this;
this.initialize = function () {
this.error = null;
this.isContentReadyForCrawler = false;
this.ticker = null;
this.tickerInterval = 150;
this.tickerElapsed = 0;
this.url = '';
this.loadDependencies();
this.processArgs();
this.openPage();
};
this.loadDependencies = function () {
this.system = require('system'),
this.page = require('webpage').create(),
this.page.injectJs('jquery-1.10.2.min');
this.fs = require('fs');
};
this.processArgs = function () {
if (this.system.args.length == 0) {
outputAndExit(new DTO(false, null, new Error('No arguments given')));
}
//system.args[0] Was the name of this script
this.url = this.system.args[1];
};
this.updateIsContentReadyForCrawler = function () {
var updateIsContentReadyForCrawler = self.page.evaluate(function () {
self.isContentReadyForCrawler = window.isContentReadyForCrawler;
});
};
this.openPage = function () {
self.page.open(this.url, function (status) { //NB: status = 'success' || 'fail'
if (status !== 'success') {
outputAndExit(new DTO(false, null, new Error('page.open received a non-success status')));
}
self.initTicker();
});
};
this.initTicker = function () {
this.ticker = setInterval(self.handleTick, self.tickerInterval);
};
this.handleTick = function () {
self.tickerElapsed += self.tickerInterval;
self.updateIsContentReadyForCrawler();
if (self.isContentReadyForCrawler) {
clearInterval(self.ticker);
var content = self.page.content;
self.finish(true, content, null);
} else {
var tooMuchTimeElapsed = self.tickerElapsed > 7000;
if (tooMuchTimeElapsed) {
clearInterval(self.ticker);
self.finish(false, null, new Error('Too much time elapsed'));
}
}
};
this.finish = function (status, content, error) {
content = content || '';
error = error || {};
outputAndExit(new DTO(status, content, error));
};
};
/**********************************************************************************/
/***************************** Helpers *****************************/
/**********************************************************************************/
var Utility__ = function () {
var obj = new Utility();
obj.initialize();
return obj;
};
var Utility = function () {
var self = this;
this.initialize = function () {
};
this.isEmpty = function (obj) {
var isEmpty = false;
(obj == undefined || obj == null) && (isEmpty = true);
return isEmpty;
};
this.isStringEmpty = function (str) {
var isEmpty = false;
isEmpty(str) && (isEmpty = true);
(isEmpty == false && $.trim(str) == '') && (isEmpty = true);
return isEmpty;
};
};
var getDynamicPageResult = new GetDynamicPageResult__();

I think you are almost there: you need to be using page.evaluate(), but currently only use it to get window.isContentReadyForCrawler. You need to use page.evaluate() to grab the latest HTML too.
I'm going to shamelessly paste in code from another answer (https://stackoverflow.com/a/12044474/841830):
var html = page.evaluate(function () {
var root = document.getElementsByTagName("html")[0];
var html = root ? root.outerHTML : document.body.innerHTML;
return html;
});