Using jquery.go.js to retrieve raw HTTP responses - javascript

I am quite new to Jquery, node.js, Phantom.js and JavaScript in general and having a few problems wrapping my head around all of it.
I'm working on a simple script using Jquery.go.js (https://github.com/travist/jquery.go.js) that should log me in to a site, then access a specific resource and parse the response (JSON) that it returns.
So far I managed to get the login working, but am really lost now for the second part.
What I have so far is based on https://github.com/travist/makemeasandwich.js
var async = require('async'),
$ = require('jquerygo'),
path = require('path'),
fs = require('fs');
// Add some default configs.
$.config.site = 'https://host.com';
$.config.addJQuery = false;
var login = function(done) {
async.series([
$.go(false, 'visit', '/login'),
$.go(false, 'waitForElement', '#email'),
debugCapture('login1'),
$('#email').go('val', 'email#example.com'),
$('#password').go('val', 'securepassword'),
debugCapture('login2'),
$("form").go('attr','id','validForm'),
$('#validForm').go('submit'),
sleep(3000),
debugCapture('login3'),
print('Successfully logged in.'),
sleep(3000),
], done);
};
var viewJournalEntries = function(done){
$.config.addJQuery = true;
async.series([
$.go(false, 'visit', '/api/journals/show/546'),
$.getPage(function(page) {
// Is this right, what do I need to do here??
}),
debugCapture('step2'),
],done);
}
var debugCapture = function(fileName) {
// if (config.get('debug')) {
return capture(fileName);
// }
return function(done) { done(); }
};
/**
* Method to capture and ensure the screenshots directory exists.
*/
var capture = function(fileName) {
// Return the async function.
return function(done) {
// The directory to store the screenshots.
var dir = __dirname + '/screenshots-ls';
// Check that the directory exists.
fs.exists(dir, function(exists) {
if (exists) {
$.capture(dir + '/' + fileName + '.png', done);
}
else {
fs.mkdir(dir, function(err) {
if (err) return done(err);
$.capture(dir + '/' + fileName + '.png', done);
});
}
});
}
}
/**
* Helper to print something when it is executed.
*
* #param {type} text
* #returns {unresolved}
*/
var print = function(text) {
return function(done) {
console.log(text);
done();
};
};
/**
* Helper function to take a pause...
*
* #param {type} time
* #returns {unresolved}
*/
var sleep = function(time) {
return function(done) {
setTimeout(done, time);
};
};
async.series([
login,
viewJournalEntries,
], function() {
$.close();
});
The login works like a charm and I can call the viewJournalEntries function which calls the url /api/journals/show/546 that returns a typical JSON string like this:
{"data":{"id":546,"user_id":1,[etc...]
The screen capture proofs that it works, but I am just not sure how to proceed to get the JSON into a variable that I can parse.
Any help/directions are appreciated.
Thanks,
Ben

Related

How to get the value of a variable when you cant debug or use console.log()

I am trying to find where this extension I installed is storing the templates that are made. I want to know this and HOW they are stored so that I can change the code so that there is a specific template made within the extension itself. A default extension.
I think ive located where the directory for the templates is made but I cant seem to debug or console.log to get the value of this path. I believe the path will be the variable templateDir thats within the TemplatesManager.prototype.getTemplates function...
here is the templatesManager.js file so you can see how that variable is made.
It would be helpful if someone could help me understand how to obtain the value of that variable. I tried console.log(templateDir) but nothing pops up on the VS Code terminal, output, or any other console window.
'use strict';
var fs = require('fs');
var path = require('path');
var os = require('os');
/**
* Main class to handle the logic of the File Templates
* #export
* #class TemplatesManager
*/
var TemplatesManager = (function () {
function TemplatesManager(config) {
this.config = config;
}
/**
* Returns a list of available file templates by reading the Templates Directory.
* #returns string[]
*/
TemplatesManager.prototype.getTemplates = function () {
// #TODO make this async (use promises ???)
var templateDir = this.getTemplatesDir();
var templateFiles = fs.readdirSync(templateDir).map(function (item) {
if (!/^\./.exec(item)) {
return fs.statSync(path.join(templateDir, item)).isFile() ? item : null;
}
return null;
}).filter(function (filename) {
return filename !== null;
});
return templateFiles;
};
/**
* Read the contents of a templates
* #param filename The name of the template file.
* #return Buffer
*/
TemplatesManager.prototype.getTemplate = function (filename) {
return fs.readFileSync(path.join(this.getTemplatesDir(), filename), 'utf8');
};
/**
* Returns the templates directory location.
* If no user configuration is found, the extension will look for
* templates in USER_DATA_DIR/Code/FileTemplates.
* Otherwise it will look for the path defined in the extension configuration.
* #return {string}
*/
TemplatesManager.prototype.getTemplatesDir = function () {
return this.config.get('templates_dir', this.getDefaultTemplatesDir());
};
/**
* Returns the default templates location based on the user OS.
* #returns {string}
*/
TemplatesManager.prototype.getDefaultTemplatesDir = function () {
var userDataDir = null;
switch (process.platform) {
case 'linux':
userDataDir = path.join(os.homedir(), '.config');
break;
case 'darwin':
userDataDir = path.join(os.homedir(), 'Library', 'Application Support');
break;
case 'win32':
userDataDir = process.env.APPDATA;
break;
default:
throw Error("Unrecognizable operative system");
}
return path.join(userDataDir, 'Code', 'User', 'FileTemplates');
};
/**
* Creates the templates dir if not exists
* #throw Error
*/
TemplatesManager.prototype.createTemplatesDirIfNotExists = function () {
var templatesDir = this.getTemplatesDir();
fs.mkdir(templatesDir, '0755', function (err) {
if (err && err.code != 'EEXIST') {
throw Error("Failed to created templates directory " + templatesDir);
}
});
};
return TemplatesManager;
}());
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = TemplatesManager;
//# sourceMappingURL=templatesManager.js.map

How to run a script certain time using nodejs?

whenever readDirectory invoked i am checking if any file created Date is past 30 days i want to remove that file from direcotry. This is working as expected and removing the files but i dont think its efficient. So I want to move comapreDates method to separate file and call directory logs/St every night and check if any file expired passed 30 days period remove it. is it doable ?
service.js
function readDirectory(callback) {
var dirPath = './logs/St';
var files = fs.readdirSync(dirPath);
async.eachSeries(files, function(file, callback) {
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't");
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
compareDates(fileInfo, filePath);
objToReturn.push(fileInfo);
callback();
}
});
}, function(err) {
//final callback when all files completed here send objToReturn to client
callback(objToReturn);
});
}
}
function compareDates(file, path) {
var timeDiff = Math.abs(currentDate.getTime() - file.fileDate.getTime());
var dayDifference = Math.ceil(timeDiff / (1000 * 3600 * 24));
console.log('Days', dayDifference);
console.log('FileName', file.filename);
if (dayDifference >= 30) {
fs.unlink(path, function(err) {
if (err) {
// file doens't exist
console.info("File doesn't exist, won't remove it.");
} else {
console.log('removed file', file);
}
});
}
}
See one of those modules:
https://www.npmjs.com/package/node-schedule
https://www.npmjs.com/package/node-cron
https://www.npmjs.com/package/node-cron-job
and more modules here:
https://www.npmjs.com/browse/keyword/schedule
https://www.npmjs.com/browse/keyword/cron
https://www.npmjs.com/browse/keyword/job

DocumentDB connection in js: "require is not defined"

I've been able to create the database and query it. Using Microsoft's tutorial on using node.js to query, I have been able to accomplish this with this code:
// Simple Query
"use strict";
var documentClient = require("documentdb").DocumentClient;
var config = require("./config");
var url = require('url');
// use the previously saved config.endpoint and config.primaryKey to create a new DocumentClient
var client = new documentClient(config.endpoint, { "masterKey": config.primaryKey });
// These urls are how the DocumentDB client will find the right database and collection.
var HttpStatusCodes = { NOTFOUND: 404 };
var databaseUrl = `dbs/${config.database.id}`;
var collectionUrl = `${databaseUrl}/colls/${config.collection.id}`;
// Query JSON document collection
function queryCollection() {
console.log(`Querying collection through index:\n${config.collection.id}`);
return new Promise((resolve, reject) => {
client.queryDocuments(
collectionUrl,
'SELECT VALUE gd.NFL FROM GamblersDenDB gd WHERE gd.id = "SanDiego"'
).toArray((err, results) => {
if (err) reject(err)
else {
for (var queryResult of results) {
let resultString = JSON.stringify(queryResult);
console.log(`\tQuery returned ${resultString}`);
}
console.log();
resolve(results);
}
});
});
};
queryCollection()
Running that js file in my command prompt works! It results in the output:
C:\Users\kenv\Desktop\DocDB Test>node SimpleQuery.js
Querying collection through index:
GamblersDenColl
Query returned {"ID":"SDC","name":"Chargers"}
Great. So now I've transferred my code to my project's folder and try to run in the app with
taco run android --device
When I pull up the console, the first error that sticks out to me is
Uncaught ReferenceError: require is not defined(…)
It's pointing to the line var documentClient = require("documentdb").DocumentClient; in my code.
Here is my complete js file code that's throwing the error:
(function () {
"use strict";
document.addEventListener( 'deviceready', onDeviceReady.bind( this ), false );
function onDeviceReady() {
navigator.splashscreen.hide();
console.log("Cordova is READY!");
// Handle the Cordova pause and resume events
document.addEventListener( 'pause', onPause.bind( this ), false );
document.addEventListener( 'resume', onResume.bind( this ), false );
$(".btnURL").on("click", function(){loadURL($(this))});
function loadURL(theObj) {
cordova.InAppBrowser.open(theObj.data("url"), "_blank", "location=yes");
}
//********* jQuery VARIABLES ***************//
var $elBtnSaveName= $("#btnSaveName"),
$elShowClients= $("#btnShowClients"),
$elDivShow= $("#divShow"),
$elFormClient= $("#formClient");
//********** EVENT HANDLERS *****************//
$elShowClients.on("click", queryCollection);
//********************* DOCUMENT DB SECTION *********************************************************/
var documentClient = require("documentdb").DocumentClient;
var config = require("./config");
var url = require('url');
// use the previously saved config.endpoint and config.primaryKey to create a new DocumentClient
var client = new documentClient(config.endpoint, { "masterKey": config.primaryKey });
// These urls are how the DocumentDB client will find the right database and collection.
var HttpStatusCodes = { NOTFOUND: 404 };
var databaseUrl = `dbs/${config.database.id}`;
var collectionUrl = `${databaseUrl}/colls/${config.collection.id}`;
// Query JSON document collection
function queryCollection() {
console.log(`Querying collection through index:\n${config.collection.id}`);
return new Promise((resolve, reject) => {
client.queryDocuments(
collectionUrl,
'SELECT VALUE gd.NFL FROM GamblersDenDB gd WHERE gd.id = "SanDiego"'
).toArray((err, results) => {
if (err) reject(err)
else {
for (var queryResult of results) {
let resultString = JSON.stringify(queryResult);
console.log(`\tQuery returned ${resultString}`);
}
console.log();
resolve(results);
fnShowClientsTable(result.rows);
}
});
});
};
function fnShowClientsTable(data){
var str = "<p><table id='tableResults'";
str += "<tr><th>ID</th><th>Name</th><th class='thEmpty'> </th></tr>" //added class to <th> for formatting
for(var i = 0; i < data.length; i++) { // For X number of times worth of data...
str += "<tr><td>" + data[i].doc.ID +
"</td><td>" + data[i].doc.name +
"</td><td class='btnPencil'>✎</td></tr>";
}
str += "</table></p>"; // END table
$elDivShow.html(str); //Show string as HTML on screen
} // END fnShowClientsTable
//************************* END DOCUMENT DB SECTION ******************************************************/
}; // END onDeviceReady()
function onPause() {
// TODO: This application has been suspended. Save application state here.
};
function onResume() {
// TODO: This application has been reactivated. Restore application state here.
};
} )();
I realize my other function to push it to a string for display in HTML is probably wrong (which I will most certainly have another post about that one :)) but right now I'm trying to determine how I can get past this first error.

Node.js/Javascript: Failing mocha test with timeout on testing parsing XML to JSON

Can someone please help me figure out why am I failing the mocha test even after setting the timeout to 15000ms. I know it can be a memory leak etc but I cannot find it anywhere in my code. I used the chrome debugger and checked that the JS Heap uses 48MB memory at the max. (which is not too bad?) Here's the function I'm testing, it's in a file called xmlParser.js
var fs = require('fs'),
xml2js = require('xml2js'),
parser = new xml2js.Parser();
/**
* [parse - parses a xml file to json]
* #param {[string]} filename [name of the file to parse]
* #param {Function} callback [callback function]
* #return {[Function]} [returning callback function]
*/
function parse(filename, callback){
fs.readFile(filename, function(err, data){
if(err){
console.log('cannot read file.');
return callback(err);
} else {
parser.parseString(data, function(err, result){
if(err) {
console.log('cannot parse file.');
return callback(err);
}
else {
return callback(null, result);
}
});
}
});
}
module.exports = {
parse
}
What does it do? Parses a xml file to JSON. As simple as that!
Here's how I'm testing it,
var assert = require('chai').assert,
mocha = require('mocha'),
xmlParser = require('../utils/xmlParser.js'),
extractInformation = require('../utils/extractInformation.js');
//data-structures for parsed XML data (lists)
var logJSON = [], listJSON = [];
describe('parse xml files', function(){
this.timeout(150000);
it('should parse correctly', function(done){
this.timeout(150000);
setTimeout(done, 150000);
xmlParser.parse(__dirname + "/../xml/svn_log_test.xml", function(err, log) {
if(err) {
return done(err);
// assert.equal(true, false);
}
this.logJSON = log["log"]["logentry"];
xmlParser.parse(__dirname + "/../xml/svn_list_test.xml", function(err, list) {
if(err) {
return done(err);
// assert.equal(true, false);
}
this.listJSON = list["lists"]["list"][0]["entry"];
});
});
});
});
Now, before you say that I'm not using any assert statements as of now, yes I'm not. But it should pass anyway, right? It's not passing, it gets a timeout error after 15000s
Why don't you try to decrease the setTimout to 14 seconds?
setTimeout(function () {
done()
}, 140000);
The timers are not quite precise in javascript

Mocha test is failing but implementation works

I am trying to write a newer watch module that uses the fs.watch method instead of the watchFile approach.
So far, it works beautifully, but only when I run it outside of mocha. I can't figure out why my unit test is throwing a tantrum, maybe someone here can?
Here is my class code:
/**
* requirements
*/
var fs, path, events;
fs = require('fs');
path = require('path');
events = require('events');
/**
* private
*/
var Monitor = function(directory, options) {
this.directory = directory;
this.options = options || {};
(this.options.lazy && this.empty()) || this.walk(this.directory);
this.watch(this.directory);
};
Monitor.prototype = new events.EventEmitter();
Monitor.prototype.watch = function(directory, stats) {
var stats = stats || {};
if (!this.directories[directory]) {
var w = fs.watch(directory, this.options, this.justlookatit.bind(this));
}
this.directories[directory] = { 'stats': stats, 'w': w };
};
Monitor.prototype.directories = function() {
if (!Object.keys(this.directories).length) {
this.walk(this.directory);
}
return this.directories;
};
Monitor.prototype.files = function() {
if (!Object.keys(this.files).length) {
this.walk(this.directory);
}
return this.files;
};
Monitor.prototype.unwatch = function() {
if (!Object.keys(this.directories).length) {
for (var dir in this.directories) {
dir.w.close();
}
}
};
Monitor.prototype.empty = function() {
this.unwatch();
this.files = {};
this.directories = {};
};
Monitor.prototype.walk = function(directory) {
var monitor = this;
this.empty();
fs.readdir(directory, function(err, files) {
if (err) return;
for (var file in files) {
var fullname = path.resolve(files[file]);
if (!monitor.options.filter || monitor.options.filter(fullname)) {
fs.stat(fullname, function(err, stats) {
if (err) return;
if (stats.isDirectory()) {
monitor.walk(fullname);
monitor.watch(fullname, stats);
} else {
monitor.files[fullname] = stats;
}
});
}
}
});
};
Monitor.prototype.justlookatit = function(action, file) {
var monitor = this;
var fullname = path.resolve(file);
if (this.options.filter && !this.options.filer(fullname)) return;
fs.exists(fullname, function(exists) {
if (exists) {
fs.stat(fullname, function(err, stats) {
if (stats.isDirectory()) {
monitor.watch(fullname, stats);
} else {
if (monitor.files[fullname]) {
if (stats.mtime.getTime() > monitor.files[fullname].mtime.getTime()) {
monitor.emit('modified', fullname, stats);
}
} else {
monitor.emit('added', fullname, stats);
}
monitor.files[fullname] = stats;
}
});
} else {
if (monitor.files[fullname]) {
delete monitor.files[fullname];
monitor.emit('deleted', fullname);
} else if (monitor.directories[fullname]) {
monitor.directories[fullname].w.close();
delete monitor.directories[fullname];
}
}
});
};
/**
* exports
*/
exports.start = function(directory, options) {
return new Monitor(directory, options);
};
Here is my Working external test code:
var watch = require("./watch.js");
var fs = require('fs');
monitor = watch.start(__dirname);
monitor.on('added', function(file, stats) {
console.log("Caught Added: " + file);
});
monitor.on('modified', function(file, stats) {
console.log("Caught Modified: " + file);
});
monitor.on('deleted', function(file) {
console.log("Caught deleted: " + file);
});
// try creating a file immediately
fs.openSync('v.md', 'w');
The first test file runs perfectly fine, and I've tried both openSync and open. Finally, here is a version of the same test code, wrapped in a mocha unit test which is timing out:
/**
* requirements
*/
var watch, Q, fs, path, mocha, chai, assert;
watch = require('../lib/watch.js');
Q = require('q');
fs = require('fs');
path = require('path');
mocha = require('mocha');
chai = require('chai');
assert = chai.assert;
/**
* variables
*/
var watch_directory = path.join(__dirname, './watch');
/**
* tests
*/
describe('test watch', function() {
it('should create a monitor and run callbacks after fs changes', function(done) {
// I had planned to implement promises that chained the three callbacks
// but couldn't get one of them working in general
var added = function(file, stats) {
console.log("added");
done();
};
var modified = function(file, stats) {
console.log("modified");
};
var deleted = function(file, stats) {
console.log("deleted");
};
// create our service
var monitor = watch.start(watch_directory);
// assert it is defined
assert.isDefined(monitor);
// establish a listener
monitor.on('added', added);
monitor.on('modified', modified);
monitor.on('deleted', deleted);
// here is a file name using the current date to prevent duplication during tests
var file = path.join(watch_directory, (new Date()).getTime() + '.md');
// let's create the file, then delete it
fs.open(file, 'w+', function(err, fileDescriptor) {
// this prints before console output from the watch.js's `justlookatit` method
console.log(err);
console.log("writing to file");
// we probably don't want to try closing the fileDescriptor if the open failed
if (err) return;
// close the file descriptor
fs.close(fileDescriptor, function() {
// delete the file we just created
// fs.unlink(file, function() { /* not a big deal */ });
});
});
// modify a known-existing test file
fs.open('test.md', 'w+', function() {/* we don't care about this */});
})
});
I checked with console.log(fullname) inside the justlookatit method on the watch code, and it spits out the correct file name, matching the one generated by the unit test.
However, it then proceeds to return false when I run fs.exists. As I undestand it, that means the file system is notifying me that a file exists before it exists, which doesn't make sense really. So I tried adding an additional delay by wrapping my fs.exists method in a setTimeout, and that didn't change the results. I have also tried using both openSync and existsSync, and that made no difference.
I'm stumped, does anyone have any ideas why the mocha code isn't working?
So, the solution was to go for a walk. I came back, looked at the code again and figured out the cause of the problem with mocha, and also identified many other bugs.
The problem was the lack of context. The justlookatit method does not have a context, and in the test.js scenario it is watching the current directory, while the mocha test is watching a sub-directory.
The path.resolve was receiving only the file name, not the directory, and therefore merged it with the default (executables) directory, so the level of test.js, or watch_test.js for mocha. It proceeded to fail to locate any of the files in the mocha test case because they were all one level below the executable.
I won't go into detail about all the other bugs, but I may come back and post the repository link when I get to a point that I want to push it online.
You're missing the callback return(done); at the end of your test. Unless you call that callback, Mocha will time out every time.

Categories