We Keep Coding

Recursively walk website pages with phantomjs

I have to go through all the pages from a website and check for an element on every page. This has to happen recursively, and I chose to do it with PhantomJS. So, I basically have this/such code in main.js:
var page = require('webpage').create();
var allUrls = [];
var pageCheck = function(url) {
page.open(url, function(success) {
page.evaluate(function(allUrls, nextPage) {
// crawl all links, and if they are from this site ..
// add them to the allUrls array ..
// then check the page for the element ..
// and go to next eventual page ..
setTimeout(nextPage, 250);
}, allUrls, nextPage);
});
};
var nextPage = function() {
var nextUrl = allUrls.unshift();
if(nextUrl) pageCheck(nextUrl);
};
pageCheck('http://example.com/');
and I call this with phantomjs main.js.
But I see messages that "Can't find variable ...". And when I cleared all of them - I see now Can't find variable: pageCheck
How am I supposed to do this? ... And what is all this stuff with PhantomJS scopes ?? ...

I managed to figure it out, thanks to #ArtjomB :)
Basically, my mistake was that I was trying to call global stuff from page.evaluate, while I had to use it only for page manipulation. So I changed the code to this/such one:
var page = require('webpage').create();
var allUrls = [];
var pageCheck = function(url) {
page.open(url, function(success) {
var evalulation = page.evaluate(function() {
// gather urls and check element ..
return {
urls: ...,
checkedElement: ...
};
});
// manipulate the results from page.evaluate ..
someStuff(evalulation.urls);
otherStuff(evalulation.checkedElement);
// and THEN ... go to next eventual page ..
setTimeout(nextPage, 250);
});
};
var nextPage = function() {
var nextUrl = allUrls.unshift();
if(nextUrl) pageCheck(nextUrl);
else phantom.exit();
};
pageCheck('http://example.com/');

how to pass function output to another function

was playing around with casperjs lately and didnt manage to complete the below code, was using child_process and need to get function output to be passed to another function any ideas ?
success call variable scope limited to success function only, and i cant use it anywhere in my code
casper.repeat(3, function() {
this.sendKeys(x('//*[#id="text-area"]'), testvalue.call(this)); // testvalue.call(this) dosnt input anything here
})
casper.echo(testvalue.call(this)); // Print output successfully
function testvalue() {
var spawn = require("child_process").spawn
var execFile = require("child_process").execFile
var child = spawn("/usr/bin/php", ["script.php"])
child.stdout.on("data", function (data) {
console.log(JSON.stringify(data)); // Print output successfully
return JSON.stringify(data); // Problem is here i cant use Data any where in code except this scope
})
}

Since spawn is an asynchronous process, you need to use a callback for testvalue. Returning something inside the event handler doesn't return it from testvalue.
The other problem is that you need to remain in the CasperJS control flow. This is why I use testvaluedone to determine if the spawned process is done executing and I can completeData.
casper.repeat(3, function() {
var testvaluedone = false;
var completeData = "";
testvalue();
this.waitFor(function check(){
return testvaluedone;
}, function then(){
this.sendKeys(x('//*[#id="text-area"]'), completeData);
}); // maybe tweak the timeout a little
});
var testvaluedone, completeData;
function testvalue() {
var spawn = require("child_process").spawn;
var execFile = require("child_process").execFile;
var child = spawn("/usr/bin/php", ["script.php"]);
child.stdout.on("data", function (data) {
completeData += JSON.stringify(data);
});
child.on("exit", function(code){
testvaluedone = true;
});
}

I want to load local store in between the server response (making java script code synchronysed)

I have view page where am trying to display all organizations,which is
obtained by a server call..In order to feel the application responsive
, in between the server response I want to load all local store
items.. But server call is always executing first.. The code I
mentioned bellow..
initialize: function()
{
var me = this,
st = Ext.create("Ext.data.Store", {
fields : [ {......................}]});
me.callParent(arguments);
me.setStore(st);
me.on({
show : me.onShow,
scope: me
});
},
onShow:function()
{
var me = this;
Ext.create('Ext.util.DelayedTask',
//call back function ,purpose : delayed exicution
function () {
me.DelShow(function(){
_syncMgr.getOrgGroup(-1,0,5); // servercall
});
}).delay(500);
},
DelShow: function(callback)
{
orgStore = Ext.getStore('Organizations'),
orgStore.load(function(records)
{
var i=0,len = records.length,
for(;i<len;i++)
{
organization = records[i];
regId = organization.get('rg_id');
resStr = organization.Resources();
resStr.load({callback:function(resorces)
{
var i = 0,rlen =resorces.length,
obj = {},
obj.rg_id = str.boundTo.get('rg_id');
}
orgViStr.add([obj]);
});
}
});
me.lodorg(callback);
},
lodorg:function(callback)
{
callback();
console.log("I don't know why this call back works first....");
console.log("plz help me to work last....");
}

Call You callback method after You load data from local store .

What you want to do is add the content from your local store to the list that is currently set in the view then refresh the list.
var localStore = something.getStore(),
entries = [];
localStore.each(function(entry){
entries.push(entry.copy);
});
//me is the list that is set in the view
me.add(entries);
me.deselectAll();
me.refresh();
Hope that helps :)

How to call another function from the function that is passed to evaluate in CasperJS

Here is what I am trying to do:
var casper = require('casper').create();
casper.start('mysite.html', function() {
});
/* Casper configuration */
casper.on('remote.message', function(msg) {
this.echo('remote message caught: ' + msg);
})
function getAllImages()
{
// Get all images
var images = document.getElementsByTagName('img');
evaluateImages(images);
}
function evaluateImages(images)
{
console.log("I am in evaluateImages");
}
// Then find all pictures
casper.then(function() {
var product_image = this.evaluate(getAllImages);
});
casper.run();
But it never gets to evaluteImages function. What am I missing here?

Well, I have never used CasperJS, however from what i've read in the docs, I believe that it's because the evaluateImages function doesn't exist in the page's context.
I am not sure what's the best practice here, however it seems that you can return primitives from evaluate callbacks, so you could technically do something like:
function getAllImages() {
// Get all images
var images = document.getElementsByTagName('img');
//Return an array that contains all images src attribute
return Array.prototype.map.call(images, function (img) {
return img.src;
});
}
Then you can do something like:
var images = this.evaluate(getAllImages);
evaluateImages(images);
EDIT:
I need to treat all the images in the page context, as I am comparing
each of them with other elements that are in the page...
Perhaps you could define a module in the page's context first:
this.evaluate(function () {
window.yourNS = {
evaluateImages: function (images) {
//do something with images
},
getAllImages: function () {
return document.getElementsByTagName('img');
}
};
});
Then you can do something like:
this.evaluate(function () {
yourNS.evaluateImages(yourNS.getAllImages());
});

How does jQuery do async:false in its $.ajax method?

I have a similar question here, but I thought I'd ask it a different way to cast a wider net. I haven't come across a workable solution yet (that I know of).
I'd like for XCode to issue a JavaScript command and get a return value back from an executeSql callback.
From the research that I've been reading, I can't issue a synchronous executeSql command. The closest I came was trying to Spin Lock until I got the callback. But that hasn't worked yet either. Maybe my spinning isn't giving the callback chance to come back (See code below).
Q: How can jQuery have an async=false argument when it comes to Ajax? Is there something different about XHR than there is about the executeSql command?
Here is my proof-of-concept so far: (Please don't laugh)
// First define any dom elements that are referenced more than once.
var dom = {};
dom.TestID = $('#TestID'); // <input id="TestID">
dom.msg = $('#msg'); // <div id="msg"></div>
window.dbo = openDatabase('POC','1.0','Proof-Of-Concept', 1024*1024); // 1MB
!function($, window, undefined) {
var Variables = {}; // Variables that are to be passed from one function to another.
Variables.Ready = new $.Deferred();
Variables.DropTableDeferred = new $.Deferred();
Variables.CreateTableDeferred = new $.Deferred();
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'drop table Test;',
[],
Variables.DropTableDeferred.resolve()
// ,WebSqlError
);
});
$.when(Variables.DropTableDeferred).done(function() {
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'CREATE TABLE IF NOT EXISTS Test'
+ '(TestID Integer NOT NULL PRIMARY KEY'
+ ',TestSort Int'
+ ');',
[],
Variables.CreateTableDeferred.resolve(),
WebSqlError
);
});
});
$.when(Variables.CreateTableDeferred).done(function() {
for (var i=0;i < 10;i++) {
myFunction(i);
};
Variables.Ready.resolve();
function myFunction(i) {
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'INSERT INTO Test(TestID,TestSort) VALUES(?,?)',
[
i
,i+100000
]
,function() {}
,WebSqlError
)
});
};
});
$.when(Variables.Ready).done(function() {
$('#Save').removeAttr('disabled');
});
}(jQuery, window);
!function($, window, undefined) {
var Variables = {};
$(document).on('click','#Save',function() {
var local = {};
local.result = barcode.Scan(dom.TestID.val());
console.log(local.result);
});
var mySuccess = function(transaction, argument) {
var local = {};
for (local.i=0; local.i < argument.rows.length; local.i++) {
local.qry = argument.rows.item(local.i);
Variables.result = local.qry.TestSort;
}
Variables.Return = true;
};
var myError = function(transaction, argument) {
dom.msg.text(argument.message);
Variables.result = '';
Variables.Return = true;
}
var barcode = {};
barcode.Scan = function(argument) {
var local = {};
Variables.result = '';
Variables.Return = false;
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'SELECT * FROM Test WHERE TestID=?'
,[argument]
,mySuccess
,myError
)
});
for (local.I = 0;local.I < 3; local.I++) { // Try a bunch of times.
if (Variables.Return) break; // Gets set in mySuccess and myError
SpinLock(250);
}
return Variables.result;
}
var SpinLock = function(milliseconds) {
var local = {};
local.StartTime = Date.now();
do {
} while (Date.now() < local.StartTime + milliseconds);
}
function WebSqlError(tx,result) {
if (dom.msg.text()) {
dom.msg.append('<br>');
}
dom.msg.append(result.message);
}
}(jQuery, window);

Is there something different about XHR than there is about the executeSql command?
Kind of.
How can jQuery have an async=false argument when it comes to Ajax?
Ajax, or rather XMLHttpRequest, isn't strictly limited to being asynchronous -- though, as the original acronym suggested, it is preferred.
jQuery.ajax()'s async option is tied to the boolean async argument of xhr.open():
void open(
DOMString method,
DOMString url,
optional boolean async, // <---
optional DOMString user,
optional DOMString password
);
The Web SQL Database spec does also define a Synchronous database API. However, it's only available to implementations of the WorkerUtils interface, defined primarily for Web Workers:
window.dbo = openDatabaseSync('POC','1.0','Proof-Of-Concept', 1024*1024);
var results;
window.dbo.transaction(function (trans) {
results = trans.executeSql('...');
});
If the environment running the script hasn't implemented this interface, then you're stuck with the asynchronous API and returning the result will not be feasible. You can't force blocking/waiting of asynchronous tasks for the reason you suspected:
Maybe my spinning isn't giving the callback chance to come back (See code below).