We Keep Coding

Scraping Trophy Data from The Official Playstation Website

Im trying to use PhantomJS to scrape the trophy data from http://my.playstation.com/logged-in/trophies/public-trophies/
The page requires you enter a valid username and then click 'go' and the page will load the data. Ive gotten this to work somewhat, but it never loads the trophy data into the div. Im hoping im missing something ajax related thats causing this?
var fullpagehtml = page.evaluate(function()
{
document.getElementById("trophiesId").value = "<<valid user id>>";
//checkPTrophies(); btn click calls this function
$('#btn_publictrophy').click().delay( 6000 );
console.log("\nWaiting for trophy list to load...");
var trophylist = document.getElementById("trophyTrophyList").innerHtml; // all the data i want ends up inside this div
var counter = 0; //delay andset timeout wont work here so this is the best i coukld think of
while (trophylist == null)
{
//presumably the ajax query should kick in on the page and populate this div, but it doesnt.
trophylist = document.getElementById("trophyTrophyList").innerHtml;
counter ++;
if(counter == 1000000)
{
console.log($('#trophyTrophyList').html());
counter = 0;
}
}
return document.all[0].outerHTML;
});

The delay( 6000 ) does absolutely nothing as the documentation says:
The .delay() method is best for delaying between queued jQuery effects. Because it is limited—it doesn't, for example, offer a way to cancel the delay—.delay() is not a replacement for JavaScript's native setTimeout function, which may be more appropriate for certain use cases.
To wait you have to do this outside of the page context (busy waiting doesn't work in JavaScript because it is single threaded):
page.evaluate(function() {
document.getElementById("trophiesId").value = "<<valid user id>>";
//checkPTrophies(); btn click calls this function
$('#btn_publictrophy').click();
});
console.log("\nWaiting for trophy list to load...");
setTimeout(function(){
var fullpagehtml = page.evaluate(function() {
var trophylist = document.getElementById("trophyTrophyList").innerHTML;
return trophylist;
});
}, 20000);
You also might want to use waitFor to wait until #trophyTrophyList is populated instead of using setTimeout:
waitFor(function(){
return page.evaluate(function(){
var e = document.getElementById("trophyTrophyList");
return e && e.innerHTML;
});
}, function(){
// TODO: get trophies
});
This won't get you far, because just because #trophyTrophyList is loaded, doesn't mean that the descendent elements are already in the DOM. You have to find some selector which signalizes that the page is sufficiently loaded for example by waiting until a .trophy-image exists in the page. It works for me with a 20 second timeout of the waitFor function.
waitFor(function(){
return page.evaluate(function(){
var e = document.querySelector("#trophyTrophyList .trophy-image");
return e;
});
}, function(){
setTimeout(function(){
var trophiesDiv = page.evaluate(function(){
return document.getElementById("trophyTrophyList").innerHTML;
});
console.log(trophiesDiv);
}, 1000); // wait a little longer
}, 20000);
Don't forget that you need page.evaluate to actually access the DOM. Btw, it is innerHTML not innerHtml.

How would you cycle through each image and check it for an error in a casperjs script?

How would you cycle through every image on a given page and check to see if it was loaded or errored out?
The following are not seeming to work in the phantomjs/casperjs setup yet cycling through and grabbing the src is working.
.load()
and
.error()
If the above does work, could someone show a proper usage that would work in a casperjs script?
The code I used looked similar to the following:
var call_to_page = this.evaluate(function() {
var fail_amount = 0;
var pass_amount = 0;
var pass_img_src = [];
var fail_img_src = [];
var img_src = [];
$("img").each(function(){
$(this).load(function(){
pass_amount++;
pass_img_src.push($(this).attr("src"));
}).error(function(){
fail_amount++;
fail_img_src.push($(this).attr("src"));
});
img_src.push($(this).attr("src"));
});
return [img_src, fail_amount, fail_img_src, pass_amount, pass_img_src];
});
Any help as to why the above code doesnt work for me would be great. The page is properly being arrived upon and I am able to mess with the dom, just not .load or .error. Im thinking its due to the images being done loaded, so Im still looking for alternatives.

CasperJS provides the resourceExists function which can be used to check if a particular ressource was loaded. The ressource is passed into a function, so custom constraints can be raised.
casper.then(function(){
var fail_amount = 0;
var pass_amount = 0;
var pass_img_src = [];
var fail_img_src = [];
var elements = this.getElementsInfo("img");
elements.forEach(function(img){
if (img && img.attributes && casper.resourceExists(function(resource){
return resource.url.match(img.attributes.src) &&
resource.status >= 200 &&
resource.status < 400;
})) {
pass_amount++;
pass_img_src.push(img.attributes.src);
} else {
fail_amount++;
fail_img_src.push(img.attributes.src);
}
});
this.echo(JSON.stringify([fail_amount, fail_img_src, pass_amount, pass_img_src], undefined, 4));
});
This can be done after the page is loaded. So there is no need to preemptively add some code into the page context.
In turn, the problem with you code may be that the callbacks never fire because the images are already loaded of already timed out. So there is no new information.
If you're not sure what kind of errors are counted, you can use a custom resources detection for all available types or errors.
var resources = []; // a resource contains at least 'url', 'status'
casper.on("resource.received", function(resource){
if (resource.stage == "end") {
if (resource.status < 200 || resource.status >= 400) {
resource.errorCode = resource.status;
resource.errorString = resource.statusText;
}
resources.push(resource);
}
});
casper.on("resource.timeout", function(request){
request.status = -1;
resources.push(request);
});
casper.on("resource.error", function(resourceError){
resourceError.status = -2;
resources.push(resourceError);
});
function resourceExists(url){
return resources.filter(function(res){
return res.url.indexOf(url) !== -1;
}).length > 0;
}
casper.start(url, function(){
var elements = this.getElementsInfo("img");
elements.forEach(function(img){
if (img && img.attributes && resourceExists(img.attributes.src) && !resourceExists(img.attributes.src).errorCode) {
// pass
} else {
// fail
}
});
});

I don't have much experience with caperjs, in my observation I identified below points
Note:
jQuery .load( handler ) and .error( handler ) both were deprecated from verson 1.8.
If you're using jQuery 1.8+ then attaching load and error events to img(tags) does nothing.
jQuery Ajax module also has a method named $.load() is shortcut form of $.get(). Which one is fired depends on the set of arguments passed.
Here are Caveats of the load event when used with images from jQuery Docs
A common challenge developers attempt to solve using the .load() shortcut is to execute a function when an image (or collection of images) have completely loaded. There are several known caveats with this that should be noted. These are:
It doesn't work consistently nor reliably cross-browser
It doesn't fire correctly in WebKit if the image src is set to the same src as before
It doesn't correctly bubble up the DOM tree
Can cease to fire for images that already live in the browser's cache
So if you've version 1.8+ of jQuery the below block does nothing.
$(this).load(function(){
pass_amount++;
pass_img_src.push($(this).attr("src"));
}).error(function(){
fail_amount++;
fail_img_src.push($(this).attr("src"));
});
As a result this return [img_src, fail_amount, fail_img_src, pass_amount, pass_img_src]; statement will give us only img_src[with number of imgs as length] array filled with srcs of imgs in page. and other elements fail_amount, fail_img_src, pass_amount, pass_img_src will have same defaults all the time.
In the case of jQuery 1.8 below load and error events attachment with jQuery were meaningful(in your case these events were attached after they were loaded on page, so they won't show any effect with load and error callbacks), but the time where we attach events matters. We should attach these before the img tags or place the events in tag level(as attributes onload & onerror) and definitions of function handlers script should keep before any img tag or in very beginning of the body or in head
There're ways to figure out some are here:
use open-source plug-in like (Use imagesLoaded](https://github.com/desandro/imagesloaded)
Can use ajax call to find out whether the img.src.url were good or not?
Dimension based check like below function IsImageRendered
below I've but its old one not sure the browser support at this time. i recommend to go with above plug in if you can use it
var call_to_page = this.evaluate(function () {
function isImageRendered(img) {
// with 'naturalWidth' and 'naturalHeight' to get true size of the image.
// before we tried width>0 && height>0
if (typeof img.naturalWidth !== "undefined" && img.naturalWidth === 0) {
return false;
}
//good
return true;
}
var pass_img_src = [];
var fail_img_src = [];
var img_src = [];
$("img").each(function () {
var $img = $(this), $srcUrl = $img.attr("src");
img_src.push($srcUrl);
if (!$srcUrl)
fail_img_src.push($srcUrl);
else {
if (isImageRendered($img.get(0))) {
pass_img_src.push($srcUrl);
}
else {
fail_img_src.push($srcUrl);
}
}
});
var fail_count = fail_img_src.length,
pass_count = pass_img_src.length;
return [img_src, fail_count, fail_img_src, pass_count, pass_img_src];
});

Infinite scroll fired twice on refresh from the bottom of page

I have an infinite scroll set up with the following piece of code.
$(window).scroll(function () {
if ($(window).scrollTop() >= $("#home_content").height() - $(window).height()) {
if (isLastPage) {
foo();
} else {
bar(); // JQuery AJAX call
}
}
});
This is inside document.ready();
The ajax call doesn't happen when the server sends a flag for the last page. This works fine in a normal scenario. But when I press F5(Refresh) from the bottom of the page, two simultaneous scroll events are fired,and it bypasses the flag (as the second call happens even before the flag is set) and duplicate data is loaded.
The only thing i know is it happens at the end of document.ready() function. Anyone, any idea??
Thanks in advance.
EDIT
There is no much relevant code other than this.
And this happens only in FF 17.
In IE 9 when I do a fast scroll down, same scroll is fired twice

You can use this debounce routine for all sort of event calls. Clean and reusable.
// action takes place here.
function infinite_scrolling(){
if ($(window).scrollTop() >= $("#home_content").height() - $(window).height()) {
if (isLastPage) {
foo();
} else {
bar(); // JQuery AJAX call
}
}
}
// debounce multiple requests
var _scheduledRequest = null;
function infinite_scroll_debouncer(callback_to_run, debounce_time) {
debounce_time = typeof debounce_time !== 'undefined' ? debounce_time : 800;
if (_scheduledRequest) {
clearTimeout(_scheduledRequest);
}
_scheduledRequest = setTimeout(callback_to_run, debounce_time);
}
// usage
$(document).ready(function() {
$(window).scroll(function() {
infinite_scroll_debouncer(infinite_scrolling, 1000);
});
});

This is just a workaround as we cannot see your complete code, but maybe thats can help:
var timeout;
$(window).scroll(function(){
clearTimeout(timeout);
timeout = setTimeout(function(){
if ($(window).scrollTop() >= $("#home_content").height() - $(window).height()){
if (isLastPage){
foo();
}else{
bar();//AJAX call
}
}
},0);
});

Using Javascript to detect the bottom of the window and ignoring all events when a request is loading

I have an anonymous function to detect the user has scrolled to the bottom of the window. Inside of the anonymous function, I have a call to a database that takes a while to complete.
var allowing_more = 1;
$(window).scroll(function() {
if (allowing_more == 1){
if ($(window).scrollTop() + $(window).height() == $(document).height()) {
allowing_more = 0;
//query
allowing_more = 1;
}
}
});
In this time, if the user scrolls to the bottom of the window again, it seems a queue is made holding the occurences the user scrolled to the bottom of the window while the query was loading. Upon completing of the query, these occurences are then executed.
I have a boolean statement to detect if the anonymous function is accepting more query requests but this seems to be ignored.
Is there some sort of way to ignore an anonymous function temporarily and re-enable it?

All you need is you have to stop querying your database until the previous request completes is this correct?
if this is correct
var chkFlg=0;
$(window).scroll(function() {
if ($(window).scrollTop() + $(window).height() == $(document).height()) {
if(chkFlg===1){
//query your database after you get your result from db assign 1 to chkFlg
chkFlg = 1;
}
}
});

var flag = 0;
$(window).scroll(function() {
if(flag===0){
if ($(window).scrollTop() + $(window).height() == $(document).height()) {
//query
flag = 1;
}
flag = 0;
}
});
This should work

Just follow this kind of pattern and your boolean condition should work.
safeToQuery = true; //
$(window).scroll(function() {
...
if (safeToQuery) {
safeToQuery = false;
//send query request
$.ajax({...}).done(function(data,etc){
//do whatever with the results
}).always(function (){
//regardless of whether or not the last request resulted
//in an error, make it safe to query again
safeToQuery = true;
});
}
});
Note: safeToQuery doesn't have to be a global, you could declare it as a local variable in an Immediately-Invoked Function Expression (IIFE) or just make it a property of some object that lives at least as long as you need to keep the scroll handler active.

Simple client-side framework/pattern to simplify doing async calls?

We're currently not using any serious client side framework besides jQuery (and jQuery.ui + validation + form wizard plugins).
A problem that surfaces a few times in our code is this:
We have a button that initiates an Ajax call to the server.
While the call is taking place, we display a "loading" icon with some text
If the server returns a result too quickly (e.g. < 200 ms), we "sleep" for 200 millis (using setTimeout()), to prevent flickering of the waiting icon & text.
After max(the call returns, a minimal timeout), we clear the loading icon & text.
We then either display an error text, if there was some problem in the ajax call (the server doesn't return 500, but a custom json that has an "error message" property. In fact, sometimes we have such a property in the response per form field ... and we then match errors to form fields ... but I digress).
In case of success, we do ... something (depends on the situation).
I'm trying to minimize code reuse, and either write or reuse a pattern / piece of code / framework that does this. While I probably won't start using an entire new heavy-duty framework just for this use case, I would still like to know what my options are ... perhaps such a client-side framework would be good for other things as well. If there's a lightweight framework that doesn't require me to turn all my code upside down, and I could use just on specific cases, then we might actually use it instead of reinventing the wheel.
I just recently heard about Ember.js - is it a good fit for solving this problem? How would you solve it?

$(function(){
var buttonSelector = "#button";
$('body').on({'click': function(evt){
var $button = $(this);
$button.toggleClass('loading');
var time = new Date();
$.get('some/ajax').then(function(data,text,jqXhr){
// typical guess at load work
$button.empty();
$(data).wrap($button);
}).fail(function(data,text,jqXhr){
alert("failed");
}).done(function(data,text,jqXhr){
var elapsed = new Date();
if((elapsed - time) < 200){
alert("to short, wait");
}
$button.toggleClass('loading');
});
}},buttonSelector,null);
});

Just wrap the $.ajax in your own function. that way you can implement your own queing etc. I would suggest to do a jquery component for this. It can get pretty powerful, for example you can also pass http headers etc.
Regarding frameworks it depends on your requirements.
For example, you may consider Kendo UI, it has good framework for creating data sources:
http://demos.kendoui.com/web/datasource/index.html.

Working Sample Code (well, almost)
I was going for something along the lines of #DefyGravity's answer anyway - his idea is good, but is still pseudo-code/not fully complete. Here is my working code (almost working demo, up to the Ajax URL itself, and UI tweaks)
The code & usage example:
jQuery.fn.disable = function() {
$(this).attr("disabled", "disabled");
$(this).removeClass("enabled");
// Special handling of jquery-ui buttons: http://stackoverflow.com/questions/3646408/how-can-i-disable-a-button-on-a-jquery-ui-dialog
$(this).filter("button").button({disabled: true});
};
jQuery.fn.enable = function() {
$(this).removeAttr("disabled");
$(this).addClass("enabled");
// Special handling of jquery-ui buttons: http://stackoverflow.com/questions/3646408/how-can-i-disable-a-button-on-a-jquery-ui-dialog
$(this).filter("button").button({disabled: false});
};
function AjaxCallbackWaiter(ajaxUrl, button, notificationArea, loadingMessage, errorMessage, inSuccessHandler, inFailureHandler) {
// Every request that takes less than this, will be intentionally delayed to prevent a flickering effect
// http://ripper234.com/p/sometimes-a-little-sleep-is-ok/
var minimalRequestTime = 800;
var loadingIconUrl = 'http://loadinfo.net/images/preview/11_cyrcle_one_24.gif?1200916238';
var loadingImageContent = $("<img class='loading-image small' src='" + loadingIconUrl + "'/><span class='loading-text'>" + loadingMessage + "</span>");
var errorContentTemplate = $("<span class='error ajax-errors'></span>");
var requestSentTime = null;
button.click(clickHandler);
function displayLoadingMessage() {
clearNotificationArea();
notificationArea.html(loadingImageContent);
}
function clearNotificationArea() {
notificationArea.html("");
}
function displayError(message) {
var errorContent = errorContentTemplate.clone(errorContentTemplate).html(message);
notificationArea.html(errorContent);
}
function ajaxHandler(result) {
var requestReceivedTime = new Date().getTime();
var timeElapsed = requestReceivedTime - requestSentTime;
// Reset requestSentTime, preparing it for the next request
requestSentTime = null;
var sleepTime = Math.max(0, minimalRequestTime - timeElapsed);
function action() {
clearNotificationArea();
button.enable();
if (result) {
inSuccessHandler();
} else {
displayError(errorMessage);
inFailureHandler();
}
}
if (sleepTime <= 0) {
action();
} else {
setTimeout(action, sleepTime);
}
}
function failureHandler() {
}
function clickHandler(){
if (requestSentTime !== null) {
logError("Bad state, expected null");
}
requestSentTime = new Date().getTime();
displayLoadingMessage();
button.disable();
$.get(ajaxUrl, 'json').then(ajaxHandler, failureHandler);
}
}
// Usage:
var ajaxUrl = 'FILL IN YOUR OWN URL HERE';
var button = $("#clickme");
var notificationArea = $(".ajax-notification-area");
var waitingMessage = "Doing Stuff";
var errorMessage = "Not Good<br/> Please try again";
$(document).ready(function(){
new AjaxCallbackWaiter(
ajaxUrl,
button,
notificationArea,
waitingMessage,
errorMessage,
function(){
alert("All is well with the world");
},
function(){
alert("Not good - winter is coming");
});
});