I have a problem in my project.
To describe this issue I have wrote simplified code snippet:
function waitFor(fnReady, fnCallback) {
var check = function() {
if (fnReady()) {
fnCallback();
}
else {
setTimeout(check, 100); // wait another 100ms, and try again
}
};
check();
}
var result = 0;
var flag = true;
function ajaxRequest() {
setTimeout(
function() { flag = false;
console.log('ping');
},3000
);
}
function ajaxRequestHandler() {
setTimeout(
function() { flag = true;
console.log('pong');
}, 200
);
}
for(var i =0;i<10; i++){
waitFor(function() { return flag; }, ajaxRequest);
waitFor(function() { return !flag; }, ajaxRequestHandler);
}
it returns:
ping - 10 times
pong - 10 times
desired result:
ping
3 second timeout
ping
---------------------
ping
3 second timeout
pong
--------------------
.....
Can you help correct my code?
UPDATE
Actual problem:
I have a google map.
I have a lot of places when I should to redraw it.
For application logic very important that If I send
request1
request2
request3
request4
I should handle responses in the this order
handle response of request1
handle response of request2
handle response of request3
handle response of request4
Problem that I don't know order of requests.
In different places of file I see following code rows:
google.maps.event.addListener(searchBox, 'bounds_changed', renderTerminalsOnMapAndFitBounds);
...
$.getJSON('getAllTerminals.json', renderTerminalsOnMapAndFitBounds);
.....
$.getJSON('getAllTerminalsInsideRectangle.json', renderTerminalsOnMapAndFitBounds);
...
$.getJSON('getAllTerminalsInsideCircle.json', renderTerminalsOnMapAndFitBounds);
...
$.getJSON('getBigTerminals.json', renderTerminalsOnMapAndFitBounds);
........
renderTerminalsOnMapAndFitBounds method sends request to server and in succes alternative render result on map. But this event happens very often
Try this pattern
var map = "abcdefghi".split("");
var responses = []; // collect responses
$.ajaxSetup({
beforeSend : function(jqxhr, settings) {
jqxhr.id = Number(settings.data.split(/id=/)[1]); // add `id` to `request`
console.log(settings.data.split(/id=/)[1]);
}
});
var request = function(id, data) {
// append `id` to `id` data
return $.post("/echo/json/", {json:JSON.stringify([data]), id:id})
};
$.each(map, function(k, v) {
setTimeout(function() {
request(k + 1, v)
.done(function(data) {
// do stuff at each response
console.log(data); // note return values
})
.always(function(data, textStatus, jqxhr) {
// do stuff at each response
responses.push([jqxhr.id, data[0]]);
// do stuff when all requests completed , results items in `responses`
if (responses.length === map.length) {
responses.sort(); // sort `responses` based on `id`
// do stuff with `responses`
console.log(responses);
}
});
},1 + Math.random() * 1000) // async
});
jsfiddle http://jsfiddle.net/guest271314/g254bbjg/
my variant:
var index = 0;
// callback function
function tryMe (param1) {
waitFor(function(){return param1 == index},
function(){console.log(param1);
index++;
}
)
}
// callback executer
function callbackTester (callback,i) {
setTimeout( function(){callback(i);}, 20000 - i*1000);
}
// test function
for(var i=0 ; i<10 ; i++){
callbackTester ( tryMe,i );
}
function waitFor(fnReady, fnCallback) {
var check = function() {
if (fnReady()) {
fnCallback();
}
else {
setTimeout(check, 100); // wait another 100ms, and try again
}
};
check();
}
http://jsfiddle.net/x061dx75/17/
I personally would use promises for this, but you've said no promises (not sure why), so here's a generic sequencer algorithm in plain javascript (tested in the jsFiddle linked below):
function sequence(fn) {
// initialize sequence data upon first use
if (typeof sequence.low === "undefined") {
sequence.low = sequence.high = 0;
sequence.results = {};
}
// save id in local variable so we can reference it in the closure from the function below
var id = sequence.high;
// advance to next sequence number
++sequence.high;
// initialize the result value for this sequence callback
sequence.results[id] = {fn: fn, args: [], ready: false, context: null};
return function(/* args */) {
// save args and context and mark it ready
var args = Array.prototype.slice.call(arguments, 0);
// get the results object for this callback and save info in it
var thisResult = sequence.results[id];
thisResult.args = args;
thisResult.context = this;
thisResult.ready = true;
// now process any requests in order that are ready
for (var i = sequence.low; i < sequence.high; i++) {
var result = sequence.results[i];
// if this one is ready, process it
if (result.ready) {
// increment counter past this result
++sequence.low;
// remove this stored result
delete sequence.results[i];
// process this result
result.fn.apply(result.context, result.args);
} else {
// if this one not ready, then nothing to do yet
break;
}
}
};
}
// your usage:
google.maps.event.addListener(searchBox, 'bounds_changed', sequence(renderTerminalsOnMapAndFitBounds));
...
$.getJSON('getAllTerminals.json', sequence(renderTerminalsOnMapAndFitBounds));
.....
$.getJSON('getAllTerminalsInsideRectangle.json', sequence(renderTerminalsOnMapAndFitBounds));
...
$.getJSON('getAllTerminalsInsideCircle.json', sequence(renderTerminalsOnMapAndFitBounds));
...
$.getJSON('getBigTerminals.json', sequence(renderTerminalsOnMapAndFitBounds));
........
Working demo: http://jsfiddle.net/jfriend00/aqugm1fs/
Conceptually, what this does is as follows:
Pass a substitute completion handler in place of the normal completion callback.
This substitute function marks each response with a sequence id and saved the original completion handler.
If a response comes back while another response with a lower sequence id is still pending, then the result is just stored and saved for later.
As each response comes in, it processes as many responses in sequence as are ready
Note: while all the examples you have use the same callback function, this will work with any callback function so it would work with a mix of different types of operations.
I'm trying to split an API request with an offset variable in order to have partial results without waiting the end of the entire request.
Basically I make an API request for the first 100 values, and then I increase it with 100 more till reach the end. The offset is just the starting point.
/*Node.js connector to Context.io API*/
var key = xxxxxx;
var secret = "xxxxxxxxx";
var inbox_id = "xxxxxxxxx";
var end_loop = false;
var offset = 6000;
/*global ContextIO, console*/
var ContextIO = require('contextio');
var ctxioClient = new ContextIO.Client('2.0', 'https://api.context.io', { key: key, secret: secret });
while(end_loop === false) {
contextio_request(offset, function(response){
if (response.body.length < 100) { console.log("This is the end "+response.body.length); end_loop = true; }
else { offset += 100; }
console.log("Partial results processing");
});
};
/* Context.io API request to access all messages for the id inbox */
function contextio_request(offset, callback) {
ctxioClient.accounts(inbox_id).messages().get({body_type: 'text/plain', include_body: 1, limit: 100, offset: offset}, function (err, response) {
"use strict";
if (err) {
return console.log(err);
}
callback(response);
});
};
What I don't understand is why if I change the "while loop" with a "if condition", everything works, but with the "while" it enters in an infinite loop". Also, is it the correct way to make a partial request -> wait for response - > process the response -> follow with next request?
The while loop will call contextio_request() almost indefinitely, because that makes an asynchronous call which wont return immediately.
A better way could be to write a recursive method which calls contextio_request(), inside that method you check if the response body length is less than 100.
Basic logic:
function recursiveMethod = function(offset, partialCallback, completedCallback) {
contextio_request(offset, function(response) {
if (response.body.length < 100) {
completedCallback(...);
} else {
partialCallback(...);
recursiveMethod(offset, partialCallback, completedCallback);
}
});
};
Also, is it the correct way to make a partial request -> wait for response - > process the response -> follow with next request?
I see no reason why not.
I'm using PhantomJS v1.4.1 to load some web pages. I don't have access to their server-side, I just getting links pointing to them. I'm using obsolete version of Phantom because I need to support Adobe Flash on that web pages.
The problem is many web-sites are loading their minor content async and that's why Phantom's onLoadFinished callback (analogue for onLoad in HTML) fired too early when not everything still has loaded. Can anyone suggest how can I wait for full load of a webpage to make, for example, a screenshot with all dynamic content like ads?
Another approach is to just ask PhantomJS to wait for a bit after the page has loaded before doing the render, as per the regular rasterize.js example, but with a longer timeout to allow the JavaScript to finish loading additional resources:
page.open(address, function (status) {
if (status !== 'success') {
console.log('Unable to load the address!');
phantom.exit();
} else {
window.setTimeout(function () {
page.render(output);
phantom.exit();
}, 1000); // Change timeout as required to allow sufficient time
}
});
I would rather periodically check for document.readyState status (https://developer.mozilla.org/en-US/docs/Web/API/document.readyState). Although this approach is a bit clunky, you can be sure that inside onPageReady function you are using fully loaded document.
var page = require("webpage").create(),
url = "http://example.com/index.html";
function onPageReady() {
var htmlContent = page.evaluate(function () {
return document.documentElement.outerHTML;
});
console.log(htmlContent);
phantom.exit();
}
page.open(url, function (status) {
function checkReadyState() {
setTimeout(function () {
var readyState = page.evaluate(function () {
return document.readyState;
});
if ("complete" === readyState) {
onPageReady();
} else {
checkReadyState();
}
});
}
checkReadyState();
});
Additional explanation:
Using nested setTimeout instead of setInterval prevents checkReadyState from "overlapping" and race conditions when its execution is prolonged for some random reasons. setTimeout has a default delay of 4ms (https://stackoverflow.com/a/3580085/1011156) so active polling will not drastically affect program performance.
document.readyState === "complete" means that document is completely loaded with all resources (https://html.spec.whatwg.org/multipage/dom.html#current-document-readiness).
EDIT 2022:
I created this response 8 years ago and I did not use PhantomJS since then. It is very probable it won't work now in some cases. Also now I think it is not possible to create a one-size-fits-all solution to be absolutely sure the page is loaded. This is because some pages may load additional resources after document is ready. For example, there might be some JS code on the website that waits for the document to be ready an then loads some additional assets (after document state changes to ready) - in this case the onPageReady will trigger and after that the page will start loading some more resources again.
I still think the above snipped is a good starting point and may work in most cases, but may also necessary to create a specific solutions to handle specific websites.
You could try a combination of the waitfor and rasterize examples:
/**
* See https://github.com/ariya/phantomjs/blob/master/examples/waitfor.js
*
* Wait until the test condition is true or a timeout occurs. Useful for waiting
* on a server response or for a ui change (fadeIn, etc.) to occur.
*
* #param testFx javascript condition that evaluates to a boolean,
* it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or
* as a callback function.
* #param onReady what to do when testFx condition is fulfilled,
* it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or
* as a callback function.
* #param timeOutMillis the max amount of time to wait. If not specified, 3 sec is used.
*/
function waitFor(testFx, onReady, timeOutMillis) {
var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 3000, //< Default Max Timout is 3s
start = new Date().getTime(),
condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()), //< defensive code
interval = setInterval(function() {
if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
// If not time-out yet and condition not yet fulfilled
condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
} else {
if(!condition) {
// If condition still not fulfilled (timeout but condition is 'false')
console.log("'waitFor()' timeout");
phantom.exit(1);
} else {
// Condition fulfilled (timeout and/or condition is 'true')
console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms.");
typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condition is fulfilled
clearInterval(interval); //< Stop this interval
}
}
}, 250); //< repeat check every 250ms
};
var page = require('webpage').create(), system = require('system'), address, output, size;
if (system.args.length < 3 || system.args.length > 5) {
console.log('Usage: rasterize.js URL filename [paperwidth*paperheight|paperformat] [zoom]');
console.log(' paper (pdf output) examples: "5in*7.5in", "10cm*20cm", "A4", "Letter"');
phantom.exit(1);
} else {
address = system.args[1];
output = system.args[2];
if (system.args.length > 3 && system.args[2].substr(-4) === ".pdf") {
size = system.args[3].split('*');
page.paperSize = size.length === 2 ? {
width : size[0],
height : size[1],
margin : '0px'
} : {
format : system.args[3],
orientation : 'portrait',
margin : {
left : "5mm",
top : "8mm",
right : "5mm",
bottom : "9mm"
}
};
}
if (system.args.length > 4) {
page.zoomFactor = system.args[4];
}
var resources = [];
page.onResourceRequested = function(request) {
resources[request.id] = request.stage;
};
page.onResourceReceived = function(response) {
resources[response.id] = response.stage;
};
page.open(address, function(status) {
if (status !== 'success') {
console.log('Unable to load the address!');
phantom.exit();
} else {
waitFor(function() {
// Check in the page if a specific element is now visible
for ( var i = 1; i < resources.length; ++i) {
if (resources[i] != 'end') {
return false;
}
}
return true;
}, function() {
page.render(output);
phantom.exit();
}, 10000);
}
});
}
Here is a solution that waits for all resource requests to complete. Once complete it will log the page content to the console and generate a screenshot of the rendered page.
Although this solution can serve as a good starting point, I have observed it fail so it's definitely not a complete solution!
I didn't have much luck using document.readyState.
I was influenced by the waitfor.js example found on the phantomjs examples page.
var system = require('system');
var webPage = require('webpage');
var page = webPage.create();
var url = system.args[1];
page.viewportSize = {
width: 1280,
height: 720
};
var requestsArray = [];
page.onResourceRequested = function(requestData, networkRequest) {
requestsArray.push(requestData.id);
};
page.onResourceReceived = function(response) {
var index = requestsArray.indexOf(response.id);
if (index > -1 && response.stage === 'end') {
requestsArray.splice(index, 1);
}
};
page.open(url, function(status) {
var interval = setInterval(function () {
if (requestsArray.length === 0) {
clearInterval(interval);
var content = page.content;
console.log(content);
page.render('yourLoadedPage.png');
phantom.exit();
}
}, 500);
});
Maybe you can use the onResourceRequested and onResourceReceived callbacks to detect asynchronous loading. Here's an example of using those callbacks from their documentation:
var page = require('webpage').create();
page.onResourceRequested = function (request) {
console.log('Request ' + JSON.stringify(request, undefined, 4));
};
page.onResourceReceived = function (response) {
console.log('Receive ' + JSON.stringify(response, undefined, 4));
};
page.open(url);
Also, you can look at examples/netsniff.js for a working example.
In my program, I use some logic to judge if it was onload: watching it's network request, if there was no new request on past 200ms, I treat it onload.
Use this, after onLoadFinish().
function onLoadComplete(page, callback){
var waiting = []; // request id
var interval = 200; //ms time waiting new request
var timer = setTimeout( timeout, interval);
var max_retry = 3; //
var counter_retry = 0;
function timeout(){
if(waiting.length && counter_retry < max_retry){
timer = setTimeout( timeout, interval);
counter_retry++;
return;
}else{
try{
callback(null, page);
}catch(e){}
}
}
//for debug, log time cost
var tlogger = {};
bindEvent(page, 'request', function(req){
waiting.push(req.id);
});
bindEvent(page, 'receive', function (res) {
var cT = res.contentType;
if(!cT){
console.log('[contentType] ', cT, ' [url] ', res.url);
}
if(!cT) return remove(res.id);
if(cT.indexOf('application') * cT.indexOf('text') != 0) return remove(res.id);
if (res.stage === 'start') {
console.log('!!received start: ', res.id);
//console.log( JSON.stringify(res) );
tlogger[res.id] = new Date();
}else if (res.stage === 'end') {
console.log('!!received end: ', res.id, (new Date() - tlogger[res.id]) );
//console.log( JSON.stringify(res) );
remove(res.id);
clearTimeout(timer);
timer = setTimeout(timeout, interval);
}
});
bindEvent(page, 'error', function(err){
remove(err.id);
if(waiting.length === 0){
counter_retry = 0;
}
});
function remove(id){
var i = waiting.indexOf( id );
if(i < 0){
return;
}else{
waiting.splice(i,1);
}
}
function bindEvent(page, evt, cb){
switch(evt){
case 'request':
page.onResourceRequested = cb;
break;
case 'receive':
page.onResourceReceived = cb;
break;
case 'error':
page.onResourceError = cb;
break;
case 'timeout':
page.onResourceTimeout = cb;
break;
}
}
}
I found this approach useful in some cases:
page.onConsoleMessage(function(msg) {
// do something e.g. page.render
});
Than if you own the page put some script inside:
<script>
window.onload = function(){
console.log('page loaded');
}
</script>
I found this solution useful in a NodeJS app.
I use it just in desperate cases because it launches a timeout in order to wait for the full page load.
The second argument is the callback function which is going to be called once the response is ready.
phantom = require('phantom');
var fullLoad = function(anUrl, callbackDone) {
phantom.create(function (ph) {
ph.createPage(function (page) {
page.open(anUrl, function (status) {
if (status !== 'success') {
console.error("pahtom: error opening " + anUrl, status);
ph.exit();
} else {
// timeOut
global.setTimeout(function () {
page.evaluate(function () {
return document.documentElement.innerHTML;
}, function (result) {
ph.exit(); // EXTREMLY IMPORTANT
callbackDone(result); // callback
});
}, 5000);
}
});
});
});
}
var callback = function(htmlBody) {
// do smth with the htmlBody
}
fullLoad('your/url/', callback);
This is an implementation of Supr's answer. Also it uses setTimeout instead of setInterval as Mateusz Charytoniuk suggested.
Phantomjs will exit in 1000ms when there isn't any request or response.
// load the module
var webpage = require('webpage');
// get timestamp
function getTimestamp(){
// or use Date.now()
return new Date().getTime();
}
var lastTimestamp = getTimestamp();
var page = webpage.create();
page.onResourceRequested = function(request) {
// update the timestamp when there is a request
lastTimestamp = getTimestamp();
};
page.onResourceReceived = function(response) {
// update the timestamp when there is a response
lastTimestamp = getTimestamp();
};
page.open(html, function(status) {
if (status !== 'success') {
// exit if it fails to load the page
phantom.exit(1);
}
else{
// do something here
}
});
function checkReadyState() {
setTimeout(function () {
var curentTimestamp = getTimestamp();
if(curentTimestamp-lastTimestamp>1000){
// exit if there isn't request or response in 1000ms
phantom.exit();
}
else{
checkReadyState();
}
}, 100);
}
checkReadyState();
This the code I use:
var system = require('system');
var page = require('webpage').create();
page.open('http://....', function(){
console.log(page.content);
var k = 0;
var loop = setInterval(function(){
var qrcode = page.evaluate(function(s) {
return document.querySelector(s).src;
}, '.qrcode img');
k++;
if (qrcode){
console.log('dataURI:', qrcode);
clearInterval(loop);
phantom.exit();
}
if (k === 50) phantom.exit(); // 10 sec timeout
}, 200);
});
Basically given the fact you're supposed to know that the page is full downloaded when a given element appears on the DOM. So the script is going to wait until this happens.
I use a personnal blend of the phantomjs waitfor.js example.
This is my main.js file:
'use strict';
var wasSuccessful = phantom.injectJs('./lib/waitFor.js');
var page = require('webpage').create();
page.open('http://foo.com', function(status) {
if (status === 'success') {
page.includeJs('https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js', function() {
waitFor(function() {
return page.evaluate(function() {
if ('complete' === document.readyState) {
return true;
}
return false;
});
}, function() {
var fooText = page.evaluate(function() {
return $('#foo').text();
});
phantom.exit();
});
});
} else {
console.log('error');
phantom.exit(1);
}
});
And the lib/waitFor.js file (which is just a copy and paste of the waifFor() function from the phantomjs waitfor.js example):
function waitFor(testFx, onReady, timeOutMillis) {
var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 3000, //< Default Max Timout is 3s
start = new Date().getTime(),
condition = false,
interval = setInterval(function() {
if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
// If not time-out yet and condition not yet fulfilled
condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
} else {
if(!condition) {
// If condition still not fulfilled (timeout but condition is 'false')
console.log("'waitFor()' timeout");
phantom.exit(1);
} else {
// Condition fulfilled (timeout and/or condition is 'true')
// console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms.");
typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condi>
clearInterval(interval); //< Stop this interval
}
}
}, 250); //< repeat check every 250ms
}
This method is not asynchronous but at least am I assured that all the resources were loaded before I try using them.
This is an old question, but since I was looking for full page load but for Spookyjs (that uses casperjs and phantomjs) and didn't find my solution, I made my own script for that, with the same approach as the user deemstone .
What this approach does is, for a given quantity of time, if the page did not receive or started any request it will end the execution.
On casper.js file (if you installed it globally, the path would be something like /usr/local/lib/node_modules/casperjs/modules/casper.js) add the following lines:
At the top of the file with all the global vars:
var waitResponseInterval = 500
var reqResInterval = null
var reqResFinished = false
var resetTimeout = function() {}
Then inside function "createPage(casper)" just after "var page = require('webpage').create();" add the following code:
resetTimeout = function() {
if(reqResInterval)
clearTimeout(reqResInterval)
reqResInterval = setTimeout(function(){
reqResFinished = true
page.onLoadFinished("success")
},waitResponseInterval)
}
resetTimeout()
Then inside "page.onResourceReceived = function onResourceReceived(resource) {" on the first line add:
resetTimeout()
Do the same for "page.onResourceRequested = function onResourceRequested(requestData, request) {"
Finally, on "page.onLoadFinished = function onLoadFinished(status) {" on the first line add:
if(!reqResFinished)
{
return
}
reqResFinished = false
And that's it, hope this one helps someone in trouble like I was. This solution is for casperjs but works directly for Spooky.
Good luck !
this is my solution its worked for me .
page.onConsoleMessage = function(msg, lineNum, sourceId) {
if(msg=='hey lets take screenshot')
{
window.setInterval(function(){
try
{
var sta= page.evaluateJavaScript("function(){ return jQuery.active;}");
if(sta == 0)
{
window.setTimeout(function(){
page.render('test.png');
clearInterval();
phantom.exit();
},1000);
}
}
catch(error)
{
console.log(error);
phantom.exit(1);
}
},1000);
}
};
page.open(address, function (status) {
if (status !== "success") {
console.log('Unable to load url');
phantom.exit();
} else {
page.setContent(page.content.replace('</body>','<script>window.onload = function(){console.log(\'hey lets take screenshot\');}</script></body>'), address);
}
});
Do Mouse move while page is loading should work.
page.sendEvent('click',200, 660);
do { phantom.page.sendEvent('mousemove'); } while (page.loading);
UPDATE
When submitting the form, nothing was returned, so the program stopped. The program did not wait for the page to load as it took a few seconds for the redirect to begin.
telling it to move the mouse until the URL changes to the home page gave the browser as much time as it needed to change. then telling it to wait for the page to finish loading allowed the page to full load before the content was grabbed.
page.evaluate(function () {
document.getElementsByClassName('btn btn-primary btn-block')[0].click();
});
do { phantom.page.sendEvent('mousemove'); } while (page.evaluate(function()
{
return document.location != "https://www.bestwaywholesale.co.uk/";
}));
do { phantom.page.sendEvent('mousemove'); } while (page.loading);