We Keep Coding

javascript, continue downloads in background and restrict 1 download at a time

I'm developing an Ionic App using Cordova File Transfer Plugging to download set of images into the device. Currently it downloads images successfully and I need to restrict 1 download job at a time. Following is the code :
$scope.activeDownload = false;
// Download the current magazine
$scope.downloadMagazine = function() {
if($rootScope.user.user_id == undefined) {
$scope.showLoginAlert = function() {
var alertPopup = $ionicPopup.alert({
title: 'Oops!',
template: "Your must login to download magazines"
});
};
$scope.showLoginAlert();
return;
}
document.addEventListener('deviceready', function () {
var dirName = $rootScope.currentIssue.slug+'_VOL_'+$rootScope.currentIssue.vol+'_ISU_'+$rootScope.currentIssue.issue;
// First create the directory
$cordovaFile.createDir(cordova.file.dataDirectory, dirName, false)
.then(function (success) {
var count = 1;
$scope.loadedCount = 0;
$ionicLoading.show({template : "<progress max=\"100\" value=\"0\" id=\"dw-prog\"></progress><p> Downloading pages...</p><p>Please wait...</p> <button ng-controller=\"magazineIssueCtrl\" ng-click=\"downloadBackground()\" class=\"button button-full button-positive\">Continue in Background</button>"});
angular.forEach($scope.pages, function(value, key) {
function wait() {
if($scope.proceed == false) {
window.setTimeout(wait,50);
}
else {
var imgName = count+".png";
$scope.saveImage(dirName,value.link,imgName); // Then save images one by one to the created directory.
count++;
}
};
wait();
});
}, function (error) {
// Directory already exists means that the magazine is already downloaded.
$scope.showDownloadedAlert = function() {
var alertPopup = $ionicPopup.alert({
title: 'Why worry!',
template: "Your have already downloaded this magazine. You can view it on downloads"
});
};
$scope.showDownloadedAlert();
});
}, false);
};
// Save a image file in a given directory
$scope.saveImage = function(dir,imgUrl,imageName) {
$scope.proceed = false;
var url = imgUrl;
var targetPath = cordova.file.dataDirectory+ dir+"/" + imageName;
var trustHosts = true;
var options = {};
// Download the image using cordovafiletransfer plugin
$cordovaFileTransfer.download(url, targetPath, options, trustHosts)
.then(function(result) {
$scope.proceed = true;
$scope.loadedCount ++;
document.getElementById("dw-prog").value = ($scope.loadedCount / $scope.pages.length )*100;
if($scope.loadedCount == $scope.pages.length) {
$scope.activeDownload = false;
$ionicLoading.hide();
$scope.showDownloadSuccessAlert = function() {
var alertPopup = $ionicPopup.alert({
title: 'Success!',
template: "Your magazine successfully downloaded. You can view it on Downloads!"
});
};
$scope.showDownloadSuccessAlert();
}
}, function(err) {
//alert(JSON.stringify(err));
}, function (progress) {
});
};
// Continue download in background
$scope.downloadBackground = function () {
$scope.activeDownload = true;
$ionicLoading.hide();
$scope.showAlert = function() {
var alertPopup = $ionicPopup.alert({
title: 'Sent to Background!',
template: "You can view it on downloads tab"
});
};
$scope.showAlert();
$rootScope.downloadInBackground.dirName = $rootScope.currentIssue.slug+'_VOL_'+$rootScope.currentIssue.vol+'_ISU_'+$rootScope.currentIssue.issue;
};
Here everything happens as expected but I need the $scope.activeDownload variable to be true when a download is sent to background so that I can refer to that variable before starting another download job. But the problem here is that variable seems to be set to false always. Could you please help me to identify the problem here?

Wait for a URL to download all the contents of a webpage

I have to download HTML Content of a URL. The problem is that the URL takes some time to load , so I have to wait/ timeout for sometime ( ~10 - 15 secs) before logging the content. To achieve this, I tried 2 approaches, but all of them fail to produce the desired result.
First approach is the use of setTimeOut:
var page = require('webpage').create()
page.open(url, function (status) {
if (status !== 'success') {
console.log('Unable to load the address!');
phantom.exit();
} else {
window.setTimeout(function () {
console.log(page.content);
phantom.exit();
}, 10000);
}
});
But setTimeout fails to set the specified timeout. No matter what value I put as Timeout , it times out after a fixed amount of time which is less than the page load time.
The second approach was the use of OnLoadFinished:
var page = new WebPage(), testindex = 0, loadInProgress = false;
page.onConsoleMessage = function(msg) {
console.log(msg)
};
page.onLoadStarted = function() {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log("load finished");
};
var steps = [
function() {
page.open("url");
},
function() {
console.log(page.content);
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 5000);
In this approach, OnLoadFinished fires before the full page is loaded.
I am new to phantomJS , so the above two solutions are also from stack overflow. Is there something I am missing that is particular to my case ? Is there any other way to achieve the same result? ( I tried Waitfor construct also, but with no success).

Ok, you problem is to load Content after some timeout. If you are looking for DOM element, you have to use known to you WaitFor function. But if you just want to get page content after timeout, it is so much easier. So lets start.
var page = require("webpage").create();
var address = "http://someadress.com/somepath/somearticle";
var timeout = 10*1000;
page.open(address);
function getContent() {
return page.evaluate(function() {
return document.body.innerHTML;
});
}
page.onLoadFinished = function () {
setTimeout(function() {
console.log(getContent());
}, timeout);
}
Note! If you are waiting for large content in HTML body, use setInterval function, to wait for document.body.innerHTML more than you want.

JavaScript RangeError - Maximum call stack size exceeded when using jQuery.post

When using jQuery's .post() function to submit my form data, I'm getting an Uncaught RangeError: Maximum call stack size exceeded.
I know this generally means recursion but I can't see where the recursion is happening.
I've put the post request into a function ( submitRequest() ) so I can submit data from 2 different points in the code. It originally resided inside the submit event and at that point worked perfectly. The error came as soon as I moved it outside.
Any ideas?
JavaScript code (with commented logs so you can see the flow) :
$(document).ready(function() {
var downloadLink = '',
downloadName = '',
details,
detailsSaved = false;
$('.js--download').click(function(event) {
var self = $(this);
event.preventDefault();
downloadLink = self.data('filePath'); // Store clicked download link
downloadName = self.closest('.brochure').find('.brochure__name').html().replace('<br>', ' ');
if (!detailsSaved) {
$('#brochure-section').addClass('hide');
$('#capture-section').removeClass('hide');
$('html, body').animate({
scrollTop: $("#capture-section").offset().top
}, 500);
} else {
submitRequest();
}
return false;
});
$(".submit-btn").click(function(event) {
var antiSpam = $('input[name=url]').val();
if (antiSpam != "") {
outputResultText('Error - Please leave the spam prevention field blank', 'error');
proceed = false;
event.preventDefault();
return false;
}
var name = $('input[name=name]').val(),
company = $('input[name=company]').val(),
email = $('input[name=email]').val(),
phone = $('input[name=phone]').val(),
proceed = true;
if(name==""){
$('input[name=name]').addClass("error");
proceed = false;
}
if(phone==""){
$('input[name=phone]').addClass("error");
proceed = false;
}
if(email==""){
$('input[name=email]').addClass("error");
proceed = false;
}
if(!proceed) {
outputResultText('Please check all required fields', 'error');
event.preventDefault();
return false;
}
event.preventDefault();
if(proceed) {
console.log('About to request'); // Logged out
submitRequest();
}
return false;
});
//reset previously set border colors and hide all message on .keyup()
$("input, textarea").keyup(function() {
$(this).removeClass("error");
$(".form-result").fadeOut(100);
});
function submitRequest () {
console.log('Start submitRequest'); // Logged out
if (!detailsSaved) {
console.log('Details are NOT saved');
post_data = {
'name': name,
'company': company,
'phone': phone,
'email': email,
'brochure': downloadName,
'brochure_url': downloadLink
};
details = post_data;
} else {
console.log('Details are saved');
post_data = details;
post_data['brochure'] = downloadName;
post_data['brochure_url'] = downloadLink;
}
console.log('Posting data'); // Logged out
// CRASH: Uncaught RangeError: Maximum call stack size exceeded
$.post(bcf_local_args['post_url'], post_data, function(response){
console.log('Response received');
if(response.type != 'error') {
if (detailsSaved) {
outputAlert("Thank you for your request to receive our <strong>'"+downloadName+"'</strong> brochure.<br>We'll send you a copy soon to <strong>'"+email+"'</strong>, so please check your inbox.<br>Want it sent to a different email? Simply refresh the page and try again.");
} else {
//reset values in all input fields
$('#brochure-capture-form input').val('');
$('#brochure-capture-form textarea').val('');
$('#capture-section').addClass('hide');
$('#brochure-section').removeClass('hide');
outputAlert("Thank you for your request to receive our <strong>'"+downloadName+"'</strong> brochure.<br>We'll send you a copy soon to <strong>'"+email+"'</strong>, so please check your inbox.");
}
if (!detailsSaved) {
detailsSaved = true;
}
$('html, body').animate({
scrollTop: $(".brochure__alert").offset().top
}, 500);
} else {
outputResultText(response.text, response.type);
}
}, 'json');
}
function outputResultText (text, status) {
var output = '';
if(status == 'error') {
output = '<div class="error">'+text+'</div>';
} else {
output = '<div class="success">'+text+'</div>';
}
$(".form-result").hide().html(output).fadeIn(250);
}
function outputAlert (text) {
var output = '<div>'+text+'</div>';
$('.brochure__alert').hide().removeClass('hide').html(output).slideDown(250);
setTimeout( function() {
$('.brochure__alert').slideUp(250);
}, 6500);
}
// function accessStorage(action, dataKey, dataValue) {
// if(typeof(Storage) === "undefined") {
// // No support for localStorage/sessionStorage.
// return false;
// }
// if (action == 'store') {
// localStorage.setItem(dataKey, dataValue);
// } else if (action == 'retrieve') {
// return localStorage.getItem(dataKey);
// }
// }
});

I don't know if you already found a solution but I was having the "same" problem.
In my code I had this function where I was calling after an upload of images, and I was passing the images name as paramaters along with others parameters required to my POST data.
After some research I found out that browsers has some limitations on passing parameters so the problem wasn't AT $.post but in my function calling.
I don't know the technical term but I was 'overusing the stack parameters'.
So maybe your problem isn't at your $.post either, but something else exceeding the stack.
Hope this helps.
[]'s

How can I wait for the page to be ready in PhantomJS?

I'm using PhantomJS to log into a site an do something. The site used OAuth for logging in. Clicking on the "Login" button on the, takes you to the OAuth service. There you enter your credentials and clicking "Submit", you get redirected back to the original site. My script works fine but relies on timeouts which doesn't seem too robust.
How can I rewrite this code so that instead of using setTimeout, I can wait until the page is ready. I often see errors that the page isnt' ready and therefore jQuery isn't initialized.
I'm not too good with Javascript so an example would be helpful. This is what I've hacked together after a ton of Googling. Here's my code:
var page = require('webpage').create();
var system = require('system');
page.settings.resourceTimeout = 10000;
page.onResourceTimeout = function(e) {
console.log("Timed out loading resource " + e.url);
};
page.open('https://mysite.com/login', function(status) {
if (status !== 'success') {
console.log('Error opening url');
phantom.exit(1);
} else {
setTimeout(function() {
console.log('Successfully loaded page');
page.evaluate(function() {
$("#submit-field").click(); //Clicking the login button
});
console.log('Clicked login with OAuth button');
setTimeout(function() {
console.log('Addding the credentials');
page.evaluate(function() {
document.getElementById("username").value = 'user#example.com';
document.getElementById("password").value = 'P#ssw0rd';
document.getElementById("Login").click();
});
console.log('Clicked login button');
setTimeout(function() {
//Inject some jQuery into the page and invoke that here
console.log('Clicked the export button');
}, 15000);
}, 15000);
});
}
});

It seems that the only way to do this was to use callbacks from the DOM to PhantomJS.
var page = require('webpage').create();
var system = require('system');
page.onInitialized = function() {
page.onCallback = function(data) {
console.log('Main page is loaded and ready');
//Do whatever here
};
page.evaluate(function() {
document.addEventListener('DOMContentLoaded', function() {
window.callPhantom();
}, false);
console.log("Added listener to wait for page ready");
});
};
page.open('https://www.google.com', function(status) {});

An alternate method would be to extend the phantomjs waitfor.js example.
I use this personnal blend of method.
This is my main.js file:
'use strict';
var wasSuccessful = phantom.injectJs('./lib/waitFor.js');
var page = require('webpage').create();
page.open('http://foo.com', function(status) {
if (status === 'success') {
page.includeJs('https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js', function() {
waitFor(function() {
return page.evaluate(function() {
if ('complete' === document.readyState) {
return true;
}
return false;
});
}, function() {
var fooText = page.evaluate(function() {
return $('#foo').text();
});
phantom.exit();
});
});
} else {
console.log('error');
phantom.exit(1);
}
});
And the lib/waitFor.js file (which is just a copy and paste of the waifFor() function from the phantomjs waitfor.js example):
function waitFor(testFx, onReady, timeOutMillis) {
var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 3000, //< Default Max Timout is 3s
start = new Date().getTime(),
condition = false,
interval = setInterval(function() {
if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
// If not time-out yet and condition not yet fulfilled
condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
} else {
if(!condition) {
// If condition still not fulfilled (timeout but condition is 'false')
console.log("'waitFor()' timeout");
phantom.exit(1);
} else {
// Condition fulfilled (timeout and/or condition is 'true')
// console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms.");
typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condi>
clearInterval(interval); //< Stop this interval
}
}
}, 250); //< repeat check every 250ms
}
This method is not asynchronous but at least am I assured that all the resources were loaded before I try using them.

How to submit a form using PhantomJS

I'm trying to use phantomJS (what an awesome tool btw!) to submit a form for a page that I have login credentials for, and then output the content of the destination page to stdout. I'm able to access the form and set its values successfully using phantom, but I'm not quite sure what the right syntax is to submit the form and output the content of the subsequent page. What I have so far is:
var page = new WebPage();
var url = phantom.args[0];
page.open(url, function (status) {
if (status !== 'success') {
console.log('Unable to access network');
} else {
console.log(page.evaluate(function () {
var arr = document.getElementsByClassName("login-form");
var i;
for (i=0; i < arr.length; i++) {
if (arr[i].getAttribute('method') == "POST") {
arr[i].elements["email"].value="mylogin#somedomain.example";
arr[i].elements["password"].value="mypassword";
// This part doesn't seem to work. It returns the content
// of the current page, not the content of the page after
// the submit has been executed. Am I correctly instrumenting
// the submit in Phantom?
arr[i].submit();
return document.querySelectorAll('html')[0].outerHTML;
}
}
return "failed :-(";
}));
}
phantom.exit();
}

I figured it out. Basically it's an async issue. You can't just submit and expect to render the subsequent page immediately. You have to wait until the onLoad event for the next page is triggered. My code is below:
var page = new WebPage(), testindex = 0, loadInProgress = false;
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.onLoadStarted = function() {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log("load finished");
};
var steps = [
function() {
//Load Login Page
page.open("https://website.example/theformpage/");
},
function() {
//Enter Credentials
page.evaluate(function() {
var arr = document.getElementsByClassName("login-form");
var i;
for (i=0; i < arr.length; i++) {
if (arr[i].getAttribute('method') == "POST") {
arr[i].elements["email"].value="mylogin";
arr[i].elements["password"].value="mypassword";
return;
}
}
});
},
function() {
//Login
page.evaluate(function() {
var arr = document.getElementsByClassName("login-form");
var i;
for (i=0; i < arr.length; i++) {
if (arr[i].getAttribute('method') == "POST") {
arr[i].submit();
return;
}
}
});
},
function() {
// Output content of page to stdout after form has been submitted
page.evaluate(function() {
console.log(document.querySelectorAll('html')[0].outerHTML);
});
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 50);

Also, CasperJS provides a nice high-level interface for navigation in PhantomJS, including clicking on links and filling out forms.
CasperJS
Updated to add July 28, 2015 article comparing PhantomJS and CasperJS.
(Thanks to commenter Mr. M!)

Sending raw POST requests can be sometimes more convenient. Below you can see post.js original example from PhantomJS
// Example using HTTP POST operation
var page = require('webpage').create(),
server = 'http://posttestserver.example/post.php?dump',
data = 'universe=expanding&answer=42';
page.open(server, 'post', data, function (status) {
if (status !== 'success') {
console.log('Unable to post!');
} else {
console.log(page.content);
}
phantom.exit();
});

As it was mentioned above CasperJS is the best tool to fill and send forms.
Simplest possible example of how to fill & submit form using fill() function:
casper.start("http://example.com/login", function() {
//searches and fills the form with id="loginForm"
this.fill('form#loginForm', {
'login': 'admin',
'password': '12345678'
}, true);
this.evaluate(function(){
//trigger click event on submit button
document.querySelector('input[type="submit"]').click();
});
});