PhantomJS Memory Exhausted error when using JavaScript - javascript

I've searched all over the web for a solution with no luck so far. I know this is a documented issue, but I am only trying to load a single web page and all the responses have been "this happens trying to do tests on 200 pages".
I have tried:
page.settings.loadImages = true;
page.settings.loadplugins = false;
I added the ``page.close()` even though like I said, it's only with one URL right now.
I've tried setting loadImages to false as well. It works fine when I remove my one line of JavaScript (for example replacing it with document.title but as soon as that line is there, the memory usage rockets to 1000MB and then it quits.
My only goal is to check if a certain element exists on the page. How can I resolve this issue? None of the solutions I've found online are working for me.
The code in question:
var system = require('system');
var page = require('webpage').create();
page.settings.userAgent = 'Chrome/52.0.2743.116';
var url = 'http://www.example.com/';
var ending = system.args[1];
ending = encodeURIComponent(ending);
url = url + ending;
page.settings.loadImages = false;
page.settings.loadplugins = false;
page.viewportSize = { width: 1680, height: 1050 };
page.open(url, function(status) {
setTimeout(function(){
var elem = page.evaluate(function() {
return $(".primary-info")[0];
});
if(!elem){
console.log('no image found');
page.render(ending+'.png');
}
page.close();
phantom.exit();
}, 5000);
});

Don't just return the whole object, there could be quite a lot of its attributes. Only take what is necessary:
var elem = page.evaluate(function() {
var data = {};
var item = $(".primary-info")[0];
data.title = item.value;
data.color = item.style.color;
return data;
});

Related

chrome dev tools crashed during initialization call

The code below crashed on this line, document.body.appendChild(script);, where the new script gets added to the body element.The page crashed and browser's debugger tool showed this message DevTools was disconnected from the page. Once page is reloaded, DevTools will automatically reconnect. I'm not sure what is the error that caused the debugger tool to crash since the same process of adding and removing script to get random quote worked with #btn-quote click event. I've also tried to have the init function call when the page is loaded by replacing appController.init(); with window.onload = init;, but the same error still occurred. I would really appreciate with anyone can point out any logical error I made in the code below as I'm not able to figure them out.
var dataController = (function() {
var data = {
quote: '',
author: ''
}
function setRandomQuote() {
var script = document.createElement('script');
script.src = 'http://api.forismatic.com/api/1.0/?method=getQuote&lang=en&format=jsonp&jsonp=parseQuote';
document.body.appendChild(script);
window.parseQuote= function(json) {
data.quote = json.quoteText;
data.author = json.quoteAuthor;
}
script.parentNode.removeChild(script);
}
return {
getQuote: function() {
setRandomQuote();
return data;
}
};
})();
var UIController = (function(){
return {
changeQuote: function(data) {
document.querySelector('q').innerHTML = data.quote;
document.querySelector('cite').innerHTML = data.author;
}
}
})();
var appController = (function(dataCtrl, UICtrl) {
var setupEventListener = function() {
document.querySelector('#btn-quote').addEventListener('click', getRandomQuote);
};
function getRandomQuote() {
var data= {};
// 1. get quote
var data = dataCtrl.getQuote();
// 2. change UI with new quote
UICtrl.changeQuote(data);
}
return {
init: function() {
setupEventListener();
getRandomQuote();
}
}
})(dataController, UIController);
appController.init();

javascript function move to webworker

I have function:
function addModel() {
var values = new Array();
var $input = $('input[type=\'text\']');
var error = 0;
$input.each(function() {
$(this).removeClass('double-error');
var that = this;
if (that.value!='') {
values[that.value] = 0;
$('input[type=\'text\']').each(function() {
if (this.value == that.value) {
values[that.value]++;
}
});
}
});
$input.each(function(key) {
if (values[this.value]>1) {
//error++;
var name = this.value;
var product_model = $(this).parent().parent().find('.product-model').text();
var m = product_model.toLowerCase().areplace(search,replace);
$(this).parent().find('input[type=\'text\']').val(name + '-' + m);
}
});
return error <= 0; //return error > 0 ? false : true;
}
where are a lot of inputs to recheck... up to 50000. Usually are about 5000 to 20000 inputs. Of course browsers are freezing... How to move this function to web-worker and call it to get data back and fill form type="text"
thank you in advance.
Web workers don't have direct access to the DOM. So to do this, you'd need to gather the values of all 5-50k inputs into an array or similar, and then send that to the web worker (via postMessage) for processing, and have the web worker do the relevant processing and post the result back, and then use that result to update the inputs. See any web worker tutorial for the details of launching the worker and passing messages back and forth (and/or see my answer here).
Even just gathering up the values of the inputs and posting them to the web worker is going to take significant time on the main UI thread, though, as is accepting the result from the worker and updating the inputs; even 5k inputs is just far, far too many for a web page.
If maintaining browser responsiveness is the issue, then releasing the main thread periodically will allow the browser to process user input and DOM events. The key to this approach is to find ways to process the inputs in smaller batches. For example:
var INTERVAL = 10; // ms
var intervalId = setInterval((function () {
var values = [];
var $input = $('input[type=\'text\']');
var index;
return function () {
var $i = $input[index];
var el = $i.get();
$i.removeClass('double-error');
if (el.value != '') {
values[el.value] = 0;
$input.each(function() {
if (this.value == el.value) {
values[el.value]++;
}
});
}
if (index++ > $input.length) {
clearInterval(intervalId);
index = 0;
// Now set elements
intervalId = setInterval(function () {
var $i = $input[index];
var el = $i.get();
if (values[el.value] > 1) {
var name = el.value;
var product_model = $i.parent().parent().find('.product-model').text();
var m = product_model.toLowerCase().areplace(search,replace);
$i.parent().find('input[type=\'text\']').val(name + '-' + m);
}
if (index++ > $input.length) {
clearInterval(intervalId);
}
}, INTERVAL);
}
};
}()), INTERVAL);
Here, we do just a little bit of work, then use setInterval to release the main thread so that other work can be performed. After INTERVAL we will do some more work, until we finish and call clearInterval

Recursively walk website pages with phantomjs

I have to go through all the pages from a website and check for an element on every page. This has to happen recursively, and I chose to do it with PhantomJS. So, I basically have this/such code in main.js:
var page = require('webpage').create();
var allUrls = [];
var pageCheck = function(url) {
page.open(url, function(success) {
page.evaluate(function(allUrls, nextPage) {
// crawl all links, and if they are from this site ..
// add them to the allUrls array ..
// then check the page for the element ..
// and go to next eventual page ..
setTimeout(nextPage, 250);
}, allUrls, nextPage);
});
};
var nextPage = function() {
var nextUrl = allUrls.unshift();
if(nextUrl) pageCheck(nextUrl);
};
pageCheck('http://example.com/');
and I call this with phantomjs main.js.
But I see messages that "Can't find variable ...". And when I cleared all of them - I see now Can't find variable: pageCheck
How am I supposed to do this? ... And what is all this stuff with PhantomJS scopes ?? ...
I managed to figure it out, thanks to #ArtjomB :)
Basically, my mistake was that I was trying to call global stuff from page.evaluate, while I had to use it only for page manipulation. So I changed the code to this/such one:
var page = require('webpage').create();
var allUrls = [];
var pageCheck = function(url) {
page.open(url, function(success) {
var evalulation = page.evaluate(function() {
// gather urls and check element ..
return {
urls: ...,
checkedElement: ...
};
});
// manipulate the results from page.evaluate ..
someStuff(evalulation.urls);
otherStuff(evalulation.checkedElement);
// and THEN ... go to next eventual page ..
setTimeout(nextPage, 250);
});
};
var nextPage = function() {
var nextUrl = allUrls.unshift();
if(nextUrl) pageCheck(nextUrl);
else phantom.exit();
};
pageCheck('http://example.com/');

How to check table for new content, and then update browser tab title?

I am working on a way to flash a browser tab when a new message appears in a table. I have the flashing of the tab part working, my only problem is that I can't seem to get it to flash when a message is received (which is the whole point of my exercise :) )
The newMessage() function is working fine, I just can't seem to get the notification() function to work.
My code is as follows:
function newMessage()
{
var oldTitle = "Your Page";
var msg = "New Message";
var timeout = setInterval(function()
{
document.title = document.title == msg ? '' : msg;
}, 1000);
window.onmousemove = function() {
clearInterval(timeout);
document.title = oldTitle;
window.onmousemove = null;
};
}
function notification()
{
var index = 2;
var content = document.getElementById('refreshMessages').childNodes[index];
var content = document.getElementById('refreshMessages').getElementByTagName("tr")[1];
var knownContent = content.toString();
updater.start();
updater2.start();
var newContent = document.getElementById('refreshMessages').childNodes[index];
var newContent = document.getElementById('refreshMessages').getElementByTagName("tr")[1];
if(knownContent != newContent.toString())
{
newMessage();
knownContent = newContent;
}
else if(knownContent = newContent.toString())
{
alert("No need to flash title.");
}
}
notification();
In the notification() function, I am trying to call the newMessage() function by comparing the strings at the appropiate cell in the table.
I put the alert() into the else if just to see if it would be called, but it does not happen. update.start() and update2.start() are carried out however, as I can see the messages appearing in the table.
I would be happier to use JavaScript but I am open to jQuery also.
My JavaScript is very very rusty so excuse me if I have made any silly mistakes!
Thanks,
Chuck
You have several mistakes in function notification(), see my comments:
function notification()
{
var index = 2;
//Why are you assigning value to "content" for twice?
var content = document.getElementById('refreshMessages').childNodes[index];
/*
* function getElementByTagName is undefined, should be getElementsByTagName,
* 's' is missing. And [1] means the second one not the first one, make sure
* that's exactly what you want.
*/
var content = document.getElementById('refreshMessages').getElementByTagName("tr")[1];
/*
* content is a tr dom object, content.toString() is something like "[object]".
* If you want to get content inside a cell, you should use cell.innerHTML.
* e.g. A table:
* <table id="refreshMessages">
* <tr><td>Hello world</td></tr>
* </table>
* var table = document.getElementById('refreshMessages');
* var firstTr = table.getElementsByTagName("tr")[0];
* var firstTd = firstTr.getElementsByTagName("td")[0];
* alert(firstTd.innerHTML); //alerts "Hello world"
*/
var knownContent = content.toString();
//I doubt these functions really get invoked cuz there's javascript error above.
updater.start();
updater2.start();
//assigning twice, "getElementByTagName" is missing "s"
var newContent = document.getElementById('refreshMessages').childNodes[index];
var newContent = document.getElementById('refreshMessages').getElementByTagName("tr")[1];
//Remove toString(), use innerHTML i metioned above.
if(knownContent != newContent.toString())
{
newMessage();
knownContent = newContent;
}
//You miss an "=" here, to judge a equals b, you should use "=="
else if(knownContent = newContent.toString())
{
alert("No need to flash title.");
}
}

Chrome extension, using localStorage to save ip, tabid, serverfingerprint per tab

I should mention up front I'm new to code/stackoverflow so my apologies if this question doesn't makes sense. I'm beyond stumped, I'm trying to build a chrome extension that saves the ip address, url and a server finger print. The serverfingerprint is a field that lives within the response headers. Using my background.js and localStorage I can save this information and then display it in my pop up window. This is all fine and dandy except I can't figure out how to save it on a per tab basis, aka... if I have 5 tabs open, I'd like to click my extension and have it display the url for each corresponding tab. example: click tab4 and shows tab4's url, then click tab2 and it shows the url of tab2.
the below code works except for it doesn't tie to the tabId so it's not exactly ideal. Any ideas of where to start researching would be very appreciated!
what i've done thus far:
background.js:
chrome.experimental.webRequest.onCompleted.addListener(function (details)
{
var headers = details.responseHeaders;
localStorage['ip'] = details.ip;
localStorage['url'] = details.url;
for (var i = 0, length = headers.length; i < length; i++)
{
var header = headers[i];
if (header.name == 'X-Server-Fingerprint')
{
localStorage['XServerFingerprint'] = header.value.toString();
break;
}
}
},{'urls': ['http://www.someurl.com/*']},['responseHeaders']);
popup.js:
document.getElementById('url').innerText = localStorage['url'];
document.getElementById('ip').innerText = localStorage['ip'];
document.getElementById('XServerFingerPrint').innerText = localStorage['XServerFingerPrint'];
As each tab has unique id (until browser restart), you can use it to identify tabs.
You are probably interested only in current tabs, which makes things simpler as you don't need localStorage for this (which persists data between browser restarts). Just use background page's namespace to store data about all tabs:
// background.js
var tabs = {}; //all tab data
chrome.experimental.webRequest.onCompleted.addListener(function (details)
{
var tabInfo = {};
tabInfo["ip"] = ...;
tabInfo["url"] = ...;
tabInfo["XServerFingerprint"] = ...;
tabs[details.tabId] = tabInfo;
}
// popup.js
chrome.tabs.getSelected(null, function(tab){
var tabInfo = chrome.extension.getBackgroundPage().tabs[tab.id]; // get from bg page
document.getElementById('url').innerText = tabInfo['url'];
document.getElementById('ip').innerText = tabInfo['ip'];
document.getElementById('XServerFingerPrint').innerText = tabInfo['XServerFingerPrint'];
});
If you do need localStorage then you can convert tabs object to json string and store it there.
Ok, so I've sorted out my issues! Well the ones concerning chrome extensions haha, which appears to be pretty much exactly what Serg is saying (thx Serg!!) I wrote it a bit different tho.
// background.js
chrome.experimental.webRequest.onCompleted.addListener(function (details)
{
var headers = details.responseHeaders;
var tabId = details.tabId;
var ip = details.ip;
var url = details.url;
for (var i = 0, length = headers.length; i < length; i++) {
var header = headers[i];
//custom field in response headers from my site
if (header.name == 'X-Server-Fingerprint') {
var XServerFingerprint = header.value.toString();
var data = {
ip: ip,
url: url,
fingerprint: XServerFingerprint
}
//store it
localStorage[tabId] = JSON.stringify(data);
break;
}
}
},{'urls': ['http://www.corbisimages.com/*']},['responseHeaders']);
}
// and then on my popup.js
chrome.tabs.getSelected(null, function(tab) {
var parseData = JSON.parse(localStorage[tab.id]);
document.getElementById('XServerFingerprint').innerText = parseData.fingerprint;
document.getElementById('url').innerText = parseData.url;
document.getElementById('ip').innerText = parseData.ip;
});

Categories