We Keep Coding

PhantomJS: TypeError: null is not an object(evaluating 'document.getElementById'())

One day ago I started messin' with phantomjs and their ability to read javascript generated data from the websites.[web scraping]
I'm trying to get element's text content by ID, but sometimes the particular website I try to crawl through doesn't have it, so then I get this error:
ERROR: TypeError: null is not an object (evaluating 'document.getElementById('resInfo-0').textContent')
TRACE:
-> undefined: 2
-> : 5
Screenshot from the Command Prompt:
My code so far:
1 step: reading the data from the file.
var file = "path to text file";
var fs = require('fs');
var stream = fs.open(file, 'r');
var urls = new Array();
var index = 0;
console.log("READING A FILE...");
while(!stream.atEnd()) {
var line = stream.readLine();
urls[index] = line;
index++;
}
console.log("FINISHED READING THE FILE");
index = 0;
2 step: Reading the data from the websites.
function web_page()
{
webPage = require('webpage');
page = webPage.create();
page.onError = function(msg, trace)
{
var msgStack = ['ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + t.file + ': ' + t.line + (t.function ? ' (in
function "' + t.function +'")' : ''));
});
}
console.log(msgStack.join('\n') + " URL: " + urls[index]);
phantom.exit(0);
};
phantom.onError = function(msg, trace)
{
var msgStack = ['PHANTOM ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line +
(t.function ? ' (in function ' + t.function +')' : ''));
});
}
console.log(msgStack.join('\n')+ " URL: " + urls[index]);
phantom.exit(0);
};
page.open('http://www.delfi.lt/paieska/?q='+urls[index], function(status)
{
if (status !== 'success')
{
console.log('Unable to access network');
}
else
{
var fs = require('fs');
var path = 'output.txt';
var content = page.content;
var ua = page.evaluate(function()
{
var x = document.getElementById('resInfo-0').textContent;
return x;
});
if ( ua != null && ua != "" )
{
var indexas = ua.indexOf("(");
ua = ua.substr(0,indexas);
ua = ua.replace(/\D/g,'');
fs.write(path,urls[index] + " - "+ ua + "\r\n", 'a+');
}
}
});
setTimeout(next,1000);
}
console.log("STARTING TO CRAW WEBSITES...");
web_page();
function next()
{
if ( index + 1 <= 288103 )
{
page.close();
index++;
web_page();
}
else if ( index + 1 > 288103 )
{
console.log("FINISHED CRAWLING PROCESS");
phantom.exit(0);
}
}
var ua = page.evaluate(function()
{
var x = document.getElementById('resInfo-0').textContent;
return x;
});
The error comes from here probably:
var ua = page.evaluate(function()
{
var x = document.getElementById('resInfo-0').textContent;
return x;
});
What I've tried:
if ( document.getElementById('resInfo-0').textContent != null )
if ( document.getElementById('resInfo-0').textContent != "" )
So why can't it become null without triggering this error?
PhantomJS version is 2.1.1 binary windows package.

Sometimes document.getElementById('resInfo-0') is null, but you're still trying to get .textContent of it, hence the error. Try
var elem = document.getElementById('resInfo-0');
if(elem !== null) {
return elem.textContent;
}
return false;

node.js large number of requests throwing error

I got some troubles with my image downloader script, so I have array with images names (size around 5000 elements), and I do for cycle in its array, and for every iteration download image with requests module.
All works fine but only then I have arrays with size not bigger then 500+- elements.
If I run script with 5000+ elements, I see many errors spamed from request module(err or underfined responce object) , and finnaly all application could fails with EMPTY FILE ERROR. I think that there some async troubles cause NODE.JS didnt handles so many operation at time.
Maybe I can solve it by splitting my large 5000 size array by 300 items, and dont iterate over(and dont call fetchImage()) on the next chunck before previous chunk. Or maybe there exists more nicer way to solving my problem.?
products.map(function (product) {
fetchImage(product.imageUrl,'./static/' + product.filename + '_big.', 0, 0);
return;
});
function fetchImage(url, localPath, index, iteration) {
var extensions = ['jpg', 'png', 'jpeg', 'bmp' , ''];
if (iteration > 2 || index === extensions.length) { // try another extensions
iteration++;
if(iteration < 3) {
setTimeout(function(){
fetchImage(url, localPath, 0, iteration);
}, 3000);
}else{
console.log('Fetching ' + url + ' failed or no image exists ');
return;
}
return;
}
var fileExtension;
if(extensions[index] === '' ) {
fileExtension = extensions[0];
}else{
fileExtension = extensions[index];
}
request.get(url + extensions[index], function(err,response,body) {
if(err || undefined === response){ // if err try after 3 sec timeout
setTimeout(function(){
console.log('Error URL : ' + url + extensions[index]);
fetchImage(url, localPath, index, iteration);
}, 3000);
return;
}else{
if(response.statusCode === 200) {
request(url + extensions[index])
.on('error', function(err) {
console.log("ERRRRRROR " + url + extensions[index] + " " + err);
setTimeout(function(){
console.log('Error URL : ' + url + extensions[index]);
fetchImage(url, localPath, index, iteration);
}, 3000);
return;
})
.pipe(fs.createWriteStream(localPath + fileExtension ));// write image to file
console.log('Successfully downloaded file ' + localPath + fileExtension);
return;
}else {
fetchImage(url, localPath, index + 1, iteration);
}
}
});
};

Fixed using setTimeOut beetween each request
setTimeout(
function () {
fetchImage(imageUrl,'./static/' + filename + '_big.', 0, 0);
},
300 * (i + 1) // where they will each progressively wait 300 msec more each
);

Cannot Claim OPOS MSR in JS

I have Posiflex MSR MR2000 Series. I've intalled Posiflex OPOS Control.
I need to manage MSR on html page under IE 8. I use ocx control OPOS.MSR in my JS code / object with regarding CLSID. I have such a code (ActiveX version)
function msrop(){
try {
if ( r = msro.Open('MR1') ) { log ( ER100 + ' ' + 'MSROP' + ' ' + r ); return }
log(r);
}
catch ( e ) {
log ( 'MSROP' , e ) ; return ;
}
}
function msrcl(){
try {
if ( r = msro.Claim(-1) ) { log ( ER100 + ' ' + 'MSRCL' + ' ' + r ); return }
msro.DeviceEnabled = true ;
msro.DataEventEnabled = true ;
log(r);
}
catch ( e ) {
log ( 'MSRCL' , e ) ; return ;
}
}
function ol(){
try { msro = new ActiveXObject ( 'OPOS.MSR' ) ; } catch ( e ) { alert(e) }
bind ( msro ) ;
}
function bind( o ){
function o::DataEvent(){
log('<br/>' + '->Tracks read ' + 'T1: ' + o.Track1Data + ' T2: ' + o.Track2Data + 'T3: ' + o.Track3Data + 'T4: ' + o.Track4Data );
log('<br/>' + '->DataEventEnabled ' + ( msro.DataEventEnabled = true ))
var d = document.getElementById('track');
d.innerHTML = o.AccountNumber;
}
}
...
<button onclick="msrop()">Open</button>
<button onclick="msrcl()">Claim</button>
This code works pretty well within HTA application: Claim method returns OPOS_SUCCESS (0). But when I try it as HTM in Internet Explorer 8, Claim method raises exception and object ResultCode property is 104 (OposENoservice). Recall returns ResultCode 102 (OposEClaimed).
I need to get work Claim method properly in Internet Explorer. Why does the same code work different in HTA, which is based on IE??

Unit testing a function and simulating an event emitter response

I am currently writing some tests against the code shown below in node.js using mocha. I want to simulate the response from the event emitter 'check_user'. I have sinon.js but I can't get my mind in the right place to work out the best way to simulate the response.
Anyone able to offer some advice on how to go about this?
TgCustomCommand.contact = new Command("contact", "Will send you a users contact card to add to your contact list. Usage is .contact nick", function (input, callback) {
var payload = input;
//Switch our contact to payload.to as this is what our checkuser function looks at
//Check the command over
if (payload.command_args === false) {
payload.response = 'msg ' + payload.to + ' ' + this.description;
return callback(null, payload);
} else {
payload.return = payload.to;
payload.to = payload.command_args; //Set up ready for nick check
//Check the nick exists
emitter.emit('check_user', payload, function (err, result) {
if (err) return callback(err, null);
payload = input; //Reset our payload so we have correct payload.to
//Check how many users we returned
if (result.length === 0) { //Not in our contact list
payload.response = 'msg ' + payload.return + ' I do not have that person in my contact list!';
return callback(null, payload);
} else if (result.length === 1) {
payload.to = result[0].Nick;
payload.response = "send_contact " + payload.return + ' ' + result[0].Phone + ' ' + result[0].Nick + " _";
return callback(null, payload);
}
else {
//loop through our object and create a list of those returned
payload.response = "msg " + payload.return + " I know multiple people with a similar nick: ";
for (var i = 0; i < result.length; i++) {
log.debug(result[i].Nick);
payload.response = payload.response + result[i].Nick + " ";
}
return callback(null, payload);
}
});
}
;
});

callbacks in browser after socket.io emit

i am developing a way to get callbacks in the browser page, following a emit to the socketio server.
server code:
/** exec php file and retrieve the stdout */
socket.on('php', function(func, uid, data) {
console.log('php'.green + ' act:' + func + ' uid:' + uid);
runner.exec("php " + php_[func].exec,
function(err, stdout, stderr) {
if (err == null) {
socket.emit('php', {uid: uid, o: stdout});
console.log('emitted');
} else {
console.log('err '.red + stdout + ' ' + stderr);
}
});
});
this code executes a php page and retrieves the output to display or parse in the browser
it receives an id to echo back to the page, so I can know what function to execute
browser code to execute callbacks:
function log(text) {
$('.out').append(text + '<br />');
}
window.callbacks = [];
function setcb(c) {
var max = 0;
$.each(window.callbacks, function(index) {max = (index > max ? index : max);});
window.callbacks[max+1] = c;
return max+1;
};
function C(k){return(document.cookie.match('(^|; )'+k+'=([^;]*)')||0)[2]}
var s = io.connect("http://"+ window.location.host +":8088");
//s.emit('debug', 'here');
s.on('getid', function(){
console.log('getid cookieis: ' + C('igr') );
s.emit('setid', C('igr'));
});
s.emit('php', 'test',
setcb(
function () {
var data = JSON.parse(this);
log('callback passed' + this);
}
), null
);
s.on('php', function(data) {
//window.callbacks[j.uid].call(j.o);
log('rec' + JSON.stringify(data));
//var jsn = JSON.parse(data);
console.log(data);
console.log(window.callbacks[data.uid]);
window.callbacks[data.uid].call(data.o);
delete window.callbacks[data.uid];
window.callbacks.splice(data.uid, 1);
console.log(window.callbacks);
});
this is working, but when I try to make two requests at the same time, it doesn't run like expected, leaving one callback to execute and in the callbacks array.
test code:
s.emit('php', 'test',
setcb(
function (data) {log('callback passed' + this);}
), null
);
s.emit('php', 'test',
setcb(
function (data) {log('callback passed' + this);}
), null
);
I want to eliminate this error, and for each event received, execute the callback I define.

This is way more simple than I've imagined
You can pass by reference the callback.
server side code:
/** exec php file and retrieve the stdout */
socket.on('php', function(func, data, callback) {
console.log('php'.green + ' act:' + func);
runner.exec("php " + php_[func].exec,
function(err, stdout, stderr) {
if (err == null) {
callback(stdout);
//socket.emit('php', {uid: uid, o: stdout});
console.log('emitted');
} else {
console.log('err '.red + stdout + ' ' + stderr);
}
});
});
client side code:
function log(text) {
$('.out').append(text + '<br />');
}
function C(k){return(document.cookie.match('(^|; )'+k+'=([^;]*)')||0)[2]}
var s = io.connect("http://"+ window.location.host +":8088");
//s.emit('debug', 'here');
s.on('getid', function(){
console.log('getid cookieis: ' + C('igr') );
s.emit('setid', C('igr'));
});
s.emit('php', 'test', null,
function(data) { log('d: ' + JSON.stringify(data)); }
);