Can ghost.py scrape a web page with javascript setInterval update repeatedly? - javascript

I plan to use Ghost.py to scrape a web page which is updated every 5 seconds by setInterval.
How can I scrape the updated data by Ghost.py repeatedly, once there is a new update by this script?
The setInterval code below:
(function () {
var template;
(function () {
template = $("#template1 tbody").html();
GetData();
})();
function GetData() {
var max = 100,
sn = $.data(document, "sn");
if (sn === undefined) {
sn = 0;
} else {
sn = parseInt(sn, 10);
}
$.ajax({
url: path + "ashx/notice.ashx",
async: true,
cache: false,
data: {
act: "GetBotSignal",
sn: sn,
max: max
},
type: "get",
dataType: "json",
success: function (data) {
if (data !== null && data.length !== 0) {
var html = "";
var expselbox = $(".exp-sel-box");
for (var i = 0, j = data.length; i < j; i++) {
var item = template;
item = item.replace(/\{0}/g, data[i].SignalOccurTime);
(parseFloat(data[i].Ratio) >= 2 && parseFloat(data[i].Probability) >= 50) ? " sp" : "");
html += item;
}
expselbox.prepend("<tr ef=\"1\" style=\"height:0px;\"></tr>");
expselbox.find("tr[ef=1]").animate({ "height": (34 * data.length) + "px" }, "fast", function () {
expselbox.find("tr[ef='1']").remove();
expselbox.prepend(html);
expselbox.find("tr:hidden").fadeIn("fast");
expselbox.find("tr:gt(" + max + ")").remove();
$.data(document, "sn", data[0].SN);
ScrollBar($(".expscall-out"), 642, 342, true);
});
}
}
});
}
if (srvTime.getHours() > 7 && srvTime.getHours() < 14) {
setInterval(function () {
GetData();
}, 5000);
}
})();

Related

Lazy loading works for loading data, but not when filtering

Here the first script is written for lazy loading the table data and the second script is written for filtering with lazy loading but the second one is not working.
I have a Codeigniter report in which I did some filtering on the table data. I am using jQuery AJAX to lazy load data. What I expected is that when I fetch the data with a filter the lazy loading is not working. Shall i use first script for both table load by default and for filter. i am getting confusion. Can anyone please tell me how to merge both script as a single script for both. Please help.
$(document).ready(function() {
$('#filter').popover({
placement: 'bottom',
title: (' ') + '<button type="button" class="close pull-right" data-dismiss="alert" style="color:black;">×</button>',
html: true,
content: $('#customdiv').html()
});
$(document).on("click", ".popover .close", function() {
$(this).parents(".popover").popover('hide');
});
var limit = 20;
var start = 0;
var action = 'inactive';
function lazzy_loader(limit) {
var output = '';
for (var count = 0; count < limit; count++) {
output += '<tr class="post_data">';
output += '</tr>';
}
$('#load_data_message').html(output);
}
lazzy_loader(limit);
function search_fields(limit, start) {
$(".spinner").show();
$.ajax({
url: "<?=base_url()?>missed_call_campaign/fetch_data",
method: "POST",
data: {
limit: limit,
start: start
},
cache: false,
success: function(data) {
$(".spinner").hide();
if (data == '') {
$('#load_data_message').html('<p class="content-desc">No More Data Found</p>');
action = 'active';
} else {
$('#load_data').append(data);
$('#load_data_message').html("");
action = 'inactive';
}
}
});
}
if (action == 'inactive') {
action = 'active';
search_fields(limit, start);
}
$(window).scroll(function() {
if ($(window).scrollTop() + $(window).height() > $("#load_data").height() && action == 'inactive') {
lazzy_loader(limit);
action = 'active';
start = start + limit;
setTimeout(function() {
search_fields(limit, start);
}, 100);
}
});
});
function subcategory() {
var ClickedCategory = new Array();
$('.CategoryClicked').each(function() {
if ($(this).is(':checked')) {
ClickedCategory.push($(this).val());
}
});
$.ajax({
type: 'POST',
url: "<?=base_url()?>missed_call_campaign/subcategory_checkbox",
data: {
type: 'text',
ClickedCategory: ClickedCategory
},
success: function(response) {
$("#collapsepp").hide();
$("#collapseqq").html(response);
}
});
}
function subsource() {
var ClickedSource = new Array();
$('.SourceClicked').each(function() {
if ($(this).is(':checked')) {
ClickedSource.push($(this).val());
}
});
$.ajax({
type: 'POST',
url: "<?=base_url()?>missed_call_campaign/subsource_checkbox",
data: {
type: 'text',
ClickedSource: ClickedSource
},
success: function(response) {
$("#collapserr").hide();
$("#collapsess").html(response);
}
});
}
function clearFilter() {
location.reload();
}
$(document).ready(function() {
$(document).on("click", "#data_filter", function() {
var CheckedRep = new Array();
var ClickedStatus = new Array();
var ClickedType = new Array();
var ClickedCategory = new Array();
var ClickedSubCategory = new Array();
var ClickedSubCategory_Filter = new Array();
var ClickedSource = new Array();
var ClickedSubSource = new Array();
var ClickedSubSource_Filter = new Array();
$('.RepClicked').each(function() {
if ($(this).is(':checked')) {
CheckedRep.push($(this).val());
}
});
$('.StatusClicked').each(function() {
if ($(this).is(':checked')) {
ClickedStatus.push($(this).val());
}
});
$('.TypeClicked').each(function() {
if ($(this).is(':checked')) {
ClickedType.push($(this).val());
}
});
$('.CategoryClicked').each(function() {
if ($(this).is(':checked')) {
ClickedCategory.push($(this).val());
}
});
$('.SourceClicked').each(function() {
if ($(this).is(':checked')) {
ClickedSource.push($(this).val());
}
});
$('.SubSourceClicked').each(function() {
if ($(this).is(':checked')) {
ClickedSubSource.push($(this).val());
}
});
$('.SubCategoryClicked').each(function() {
if ($(this).is(':checked')) {
ClickedSubCategory.push($(this).val());
}
});
$('.SubCategoryChecked_Filter').each(function() {
if ($(this).is(':checked')) {
ClickedSubCategory_Filter.push($(this).val());
}
});
$('.SubSourceClicked_filter').each(function() {
if ($(this).is(':checked')) {
ClickedSubSource_Filter.push($(this).val());
}
});
if ((CheckedRep.length > 0) || (ClickedStatus.length > 0) || (ClickedType.length > 0) || (ClickedCategory.length > 0)
(ClickedSource.length > 0) || (ClickedSubSource.length > 0) || (ClickedSubCategory.length > 0) ||
(ClickedSubCategory_Filter.length > 0) || (ClickedSubSource_Filter.length > 0)) {
var limits = 20;
var starts = 0;
var actions = 'inactive';
lazzy_loading(limits);
if (actions == 'inactive') {
actions = 'active';
filter_data(limits, starts, CheckedRep, ClickedStatus, ClickedType, ClickedCategory, ClickedSubCategory, ClickedSubCategory_Filter,
ClickedSource, ClickedSubSource, ClickedSubSource_Filter);
}
$(window).scroll(function() {
if ($(window).scrollTop() + $(window).height() > $("#load_data_filter").height() && actions == 'inactive') {
lazzy_loading(limits);
actions = 'active';
starts = starts + limits;
setTimeout(function() {
filter_data(limits, starts, CheckedRep, ClickedStatus, ClickedType, ClickedCategory, ClickedSubCategory, ClickedSubCategory_Filter,
ClickedSource, ClickedSubSource, ClickedSubSource_Filter);
}, 100);
}
});
}
});
function lazzy_loading(limits) {
var output = '';
for (var counts = 0; counts < limits; counts++) {
output += '<tr class="post_data">';
output += '</tr>';
}
$('#load_data_filter').html(output);
}
function filter_data(limits, starts, CheckedRep, ClickedStatus, ClickedType, ClickedCategory, ClickedSubCategory, ClickedSubCategory_Filter,
ClickedSource, ClickedSubSource, ClickedSubSource_Filter) {
$.ajax({
url: "<?=base_url()?>missed_call_campaign/toDoAjax",
method: "POST",
data: {
type: 'text',
CheckedRep: CheckedRep,
ClickedStatus: ClickedStatus,
ClickedType: ClickedType,
ClickedCategory: ClickedCategory,
ClickedSource: ClickedSource,
ClickedSubSource: ClickedSubSource,
ClickedSubCategory: ClickedSubCategory,
ClickedSubCategory_Filter: ClickedSubCategory_Filter,
ClickedSubSource_Filter: ClickedSubSource_Filter,
limits: limits,
starts: starts
},
cache: false,
success: function(response) {
$(".spinner").hide();
$("#load_data").hide();
if (response == '') {
$('#load_data_message').html('<p class="content-desc">No More Data Found123</p>');
action = 'active';
} else {
$('#load_data_filter').append(response);
$('#load_data_message').html("");
action = 'inactive';
}
}
});
}
});
I suggest you to reinitialize the lazy load after success you fetched data from the back end.

Break the for loop based on the ajax success variable

For the following code, the emailCnt is 50 for first iteration, I need 25 in next iteration. What is the possible way to access the variable value outside the ajax success and break the for loop execution?
var limit = 50;
var emailCnt = limit;
for (var i = 0; i < 20; i++) {
console.log(emailCnt);///this value is 50 instead I need 25
if (emailCnt < limit && i != 0) {
break;
}
setTimeout(function () {
submit_post(slNo, limit, function (output) {
slNo = output;
emailCnt = 25;
$('#load_data').html('Hello');
});
}, 1000);
}
function submit_post(slNo, limit, handleData) {
$.ajax({
type: 'POST',
async: false,
url: url,
data: { slNo: slNo, limit: limit },
success: function (data) { handleData(data); }
});
}
This successfully worked for me
var limit = 50;
var emailCnt = limit;
function submit_post(slNo, limit)
{
var result="";
$.ajax({
type: 'POST',
async: false,
url: url,
data: {slNo:slNo, limit:limit},
success: function(data) { result = data; }
});
return result;
}
for(var i=0;i<20;i++)
{
if(emailCnt < limit && i != 0)
{
break;
}
setTimeout(function () {
var output = submit_post(slNo, limit);
slNo = output;
emailCnt = 25;
$('#load_data').html('Hello');
}, 1000);
}

Update textarea after get response from servlet in javascript

I have a program which calls a function in javascript with 1 o more requests to 1 servlet, I want to execute request after request and get the response after each exucution, to make this I have 1 function, but it only shows the result after all requests have been executed.
function cmd(args) {
width = 0;
var res = args.split('\n');
var largo = res.length;
var progressLength = 100 / largo;
for (var i = 0; i < largo; i++)
{
if (res[i] == 'desconectar')
{
desconectar();
break;
}
else
{
executeCMD(res[i]);
}
}
}
function executeCMD(args)
{
$.ajax({
type: "POST",
url: 'Controlador',
data: {cmd: args, operacion: 1},
success: function (response) {
document.getElementById('respuesta').value = document.getElementById('respuesta').value + response;
},
dataType: 'text',
async: false
});
}
If I add window.alert(response); inside success field it shows the progress step by step and works fine, but it show alerts which I don't want.
This is I want http://imgur.com/a/9nclR but I'm getting only last picture.
The solution if anyone is intersting was using a recursive function as next:
function cmd(args) {
width = 0;
move(0);
var res = args.split('\n');
var largo = res.length;
var valInit = 0;
if (largo > valInit)
{
executeCMD(res, valInit);
}
}
function executeCMD(args, i)
{
$(document).ready(function () {
$.ajax({
type: "POST",
url: 'ControladorServlet',
data: {cmd: args[i], operacion: 1, ticket: ticket, iddispositivo: sesion},
success: function (response) {
var textarea = document.getElementById('respuesta');
var res = response.trim().split('\n');
if(error){//dc}
else
{
document.getElementById('respuesta').value = document.getElementById('respuesta').value + response.trim() + "\n\n";
var valor = (100) * (i + 1) / args.length;
move(valor);
if (i + 1 < args.length)
{
executeCMD(args, i + 1);
}
}
},
dataType: 'text'
});
});
}

Ad Block Plus Blocking jQuery Script?

I have a script that pulls data from my CMS and then allows a person to vote on a poll. The script works fine. However, I have Ad Block Plus Plugin installed in Firefox. When that is enabled to blocks the script from submitting the form correctly. It appears to submit correctly in the front end but is never registered in the back end.
Why does Ad Block Plus block my script that has nothing to do with ads?
The script is below:
$(document).ready(function () {
var Engine = {
ui: {
buildChart: function() {
if ($("#pieChart").size() === 0) {
return;
}
var pieChartData = [],
totalVotes = 0,
$dataItems = $("ul.key li");
// grab total votes
$dataItems.each(function (index, item) {
totalVotes += parseInt($(item).data('votes'));
});
// iterate through items to draw pie chart
// and populate % in dom
$dataItems.each(function (index, item) {
var votes = parseInt($(item).data('votes')),
votePercentage = votes / totalVotes * 100,
roundedPrecentage = Math.round(votePercentage * 10) / 10;
$(this).find(".vote-percentage").text(roundedPrecentage);
pieChartData.push({
value: roundedPrecentage,
color: $(item).data('color')
});
});
var ctx = $("#pieChart").get(0).getContext("2d");
var myNewChart = new Chart(ctx).Pie(pieChartData, {});
}, // buildChart
pollSubmit: function() {
if ($("#pollAnswers").size() === 0) {
return;
}
var $form = $("#pollAnswers"),
$radioOptions = $form.find("input[type='radio']"),
$existingDataWrapper = $(".web-app-item-data"),
$webAppItemName = $existingDataWrapper.data("item-name"),
$formButton = $form.find("button"),
bcField_1 = "CAT_Custom_1",
bcField_2 = "CAT_Custom_2",
bcField_3 = "CAT_Custom_3",
$formSubmitData = "";
$radioOptions.on("change", function() {
$formButton.removeAttr("disabled"); // enable button
var chosenField = $(this).data("field"), // gather value
answer_1 = parseInt($existingDataWrapper.data("answer-1")),
answer_2 = parseInt($existingDataWrapper.data("answer-2")),
answer_3 = parseInt($existingDataWrapper.data("answer-3"));
if (chosenField == bcField_1) {
answer_1 = answer_1 + 1;
$formSubmitData = {
ItemName: $webAppItemName,
CAT_Custom_1: answer_1,
CAT_Custom_2: answer_2,
CAT_Custom_3: answer_3
};
}
if (chosenField == bcField_2) {
answer_2 = answer_2 + 1;
$formSubmitData = {
ItemName: $webAppItemName,
CAT_Custom_1: answer_1,
CAT_Custom_2: answer_2,
CAT_Custom_3: answer_3
};
}
if (chosenField == bcField_3) {
answer_3 = answer_3 + 1;
$formSubmitData = {
ItemName: $webAppItemName,
CAT_Custom_1: answer_1,
CAT_Custom_2: answer_2,
CAT_Custom_3: answer_3
};
}
prepForm($formSubmitData);
});
function prepForm(formSubmitData) {
$formButton.click(function(e) {
e.preventDefault();
logAnonUserIn("anon", "anon", formSubmitData); // log user in
}); // submit
} // prepForm
function logAnonUserIn(username, password, formSubmitData) {
$.ajax({
type: 'POST',
url: '/ZoneProcess.aspx?ZoneID=-1&Username=' + username + '&Password=' + password,
async: true,
beforeSend: function () {},
success: function () {},
complete: function () {
fireForm(formSubmitData);
}
});
} // logAnonUserIn
function fireForm(formSubmitData) {
// submit the form
var url = "/CustomContentProcess.aspx?A=EditSave&CCID=13998&OID=3931634&OTYPE=35";
$.ajax({
type: 'POST',
url: url,
data: formSubmitData,
async: true,
success: function () {},
error: function () {},
complete: function () {
window.location = "/";
}
});
}
} // pollSubmit
} // end ui
};
Engine.ui.buildChart();
Engine.ui.pollSubmit();
});
As it turns out easylist contains this filter:
.aspx?zoneid=
This is why my script is being blocked.
I was told I can try this exception filter:
##||example.com/ZoneProcess.aspx?*$xmlhttprequest
I could also ask easylist to add an exception.
Answer comes from Ad Block Plus Forums.

Why the ajax success functions runs 2 times?

main.html
<script src="jsv3/onload.js"></script>
<script>
var countImage = 0;
function load_pages(page) {
$.ajax({
type: "GET",
url: "scandir.php",
data: "page=" + page,
dataType: 'json',
success: function (data) {
var num = 0;
var cache = [];
var startNum = 0;
var endNum = 0;
$.each(data, function(i,paths){
if ( !(page == countImage) && !(page+1 == countImage))
{
if (paths[0] != ''){
num = parseInt(paths[0].split("_P")[1],10);
if (!$('#img_'+num).length){
$("#div_"+num).append("<img id = 'img_"+num+"' src = '"+paths[0]+"' alt = 'flip book' />");
}
cache.push (num);
}
if (paths[1] != ''){
num = parseInt(paths[1].split("_P")[1],10);
if (!$('#img_'+num).length){
$("#div_"+num).append("<img id = 'img_"+num+"' src = '"+paths[1]+"' alt = 'flip book' />");
}
cache.push (num);
}
}
});
startNum = cache[0];
endNum = cache[cache.length-1];
for (var z = 0; z < 2; z++) {
startNum--;
if ($('#img_'+startNum).length){
$("#img_"+startNum+":last-child").remove();
}
}
for (var x = 0; x < 2; x++) {
endNum++;
if ($('#img_'+endNum).length){
$("#img_"+endNum+":last-child").remove();
}
}
/*if ($('#img_6').length)
alert ('img6 exist');
else
alert ('img6 not exist');*/
}
});
}
function create_div() {
var counter = 1;
$.ajax({
type: "GET",
url: "countImg.php",
dataType: 'json',
success: function (data) {
//$("#book").append("<div id='cover'></div>");
countImage = data;
for (var j = 0; j < data; j++) {
$("#book").append("<div id = 'div_"+counter+"'></div>");
counter++;
}
counter = 1;
}
});
}
$(document).ready(function(){
console.log('test');
create_div();
});
</script>
onload.js( call the init function):
/* = Start
-------------------------------------------------------------- */
$(window).bind('keydown', function(e){
if (e.keyCode==37)
$('#book').turn('previous');
else if (e.keyCode==39)
$('#book').turn('next');
});
/* Moved to the html file (FlipV5.html) to ensure the page loading is finished before initialize turn.js */
$(window).load(function(){
$('#page').show();
Book.init();
if (isiPhone()) {
$('#page').addClass('mobile');
} else {
Book.zoom_auto();
Book.book_position();
}
Book.dragdrop_init();
Navigation.init();
calculate_zoom_factor();
});
$(window).resize(function() {
if (!isiPhone()) {
Book.book_position();
Book.zoom_auto();
Book.dragdrop_init();
}
calculate_zoom_factor();
});
function resizeDetect() {
var rtime = new Date(1, 1, 1, 1,00,00);
var timeout = false;
var delta = 200;
$(window).resize(function() {
rtime = new Date();
if (timeout === false) {
timeout = true;
setTimeout(resizeend, delta);
}
});
function resizeend() {
if (new Date() - rtime < delta) {
setTimeout(resizeend, delta);
} else {
timeout = false;
window_width = $(window).width();
window_height = $(window).height();
if ( $(window).width() > $(window).height() ) {
//Book.scaleVertical();
} else {
}
}
}
}
resizeDetect();
onload.js (init function)
init: function() {
default_book_width = WIDTH_BOOK;
default_book_height = HEIGHT_BOOK;
default_page_width = WIDTH_BOOK;
default_page_height = HEIGHT_BOOK;
window_height = $(window).height();
window_width = $(window).width();
zoom_steps = ZOOM_STEPS_LENGTH;
current_zoom_step = 0;
dbl_clicked = false;
on_start = true;
self = this;
$('#book').turn({
display: 'double',
acceleration: true,
elevation:50,
when: {
first: function(e, page) {
$('.nav_arrow.prev').hide();
},
turned: function(e, page) {
if (page > 1) {
$('.nav_arrow.prev').fadeIn();
$('#about').hide();
}
if ( page < $(this).turn('pages') ) {
$('.nav_arrow.next').fadeIn();
}
pageNo = $('#book').turn('page');
load_pages(pageNo);
},
turning: function(e, page) {
if (page < 2) {
$('#about').show();
}
},
last: function(e, page) {
$('.nav_arrow.next').hide();
}
}
});
Book.arrows();
},
scaleHorizontal: function() {
new_width = $(window).width()-100;
ratio = new_width / $('#page').width();
new_height = $('#page').height() * ratio;
$('#page').css({
width: new_width,
height: new_height
});
$('#book').turn('size', new_width, new_height);
},
scaleStart: function() {
if ( on_start == true ) {
bookHeightCheck();
if ( higherThanWindow == true ) {
Book.scaleVertical();
if ( $('#page').width() > $(window).width() ) {
Book.scaleHorizontal();
}
} else {
Book.scaleHorizontal();
}
on_start = false;
}
},
The flow is like that: main.html call create_div to do some function, and since it append the onload.js, it runs Book.init() , which will call the load_pages function. So, I believe the problem is due to two ajax call is implement at the same time?
The create_div() ajax accidentally detected the ajax call in load_pages is success, so it runs two time the success function.
How to fix the problem like this? Thanks

Categories