We Keep Coding

How to get html source of page, after all it scripts was called?

I'am trying to parse site. The site (i suppose) using scripts and data bases to load data from (dynamically?). And this is my problem... I am trying to grab data through C# (unfortunately i don't have access to code right now) or JS. And it seems like either C# and JS, get only template of the site, but don't wait until all scripts executed. So this is my question, is there any way to get ALL html source? Maybe call scripts somehow. Or make a request, wait for 10 seconds, and then write source html data into variable?
Here is my JS code.
function request(link)
{
var xhr = new XMLHttpRequest();
xhr.open('GET', link, true);
xhr.onreadystatechange = function() .
{console.log(xhr.readyState);};
xhr.send();
let data = xhr.responseText;
var tempDiv = document.createElement('div');
tempDiv.innerHTML = data.replace(/<script(.|\s)*?\/script>/g,
'');
return tempDiv;
}
function loadFile(url, timeout, callback)
{
var args = Array.prototype.slice.call(arguments, 3);
var xhr = new XMLHttpRequest();
xhr.ontimeout = function () {
console.error("The request for " + url + " timed out.");
};
xhr.onload = function() {
if (xhr.readyState === 4) {
if (xhr.status === 200) {
callback.apply(xhr, args);
} else {
console.error(xhr.statusText);
}
}
};
xhr.open("GET", url, true);
xhr.timeout = timeout;
xhr.send(null);
let data = xhr.responseText;
return data;
}
function showMessage (message) {
console.log(message + this.responseText);
}
function include(scriptUrl)
{
var xmlhttp = new XMLHttpRequest();
xmlhttp.open("GET", scriptUrl);
xmlhttp.onreadystatechange = function()
{
if ((xmlhttp.status == 200) && (xmlhttp.readyState == 4))
{
eval(xmlhttp.responseText);
}
};
xmlhttp.send();
let data = JSON.parse(xmlhttp.responseText);
var tempDiv = document.createElement('div');
tempDiv.innerHTML = data.replace(/<script(.|\s)*?\/script>/g,
'');
return tempDiv;
}
All this functions do not work as i want.

This isn't really practical - you're trying to load an HTML page, all associated scripts, then run them on the HTML page as if they were in a proper browser environment, but within your current browser session.
This sort of thing is feasible with the jsdom library if you were running on the server-side (NodeJS), because it simulates browser behaviour: https://github.com/jsdom/jsdom. So you could do
JSDOM.fromURL("https://example.com/", { runScripts: "dangerously" }).then(dom => {
console.log(dom.serialize()); //turn the page back into HTML
});
...to get the whole thing.

An AJAX call with JavaScript

I'm trying to learn how to make an AJAX call using vanilla JavaScript in an effort to move away from JQuery for a little project that I'm working on but don't seem to be getting past xmlhttp.onreadystatechange. Can anyone point to what I'm doing wrong (the function getDVDsAndBluRays() is getting invoked on DOMContentLoaded)? Thanks!
function getDVDsAndBluRays() {
console.log("Getting logged");
var xmlhttp = new XMLHttpRequest();
var url = 'http://www.omdbapi.com/?t=metropolis&y=&plot=short&r=json';
xmlhttp.onreadystatechange = function() {
console.log("Not getting logged");
if (xmlhttp.readyState == 4 && xmlhttp.status == 200) {
console.log('responseText:' + xmlhttp.responseText);
var myMovies = JSON.parse(xmlhttp.responseText);
myFunction(myMovies);
}
xmlhttp.open('GET', url, true);
xmlhttp.send();
};
}
function myFunction(myMovies) {
for (var i = 0; i < myMovies.length; i++) {
var title = myMovies[i].Title.toLowerCase().split(' ').join('+');
var year = myMovies[i].Year;
console.log(title + ", " + "year");
}
}

It should be like that, notice the location of open and send functions:
function getDVDsAndBluRays() {
console.log("Getting logged");
var xmlhttp = new XMLHttpRequest();
var url = 'http://www.omdbapi.com/?t=metropolis&y=&plot=short&r=json';
xmlhttp.open('GET', url, true);
xmlhttp.send();
xmlhttp.onreadystatechange = function() {
console.log("Not getting logged");
if (xmlhttp.readyState == 4 && xmlhttp.status == 200) {
console.log('responseText:' + xmlhttp.responseText);
var myMovies = JSON.parse(xmlhttp.responseText);
myFunction(myMovies);
}
};
}
function myFunction(myMovies) {
for (var i = 0; i < myMovies.length; i++) {
var title = myMovies[i].Title.toLowerCase().split(' ').join('+');
var year = myMovies[i].Year;
console.log(title + ", " + "year");
}
}
onreadystatechange is executed after the call, you were actually "calling the service when it replies"

You have your .open() and .send() inside your onreadystatechange() handler. Put those outside of the onreadystatechange function and you should be good to go.
Onreadystatechange() is the event handler for when there is a change in state in the xmlhttp request, and will not get called until you open the request and send it.
Hope this helped!

You have put the calls to open and send inside the onreadystatechange event handler so they will never be called.
Move them outside it.

Get Date Header Asyncronously

As the title says, I want to get the Response Header Date value, but I keep getting the following warning :
Synchronous XMLHttpRequest on the main thread is deprecated because of
its detrimental effects to the end user's experience. For more help,
check https://xhr.spec.whatwg.org/.
My code :
function getxmlhttp () {
// although IE supports the XMLHttpRequest object, but it does not work on local files.
var forceActiveX = (window.ActiveXObject && location.protocol === "file:");
if (window.XMLHttpRequest && !forceActiveX) {
return new XMLHttpRequest();
}else {
try {
return new ActiveXObject("Microsoft.XMLHTTP");
} catch(e) {}
}
alert ("Your browser doesn't support XML handling!");
return null;
};
function srvTime(){
xmlHttp = getxmlhttp();
//xmlHttp.open('HEAD',window.location.href.toString(),false);
//need to send this to a non-volitile page
xmlHttp.open('GET',"blank.php",false);
xmlHttp.setRequestHeader("Content-Type", "text/html");
xmlHttp.send(null);
console.log("raw " + xmlHttp.getResponseHeader("Date"));
return xmlHttp.getResponseHeader("Date");
};
When I switch this line:
xmlHttp.open('GET',"blank.php",true);
To be true, the value returns NULL.
So can this be done, or do I have to just live with the warning in the console?
Thank you

As your title states, you must make the request asynchronously. That means you have to issue the request and wait for it to complete to get the information. Something like this should work:
function srvTime(callback) {
xmlHttp = getxmlhttp();
//xmlHttp.open('HEAD',window.location.href.toString(),false);
//need to send this to a non-volitile page
xmlHttp.onreadystatechange = function () {
if (xmlHttp.readyState == 4) { // The operation is complete
console.log("raw " + xmlHttp.getResponseHeader("Date"));
callback(xmlHttp.getResponseHeader("Date"));
xmlHttp = null;
}
};
xmlHttp.open('GET', "blank.php", true);
xmlHttp.setRequestHeader("Content-Type", "text/html");
xmlHttp.send(null);
};
Note that you must change the signature of your srvTime method. You can't return the data from it, the caller must supply a callback function that receives the date once the request completes.
An example of how you would use this function with the new signature is as follows:
srvTime(function (serverDate) {
document.getElementById("clock").innerHTML = "Game Time: " + serverDate;
});

How to wait for ajax request to complete in javascript when synchronous option is not available?

I'm trying to wait for the AJAX request to complete. It would be easy if the method xmlhttp.open would support async = false but Ant Galio does not support this option and only asynchronous requests are permitted. The question is how can I wait for the callback to be called.
var ajaxFinished = false;
var xmlhttp = new XMLHttpRequest();
this.debug("-- onreadystatechange is being defined");
xmlhttp.onreadystatechange = function() {
if (xmlhttp.readyState == 4) {
ajaxFinished = true;
var data = xmlhttp.responseText;
if (xmlhttp.status == 200) {
that.debug('downloadSettings: SUCCESS');
[...]
} else {
that.debug('downloadSettings:');
that.debug('-- Error: ');
that.debug('-- ResponseText: "'+data+'"')
}
}
}
while (ajaxFinished == false) {
}
this.debug("-- open connection");
xmlhttp.open("GET", requestUrl, true); /* Ant Galio does not support synchronous */
this.debug("-- send");
xmlhttp.send();
I'm looking for some kind of active waiting. I know about another solution but I'm interested in a solution that would not require changing more of the code than is my example above.
Thanks!

yes, you can
function getFile(url) {
if (window.XMLHttpRequest) {
AJAX=new XMLHttpRequest();
} else {
AJAX=new ActiveXObject("Microsoft.XMLHTTP");
}
if (AJAX) {
AJAX.open("GET", url, false);
AJAX.send(null);
return AJAX.responseText;
} else {
return false;
}
}
var fileFromServer = getFile('http://somedomain.com/somefile.txt');
w3c definition http://www.w3.org/TR/XMLHttpRequest/#the-open()-method
client . open(method, url [, async = true [, user = null [, password = null]]])

You can't. There is no "active waiting" in JavaScript, there can be only one active execution a time ("single-threaded").

There is a workaround.
Instead of using the the blocking while loop for poll use the nonblocking setInterval()..
so your code might look something like this.
var ajaxFinished = false;
var xmlhttp = new XMLHttpRequest();
this.debug("-- onreadystatechange is being defined");
xmlhttp.onreadystatechange = function() {
if (xmlhttp.readyState == 4) {
ajaxFinished = true;
var data = xmlhttp.responseText;
if (xmlhttp.status == 200) {
that.debug('downloadSettings: SUCCESS');
[...]
} else {
that.debug('downloadSettings:');
that.debug('-- Error: ");
that.debug('-- ResponseText: "'+data+'"')
}
}
}
//Polling function
function checkEvent(){
if(ajaxFinished == true){
//your code i.e xmlhttp.open("GET", requestUrl, true);
}
clearInterval(chkeventid);//Clear Interval via ID for single time execution
}
var chkeventid=self.setInterval("checkEvent()",100);//The poll call
The setInterval method is treated a bit differently in JS as you know so you may use it as against the while loop.

with calling web services from javascript functions

while running these functions, I am calling such web service through which I have to generate session Id.
consider url is correct
I want to know,that I am calling function from onreadystatechange.wheteher it is correct way.
if you have another way please reply.
function getData(_url) {
var xmlhttpRequest = null;
xmlhttpRequest = new XMLHttpRequest();
xmlhttpRequest.open("GET", _url, true);
xmlhttpRequest.send();
xmlhttpRequest.onreadystatechange = function() {
//alert(xmlhttpRequest.status);
if(xmlhttpRequest.readyState == 4)// 4: The Request is complete
{
var request = xmlhttpRequest.responseXML;
var items = request.getElementsByTagName("id")[0].firstChild.nodeValue;
var hashcode = GetHashCode(passwordvalue + items);
var strUrl = commonURL + 'data/' + userName + ';' + hashcode;
data1(strUrl, 'tagname');//calling another function to generate session id
}
}
}
function data1(_url, _tagName)
{
var xmlhttpRequest = null;
xmlhttpRequest = new XMLHttpRequest();
xmlhttpRequest.open("GET", _url, true);
xmlhttpRequest.send();
xmlhttpRequest.onreadystatechange = function()
{
if(xmlhttpRequest.readyState == 4 && xmlhttpRequest.status==200)// 4: The Request is complete
{
var request = xmlhttpRequest.responseXML;
//alert('items .....= '+ request);
var sessionid = request.getElementsByTagName(_tagName)[0].firstChild.nodeValue;
alert('session ID='+sessionid);
}
}
}
Thanks,

i would place xmlhttpRequest.send(); after the onreadystatechange function
Also when you using the GET method you send a null value, that holds zero