We Keep Coding

How to get html source of page, after all it scripts was called?

I'am trying to parse site. The site (i suppose) using scripts and data bases to load data from (dynamically?). And this is my problem... I am trying to grab data through C# (unfortunately i don't have access to code right now) or JS. And it seems like either C# and JS, get only template of the site, but don't wait until all scripts executed. So this is my question, is there any way to get ALL html source? Maybe call scripts somehow. Or make a request, wait for 10 seconds, and then write source html data into variable?
Here is my JS code.
function request(link)
{
var xhr = new XMLHttpRequest();
xhr.open('GET', link, true);
xhr.onreadystatechange = function() .
{console.log(xhr.readyState);};
xhr.send();
let data = xhr.responseText;
var tempDiv = document.createElement('div');
tempDiv.innerHTML = data.replace(/<script(.|\s)*?\/script>/g,
'');
return tempDiv;
}
function loadFile(url, timeout, callback)
{
var args = Array.prototype.slice.call(arguments, 3);
var xhr = new XMLHttpRequest();
xhr.ontimeout = function () {
console.error("The request for " + url + " timed out.");
};
xhr.onload = function() {
if (xhr.readyState === 4) {
if (xhr.status === 200) {
callback.apply(xhr, args);
} else {
console.error(xhr.statusText);
}
}
};
xhr.open("GET", url, true);
xhr.timeout = timeout;
xhr.send(null);
let data = xhr.responseText;
return data;
}
function showMessage (message) {
console.log(message + this.responseText);
}
function include(scriptUrl)
{
var xmlhttp = new XMLHttpRequest();
xmlhttp.open("GET", scriptUrl);
xmlhttp.onreadystatechange = function()
{
if ((xmlhttp.status == 200) && (xmlhttp.readyState == 4))
{
eval(xmlhttp.responseText);
}
};
xmlhttp.send();
let data = JSON.parse(xmlhttp.responseText);
var tempDiv = document.createElement('div');
tempDiv.innerHTML = data.replace(/<script(.|\s)*?\/script>/g,
'');
return tempDiv;
}
All this functions do not work as i want.

This isn't really practical - you're trying to load an HTML page, all associated scripts, then run them on the HTML page as if they were in a proper browser environment, but within your current browser session.
This sort of thing is feasible with the jsdom library if you were running on the server-side (NodeJS), because it simulates browser behaviour: https://github.com/jsdom/jsdom. So you could do
JSDOM.fromURL("https://example.com/", { runScripts: "dangerously" }).then(dom => {
console.log(dom.serialize()); //turn the page back into HTML
});
...to get the whole thing.

protractor : passing variable 'url Json file' in executeAsyncScript

I'm new in protractor and I hope to have help regarding how to pass a variable in the executeAsyncScript function used in protractor tests.
I'm testing an application and i need to have a JSON file that contain the labels translation (French and English), at first I could correctly recuperate the language of the user from a first json file(let call getUser.json). Second on depending of the userlanguage I need to pass dynamically the url to get the labels json file (let call lang_fr.json and lang_en.json):
based on the code, 3rd exemple I could access to any json file.
browser.executeAsyncScript(function() {
var callback = arguments[arguments.length - 1];
var xhr = new XMLHttpRequest();
var url='path_to/getUser';
xhr.open("GET", url , true);
xhr.onreadystatechange = function() {
if (xhr.readyState == 4 && xhr.status == 200) {
callback(xhr.responseText);
}
};
xhr.send('');
}).then(function(str) {
browser.params.lang=JSON.parse(str)['userLanguage'];
if(JSON.parse(str)['userLanguage']==='fr')
browser.params.url='path_to/lang_en.json';
else
browser.params.url='path_to/lang_fr.json';
UserLanguage.UserLanguage(); /// excuting the fct that call the second jsonfile
now I need to pass the browser.params.urlexecuteAsyncScript to get the json file labels So i do this in another export file:
var url = browser.params.url;
browser.logger.info(browser.params.url); // The url depending the user language is correctly displayed
browser.executeAsyncScript(function() {
var callback = arguments[arguments.length - 1];
var xhr = new XMLHttpRequest();
xhr.open('GET', url /*here to pass the url depending the user language*/, true);
xhr.onreadystatechange = function() {
if (xhr.readyState == 4) {
callback(xhr.responseText);
}
}
xhr.send()
}, url/*passed on second argument*/).then(function(jsonlabel) {
/// rest of the code to resolve the labels
I got that the url is not defined
I checked the following examples:
Executing Async Javascript in Protractor http://blog.ng-book.com/executing-async-javascript-in-protractor/
But in every example I got errors.
So could you please provide me with your suggestion how can I pass correctly the vaiable url in the script? note that when I put the absolute file path it work:
xhr.open('GET', 'path_to/lang_en.json', true);
But I need to pass it dynamically. Hope it's clear and I am here for further details.

So I tried to put the variable url on function(){ and it works!
{
var url = browser.params.url;
var languageUser = browser.params.lang
browser.logger.info(browser.params.url); // The url depending the user language is correctly displayed
browser.executeAsyncScript(function(url /// added here){
var callback = arguments[arguments.length - 1];
var xhr = new XMLHttpRequest();
xhr.open('GET', url, true); xhr.onreadystatechange = function() {
if (xhr.readyState == 4 && xhr.status == 200) {
callback(xhr.responseText);
}
}
xhr.send()
}, url).then(function(resJson) {//rest of the code

convert URL into file object using javascript only

Link given:
example.com/data/videos/videoname.mp4
How to pass this link as fileInput?
var fileUrl = window.URL.createObjectURL(fileInput);
All should be done in javascript only.
Need a solution in pure javascript only not using any jquery.

You can use ajax and get blob
var url = 'http://example.com/data/videos/videoname.mp4';
var xhr = new XMLHttpRequest();
xhr.open('GET', 'blob:'+url, true);
xhr.responseType = 'blob';
xhr.onload = function(e) {
if (this.status == 200) {
var myObject = this.response;
}
};
xhr.send();

Tranfer data from local variable to global in javascript

I have some problem with transfer of variable outside the function.
It's seems to be very simple but I have some problem with it.
var myJson;
var url = "https://openbook.etoro.com/api/Markets/Symbol/?name=" + symbol;
var xhr = (window.XMLHttpRequest ? new XMLHttpRequest() : new ActiveXObject("Microsoft.XMLHTTP"));
xhr.onreadystatechange = XHRhandler;
xhr.open("GET", "proxy.php?url=" + url, true);
xhr.send(null);
function XHRhandler() {
if (xhr.readyState == 4) {
var json;
if (JSON && JSON.parse) {
json = JSON.parse(xhr.responseText);
} else {
eval("var json = " + xhr.responseText);
}
console.log(json);
myJson= json;
xhr = null;
}
}
console.log(myJson);
What I need is to pass the data from local variable json to global myJson;
But when i do console.log(myJson) i get undefined.
What is the problem?
Thank you

Try moving the statement console.log(myJson); inside your if condition or alternately initialize your variable with some value. It seems your statement is getting called before it is getting populated with any value.

The XMLHttpRequest is async so it is not done yet when you try to write the myJson variable to console. Wrap it in a function and call that function after the XMLHttpRequest is completed instead.
var myJson;
var url = "https://openbook.etoro.com/api/Markets/Symbol/?name=" + symbol;
var xhr = (window.XMLHttpRequest ? new XMLHttpRequest() : new ActiveXObject("Microsoft.XMLHTTP"));
xhr.onreadystatechange = XHRhandler;
xhr.open("GET", "proxy.php?url=" + url, true);
xhr.send(null);
function XHRhandler() {
if (xhr.readyState == 4) {
var json;
if (JSON && JSON.parse) {
json = JSON.parse(xhr.responseText);
} else {
eval("var json = " + xhr.responseText);
}
console.log(json);
myJson= json;
xhr = null;
writeToConsole();
}
}
function writeToConsole() {
console.log(myJson);
}

Running JavaScript downloaded with XMLHttpRequest

I have a site that loads information using the XMLHttpRequest when a user clicks a link. The system works well but I would like to be able to execute JavaScript gathered in this process.
This is a problem as I would like to download the scripts 'on demand' if it were, rather than loading them all when the page is loaded.
Thanks for any help

I believe the recommended solution is something like this:
function include(scriptUrl)
{
var xmlhttp = new XMLHttpRequest();
xmlhttp.open("GET", scriptUrl);
xmlhttp.onreadystatechange = function()
{
if ((xmlhttp.status == 200) && (xmlhttp.readyState == 4))
{
eval(xmlhttp.responseText);
}
};
xmlhttp.send();
}
Or something like it.
However, be wary of this approach. It's vulnerable to cross-site scripting, which can open you (and your users) up to all sorts of nastiness. You'll want to take suitable precautions.

Recently I found the answer (It works in Chrome, in another browsers it was not tested).
You can create dataURL string and put it into src attribute of script element.
var xhr = XMLHttpRequest(),
doc = document;
xhr.open('GET', pathToJSFile, true);
xhr.onload = function () {
var script = doc.createElement('script'),
base64 = 'data:application/javascript;base64,';
try {
base64 += btoa(data.responseText);
} catch (e) {
// script file may contain characters that not included in Latin1
var symbols = data.responseText.split('');
for (var i = 0, l = symbols.length; i < l; i++) {
var symbol = symbols[i];
// here we are trying to find these symbols in catch branch
try {
btoa(symbol);
} catch (e) {
var code = symbol.charCodeAt(0).toString(16);
while (code.length < 4) {
code = '0' + code;
}
// replace original symbol to unicode character
symbols[i] = '\\u' + code;
}
}
// create new base64 string from string with replaced characters
base64 += btoa(symbols.join(''));
} finally {
script.src = base64;
// run script
doc.body.appendChild(script);
}
};
xhr.send();
You can subscribe to xhr.onprogress to show progress bar.
Update. You can download your script file as blob, and then create blob-url.
var xhr = XMLHttpRequest(),
doc = document;
xhr.responseType = 'blob';
xhr.open('GET', pathToJSFile, true);
xhr.onload = function () {
var script = doc.createElement('script'),
src = URL.createObjectURL(xhr.response);
script.src = src;
doc.body.appendChild(script);
};
xhr.send();

You can run script downloaded in form of a string using
eval()
However I would recommend you to add new
<script src='..'></script>
to your document and have a callback which will be called when it will be downloaded. There are many utils and jquery plug-ins for that.

I had the challenge on a mobile web-project, the magic was to set "overrideMimeType".
This has been verified to work on Android 4.1 to Android 6.0.
var head = document.getElementsByTagName('head');
var injectedScript = document.createElement('script');
head[0].appendChild(injectedScript);
var myRequest = new XMLHttpRequest();
myRequest.onreadystatechange = function() {
if (myRequest.readyState == 4 && myRequest.status == 200) {
injectedScript.innerHTML = myRequest.responseText;
//run a function in the script to load it
}
};
function start(){
myRequest.open('GET', 'javascript-url-to-download', true);
myRequest.overrideMimeType('application/javascript');
myRequest.send();
}
start();

You would need to use eval to parse the javascript from the XHR, note that this is EXTREMELY dangerous if you don't have absolute trust in the source of the javascript.