I'm trying to get 3 components to all websites:
Website title
Description
Image/og:image
The way I thought would work was using ajax to fetch the website and then read the meta data, but it doesn't seem to work consistently..
$('#getResults').on('click', (event) => {
let postURL = $('#fetch').val()
$.ajax({
url: 'https://cors-anywhere.herokuapp.com/' + postURL
}).then((data) => {
let html = $(data);
let website_data = {
title: getNameContent(html, 'og:title') || null,
description: getNameContent(html, 'og:description') || null,
image: getNameContent(html, 'og:image') || null,
url: postURL
}
console.log(website_data)
})
})
function getNameContent(html, name) {
return html.filter((index, tag) => tag && tag.name && tag.name == name).attr('content');
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<p>
https://www.youtube.com/watch?v=dHHt_VBefeQ //doesn't work<br>
</p>
<input type="text" id="fetch" placeholder="url"/>
<button id="getResults">Search</button>
Some URL's work though, but others don't... Any idea how I can get consistent results? Thank you!
Related
I've been using Python and Selenium to try to automate an export process in Qualtrics. I have the web driver navigate to the login page, input my login credentials, and then navigate to the Data and Analytics page of the relevant survey. However, once I navigate to the Data and Analysis page, I'm having trouble selecting the element for the dropdown icon under the action header, which I need to click in order to export the responses.
I've tried to use this code (omitting the part where I login):
from selenium import webdriver
survey_url= {The Survey Url}
browser= webdriver.Chrome()
browser.get(survey_url)
browser.find_element_by_xpath("//div[#id='itemContext']").click()
However, I keep getting this error: NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//div[#id='itemContext']"}
I've tried using both the absolute (/html/body/div[6]) and the relative (//*[#id="itemContext"]) xpath given to me by inspect element, but neither worked.
This is the HTML that I see for the element:
<div id="itemContext" ng-click="removeSelf()" qmenu items="recordActions" on-select="selectRecordAction()(value,rowId, recordIndex)" class="response-context ng-scope ng-isolate-scope" style="left:1315px;top:357px">
<span ng-transclude></span></div>
Here is the HTML for the webpage:
<html ng-app="responses-client">
<head>
<meta charset="utf-8">
<title>Data | Qualtrics Experience Management</title>
<script>
(function (window) {
'use strict';
/*
* This function is used to attempt to get the initial definition before
* angular is loaded. If this fails for some reason the definition will
* still be loaded, but not via this function. See fast-definition-service.
*/
function rejectError(data, status, statusText) {
data = data || {};
window.fastDefinitionState = 'ERROR';
window.fastDefinitionError = {
status: status || 500,
statusText: statusText || 'Internal Server Error',
data: data,
config: {
method: 'GET',
url: url
}
};
}
function handleResponse() {
var status = request.status;
var data;
try {
data = JSON.parse(request.responseText);
} catch (e) {
data = request.responseText;
}
function handleSuccess() {
window.returnedDefinition = { data };
window.fastDefinitionState = 'SUCCESS';
}
if (status >= 200 && status < 400) {
return handleSuccess();
} else {
return rejectError(data, status, request.statusText);
}
}
function attemptRequest() {
request.open('GET', url, true);
request.onload = handleResponse; // Recieved response from server
request.onerror = rejectError; // Connection to server error
request.send();
window.fastDefinitionState = 'PENDING';
}
var surveyIdRegex = /surveys\/(SV_[a-zA-Z0-9]{15})/;
var canonicalId = (surveyIdRegex.exec(window.location.hash) || [])[1];
var fieldsetIdRegex = /fieldsets\/([\w-]+)/;
var fieldsetId = (fieldsetIdRegex.exec(window.location.hash) || [])[1];
var conjointIdRegex = /surveys\/SV_[a-zA-Z0-9]{15}\?conjointProjectId=(.+)/;
var conjointId = (conjointIdRegex.exec(window.location.hash) || [])[1];
var idpProjectRegex = /fieldsets\/[\w-]+\?(?:projectId|idpProjectId)=(IDP_.+|PROJ_.+)/;
var idpProjectId = (idpProjectRegex.exec(window.location.hash) || [])[1];
var resourceIdRegex = /surveys\/SV_[a-zA-Z0-9]{15}\?resourceId=(.+)/;
var resourceId = (resourceIdRegex.exec(window.location.hash) || [])[1];
var projectIdRegex = /surveys\/SV_[a-zA-Z0-9]{15}\?projectId=(.+)/;
var projectId = (projectIdRegex.exec(window.location.hash) || [])[1];
var url;
if (canonicalId) {
url = '/responses/surveys/' + canonicalId;
} else if (fieldsetId) {
url = '/responses/fieldsets/' + fieldsetId;
} else {
return rejectError(new Error('Missing surveyId for fast definition'));
}
if (conjointId) {
url += '?optProjectId=' + conjointId;
} else if (idpProjectId) {
url += '?idpProjectId=' + idpProjectId;
} else if (resourceId) {
url += '?resourceId=' + resourceId;
} else if (projectId) {
url += '?projectId=' + projectId;
}
window.fastDefinitionUrl = url;
var request = new XMLHttpRequest();
return attemptRequest();
})(window);
</script>
<link rel="icon" href="/brand-management/favicon">
<link rel="apple-touch-icon-precomposed" href="/brand-management/apple-touch-icon">
<link href="/responses/static/vendor/vendor.min.a113b2563.css" rel="stylesheet" type="text/css">
<link href="/responses/static/css/responsesclient.min.a49de54f5.css" rel="stylesheet" type="text/css">
</head>
<body ng-click="checkMenu()" ng-controller="MainController" keep-alive style="margin:0px;">
<div class="dp-container">
<div ng-view></div>
<div q-footer user-id="userId" language="userLanguage"></div>
</div>
</body>
<script src="/responses/static/vendor/vendor.min.a4f381856.js"></script>
<script src="/responses/static/js/responsesclient.min.ad8d40058.js"></script>
<script src="/wrapper/static/client.js"></script>
<script type="text/javascript" src="/project-workflow/static/bundle.js"></script>
<script type="text/javascript" src="https://static-assets.qualtrics.com/static/expert-review-ui/prod/response-iq.js"></script>
<script>
window.Qualtrics = {"User":{"brandId":"mindshare";
function hackAFixForProjectFramework() {
var surveyIdRegex = /surveys\/(SV_[a-zA-Z0-9]{15})/;
var fieldsetIdRegex = /fieldsets\/([\w-]+)/;
var canonicalId = (surveyIdRegex.exec(window.location.hash) || [])[1];
var fieldsetId = (fieldsetIdRegex.exec(window.location.hash) || [])[1];
window.Qualtrics.surveyId = canonicalId || fieldsetId;
}
// Hack a fix for updating to jquery 3.5 breaking project framework as they assumed it was ready sooner with jquery 3.5. Couldn't discover why, but it's somewhere in their code base.
hackAFixForProjectFramework();
var user = _.get(window, 'Qualtrics.User', {}),
requiredUserKeys = [
'userId',
'brandId',
'brandName',
'brandType',
'language',
'email',
'userName',
'firstName',
'lastName',
'accountType',
'accountCreationDate'
];
angular.module('responses-client').config([
'rsw.userServiceProvider',
function(userServiceProvider) {
try {
userServiceProvider.setUser(_.pick(user, requiredUserKeys));
} catch (e) {
console.log(e);
}
}
]);
</script>
</html>
I suspect the problem may be that the website is using a lot of javascript to dynamically change the responses. Above the HTML element I showed above, there are a bunch of "<'script type...'>" and src programs that I don't really understand, though I can guess from the names that they are handling the display of responses. Thinking it might be a loading problem, I've also tried making the program wait before searching for the element, but that didn't work either.
Cheers!
I have installed in my project django-contib-comments and I have an HTML that displays the list of comments and also displays the form to enter a new one.
I now want to use Ajax to submit the form without a page refresh and on success to add the submitted comment to the list.
I have done most of the work, but I'm sure there must be an easier way to achieve this.
my question is if there is a way for me to render a Django HTML tag within a javascript something like this:
document.getElementById("comments").innerHTML = {% render_comment_list for obj %}
so far this is the code I have done:
1) I don't want to change anything in the django-contrib-comments project (i am avoiding to override methods.
2) I used the standard tags in django-contrib-comments to render a list of comments.
{% render_comment_list for obj %}
3) Created a JavaScript that handles the submit of the form and then creates a new entry in the list.
function submit_comments(event) {
event.stopPropagation();
$.ajax({
type: $('#comment_form').attr('method'),
url: $('#comment_form').attr('action'),
data: $('#comment_form').serialize(),
cache: false,
dataType: "html",
success: function (html, textStatus) {
var comment_count_btn = document.getElementById('comment-text-vertical-btn');
if (comment_count_btn != null) {
if (!isNaN(parseInt(comment_count_btn.innerHTML))) {
comment_count_btn.innerHTML = parseInt(comment_count_btn.innerHTML) + 1 + " Comments";
}
}
var comment_count_head = document.getElementById('kapua-comments-header');
if (comment_count_head != null) {
if (!isNaN(parseInt(comment_count_head.innerHTML))) {
comment_count_head.innerHTML = parseInt(comment_count_head.innerHTML) + 1 + " Comments:";
}
}
if (document.getElementById("comments") != null){
submitted_timestamp = getQueryParameter("timestamp", this.data);
submitted_date = new Date();
if (submitted_timestamp == null) {
submitted_date = new Date(submitted_timestamp);
}
submitted_comment = getQueryParameter("comment", this.data);
if (submitted_comment == null) {
submitted_comment = "No value entered"
}
html_comment = "<div class=\"right-aligned\"><div class=\"comment-date\">" + submitted_date + " - " + "</div>" + submitted_comment + "</div><br><br>";
current_html = document.getElementById("comments").innerHTML;
document.getElementById("comments").innerHTML = html_comment + current_html;
}
if (document.getElementById("comment_form") != null){
document.getElementById("comment_form").reset();
}
},
error: function(XMLHttpRequest, textStatus, errorThrown) {
$('#comment_form').replaceWith('Your comment was unable to be posted at this time. We apologise for the inconvenience.');
}
});
return false;
};
Thanks in Advance
Yes there is way in django, You can use [escape][1] to render HTML tag. Here is an simple example -
from django.utils.html import escape
def example(request):
return "%s<div><i>%s</i></div>" % (escape(text), escape(other_text)
For more explanation you can refer to documentation.
https://docs.djangoproject.com/en/2.1/ref/utils/#django.utils.html.escape
Still have any doubts comment it out.
I have used Jsoup library to fetch the metadata from url.
Document doc = Jsoup.connect("http://www.google.com").get();
String keywords = doc.select("meta[name=keywords]").first().attr("content");
System.out.println("Meta keyword : " + keywords);
String description = doc.select("meta[name=description]").get(0).attr("content");
Elements images = doc.select("img[src~=(?i)\\.(png|jpe?g|gif)]");
String src = images.get(0).attr("src");
System.out.println("Meta description : " + description);
System.out.println("Meta image URl : " + src);
But I want to do it in client side using javascript
You can't do it client only because of the cross-origin issue. You need a server side script to get the content of the page.
OR You can use YQL. In this way, the YQL will used as proxy.
https://policies.yahoo.com/us/en/yahoo/terms/product-atos/yql/index.htm
Or you can use https://cors-anywhere.herokuapp.com. In this way, cors-anywhere will used as proxy:
For example:
$('button').click(function() {
$.ajax({
url: 'https://cors-anywhere.herokuapp.com/' + $('input').val()
}).then(function(data) {
var html = $(data);
$('#kw').html(getMetaContent(html, 'description') || 'no keywords found');
$('#des').html(getMetaContent(html, 'keywords') || 'no description found');
$('#img').html(html.find('img').attr('src') || 'no image found');
});
});
function getMetaContent(html, name) {
return html.filter(
(index, tag) => tag && tag.name && tag.name == name).attr('content');
}
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<input type="text" placeholder="Type URL here" value="http://www.html5rocks.com/en/tutorials/cors/" />
<button>Get Meta Data</button>
<pre>
<div>Meta Keyword: <div id="kw"></div></div>
<div>Description: <div id="des"></div></div>
<div>image: <div id="img"></div></div>
</pre>
Pure Javascript function
From node.js backend (Next.js) I use that:
export const fetchMetadata = async (url) => {
const html = await (await fetch(url, {
timeout: 5000,
headers: {
'User-Agent': 'request'
}
})).text()
var metadata = {};
html.replace(/<meta.+(property|name)="(.*?)".+content="(.*?)".*\/>/igm, (m,p0, p1, p2)=>{ metadata[p1] = decode(p2) } );
return metadata
}
export const decode = (str) => str.replace(/&#(\d+);/g, function(match, dec) {
return String.fromCharCode(dec);
})
You could use it on the client with https://cors-anywhere.herokuapp.com/corsdemo
You can use open-graph-scraper for this, for more info see this answer.
Am trying to create a login page for my windows 8 app, am using Html5 and javascript.. so have tried to use winjs.xhr to post what is in the textboxes as variables to a specific url which is a php script so this is my example of the url "http://example.com/api/username=username&password=password" am using winjs.xhr to post these variables to the url but am not getting any response even in the console.log
this is my code
<script>
function handlelogin(){
document.getElementById("box").onsubmit = function(){
if(document.getElementById("email_address").value ==""){
document.getElementById("errormessage").innerHTML= "Please Provide Your Email Address";
return false;
}else{
var email_address = document.getElementById("email_address");
var password = document.getElementById("password");
var formparams = "?username=" + email_address.value + "&password=" + password.value;
document.getElementById("errormessage").innerHTML = "";
WinJS.xhr({type: "POST",
url: "http://example.com/api/",
data: formparams,
headers: { "Content-type": "application/x-www-form-urlencoded" }
}).then(
function (success) {
console.log(success.statusText);
if(success == 1703){
WinJS.Navigation.navigate("home.html");
}
},
function (error) {
console.log(error);
}
);
}
};
}
window.onload = function () {
handlelogin();
}
</script>
<form id="box" method="post" name="loginform">
<p>Email address</p>
<div class="email_address"><input type="text" id="email_address" /></div>
<p>Password</p>
<div class="password"><input type="password" id="password" /></div>
<p><span id="errormessage"></span></p>
<div class="button"><input type="submit" id="login" value="Sign In"/></div>
<p>ForgotPassword?</p>
</form>
First - don't use a submit button. Use a input type="button". No submit required, you are simply reading the values on the page.
Second - attach the event handler for the button's click event. Doing this 'onload' for the window isn't the right place.
Third - don't use 'onsubmit' for your 'box' element. There is no form submission going on here. There shouldn't usually be a form submit in WinJS - that's for a browser posting the page to the server. You already are POSTing your data. See the updated code:
I highly recommend putting ALL javascript into separate files, as you'll get bytecode optimization that way. Inline Javascript isn't optimized for the next load. A common way you could do this is include the onload code below (where I assign onclick) in your js file like so
app.onactivated = function (args) {
..
..
..
args.setPromise(WinJS.UI.processAll().then(function () {
document.getElementById("login").onclick = handlelogin;
}));
..
..
}
};
But the answer directly for your question above is:
<script>
function handlelogin() {
if (document.getElementById("email_address").value == "") {
document.getElementById("errormessage").innerHTML = "Please Provide Your Email Address";
return false;
} else {
var email_address = document.getElementById("email_address");
var password = document.getElementById("password");
var formparams = "?username=" + email_address.value + "&password=" + password.value;
document.getElementById("errormessage").innerHTML = "";
WinJS.xhr({
type: "POST",
url: "http://example.com/api/",
data: formparams,
headers: { "Content-type": "application/x-www-form-urlencoded" }
}).then(
function (success) {
console.log(success.statusText);
if (success == 1703) {
WinJS.Navigation.navigate("home.html");
}
},
function (error) {
console.log(error);
}
);
}
}
window.onload = function() {
document.getElementById("login").onclick = handlelogin;
};
</script>
Check out some of the sessions in App Builder where they discuss JavaScript projects http://aka.ms/stackbuilder
I need to create a javascript function that will write a page based on the url, so basically I am trying to create a javascript function that will check the url, and find the corresponding xml item from there.
The reason behind this is so that the html page can just be duplicated, renamed, and the xml updated, and the page will fill in everything else from the xml sheet.
please let me know whether this is the completely incorrect way to do it, of it there is a better way. thanks!!!
XML CODE::
<!--XML INFORMATION-->
<channel>
<design>
<motion><!--design content-->
<item><!--//////////POST//////////POST//////////POST//////////-->
<tag>/portfolio_dec.html</tag>
<!--RSS INFORMATION-->
<title>Decoze</title>
<link>http://payamrajabi.com/portfoliotest.html</link>
<description><img src="http://payamrajabi.com/thumbs/small_jump.png" title="JUMP!." /></description>
<!--PROJECT CONTENT-->
<project><!--project start-->
<titl>TITLE</titl><!--project title-->
<dsc>PROJECT DESCRIPTION</dsc><!--project description-->
</project><!--project end-->
</item><!--//////////END//////////END//////////END//////////-->
</motion>
</design>
</channel>
JAVASCRIPT:
$(document).ready(function(){
$.ajax({
type: "GET",
url: "code/content9.xml",
dataType: "xml",
success: function(xml) {
var xpathname = window.location.pathname;
var xproject = $(xml).find('tag').text();
if (xpathname == xproject) {
$(xml).find('item').children('tag').text(xpathname).each(function(){
var ttl = $(this).find('titl').text();
$('<p>'+ttl+'</p>').appendTo('h1#ttl');
});
$(xml).find('item').children('tag').text(xpathname).each(function(){
var dsc = $(this).find('dsc').text();
$('<p>'+dsc+'</p>').appendTo('h1#ttl');
});
} else {
PUT ERROR MESSAGE HERE
}
}
});
});
and THE HTML:
<html>
<head>
<script type="text/javascript" src="code/jquery-1.3.1.js"></script>
<script type="text/javascript" src="code/project/project_design.js"></script>
</head>
<body>
<h1 id="ttl"></h1>
<p id="dsc"></p>
</body>
</html>
any help would really be appreciated, i am frairly new to javascript/jquery/xml, and am really having trouble with this. The primary thing I want to do is have an xml file that populates a site, with each item being the content for a new page, in this case of a portfolio item.
cheers!
willem
Hmm... I'm afraid you don't quite understand how jquery works.
Your code should look something like this:
var xpathname = window.location.pathname;
var xitem = $(xml).find('tag:contains(' + xpathname + ')').parent();
if (xproject.length != 0) {
$('#ttl').append('<p>' + xitem.find('titl').text() + '</p>');
$('#dsc').append('<p>' + xitem.find('dsc').text() + '</p>');
}
else {
$('#err').text('The page you requested does not exist');
}
Demo 1
Here's a quick demo. Take a look at the source to see the XML and the JavaScript.
http://jsbin.com/ujiho#itemOne
http://jsbin.com/ujiho#itemTwo
http://jsbin.com/ujiho#itemThree
Demo 2
I've created another demo that uses $.get to retrieve the XML from a separate URL.
http://jsbin.com/aqefo#nov
http://jsbin.com/aqefo#dec
The XML: http://jsbin.com/afiwa
Here's the JavaScript. Let me know if you need help understanding anything.
$(function(){
$.get(
'http://jsbin.com/afiwa',
function(xml){
var hash = window.location.hash.substring(1);
if ($.trim(hash) === '') {
showError();
return;
}
var xitem = $(xml).find('urlname:contains(' + hash + ')').parent();
if (xitem.length != 0) {
$('#ttl').append(xitem.find('titl').text());
$('#dsc').append( xitem.find('dsc').text());
}
else {
showError();
}
function showError() {
$('#err').text('The page you requested does not exist');
}
}
);
});