I am trying to get specific values from the general site that contains the annual reports of public-listed firms using Selenium Webdriver JS. Kindly help me investigate. I get the issue on clicking the Annual Report link.
The process that I'm trying to automate is listed below:
Go to site (http://edge.pse.com.ph/financialReports/form.do)
Filter by type of report (sendKeys('Annual Report') and fromDate attribute value to 4-23-2012 (year 2012 to get 4-5 annual reports of each firm)
click btnSearch
click Annual Report link per company row to view report (a tag is inside a td)
switch to the popup window
get specific values as object values and store it in the array
close popup window and get back to initial window
Repeat it on the succeeding company rows (take note that the site has pagination and a total of 22 pages (or tables) must be looped through
Kindly check my initial code for the process below. I also think that my implicitWaits are working. Does it apply on Selenium executeScript commands?
var webdriver = require('selenium-webdriver'),
By = webdriver.By,
until = webdriver.until;
var driver = new webdriver.Builder()
.forBrowser('chrome')
.build();
driver.get('http://edge.pse.com.ph/financialReports/form.do');
driver.wait(until.titleIs('Financial Reports'), 1000)
driver.manage().timeouts().implicitlyWait(5000);
driver.findElement(By.name('tmplNm')).sendKeys('Annual Report');
driver.executeScript("document.getElementsByName('fromDate').setAttribute('value', '4-23-2012')");
driver.findElement(By.id('btnSearch')).click();
var totalPages = 22;
var num = 0;
while (num < totalPages) {
var pagingArray = driver.executeScript("document.querySelector('.paging').getElementsByTagName('span')");
for (var p = 0; p < pagingArray.length; p++) {
var page = driver.executeScript("pagingArray[p].getElementsByTagName('a')[0]");
page.click();
var table = driver.executeScript("document.querySelector('table.list')");
for (var i = 0; i < table.rows.length; i++) {
for (var j = 0; j < table.rows[i].cells.length; j++) {
if (j == 1) {
// Store the current window handle
var winHandleBefore = driver.getWindowHandle();
var reportLink = driver.executeScript("table.rows[i].cells[j].getElementsByTagName('a')[0]");
reportLink.click();
// Switch to new window opened
for (var winHandle in driver.getWindowHandles()) {
driver.switchTo().window(winHandle);
}
var companyName = driver.executeScript("document.getElementById('companyName').innerHTML");
var stockTicker = driver.executeScript("document.getElementById('companyStockSymbol').innerHTML");
checkCompany(companyName, stockTicker);
var tableBS = driver.executeScript("document.getElementById('BS')");
var currentYear = driver.executeScript("tableBS.rows[1].cells[1].getElementsByTagName('span')[0]");
if (!checkBalanceSheets(companyName, currentYear)) {
bsCurrentYear = new Object();
bsCurrentYear.year = currentYear;
bsCurrentYear.currentAssets = driver.executeScript("tableBS.rows[2].cells[1].getElementsByTagName('span')[0]");
bsCurrentYear.totalAssets = driver.executeScript("tableBS.rows[3].cells[1].getElementsByTagName('span')[0]");
bsCurrentYear.currentLiabilities = driver.executeScript("tableBS.rows[4].cells[1].getElementsByTagName('span')[0]");
bsCurrentYear.totalLiabilities = driver.executeScript("tableBS.rows[5].cells[1].getElementsByTagName('span')[0]");
bsCurrentYear.company = companyName;
balanceSheets.push(bsCurrentYear);
}
var previousYear = driver.executeScript("tableBS.rows[1].cells[2].getElementsByTagName('span')[0]");
if (!checkBalanceSheets(companyName, previousYear)) {
bsPreviousYear = new Object();
bsPreviousYear.year = previousYear;
bsPreviousYear.currentAssets = driver.executeScript("tableBS.rows[2].cells[2].getElementsByTagName('span')[0]");
bsPreviousYear.totalAssets = driver.executeScript("tableBS.rows[3].cells[2].getElementsByTagName('span')[0]");
bsPreviousYear.currentLiabilities = driver.executeScript("tableBS.rows[4].cells[2].getElementsByTagName('span')[0]");
bsPreviousYear.totalLiabilities = driver.executeScript("tableBS.rows[5].cells[2].getElementsByTagName('span')[0]");
bsPreviousYear.company = companyName;
balanceSheets.push(bsPreviousYear);
}
var tableIS = driver.executeScript("document.getElementById('IS')");
if (!checkIncomeStatements(companyName, currentYear)) {
isCurrentYear = new Object();
isCurrentYear.year = currentYear;
isCurrentYear.grossrevenue = driver.executeScript("tableIS.rows[4].cells[1].getElementsByTagName('span')[0]");
isCurrentYear.grossexpense = driver.executeScript("tableIS.rows[7].cells[1].getElementsByTagName('span')[0]");
isCurrentYear.netincomeaftertax = driver.executeScript("tableIS.rows[10].cells[1].getElementsByTagName('span')[0]");
isCurrentYear.company = companyName;
isCurrentYear.eps = driver.executeScript("tableIS.rows[12].cells[1].getElementsByTagName('span')[0]");
incomeStatements.push(isCurrentYear);
}
if (!checkIncomeStatements(companyName, previousYear)) {
isPreviousYear = new Object();
isPreviousYear = previousYear;
isPreviousYear.grossrevenue = driver.executeScript("tableIS.rows[4].cells[2].getElementsByTagName('span')[0]");
isPreviousYear.grossexpense = driver.executeScript("tableIS.rows[7].cells[2].getElementsByTagName('span')[0]");
isPreviousYear.netincomeaftertax = driver.executeScript("tableIS.rows[10].cells[2].getElementsByTagName('span')[0]");
isPreviousYear.company = companyName;
isPreviousYear.eps = driver.executeScript("tableIS.rows[12].cells[2].getElementsByTagName('span')[0]");
incomeStatements.push(isPreviousYear);
}
// Close the new window, if that window no more required
driver.close();
// Switch back to original browser (first window)
driver.switchTo().window(winHandleBefore);
}
}
}
}
driver.executeScript("document.querySelector('.paging').getElementsByTagName('a')[2].click()");
num++;
}
PS CheckBalanceSheets and CheckIncomeStatements are for validating IF the year already exists and has been recorded before for the respective company
function checkCompany(name, ticker) {
var found = companies.some(function(el) {
return el.name === name;
});
if (!found) { companies.push({ name: name, ticker: ticker }); }
}
function checkBalanceSheets(name, year) {
var found = balanceSheets.some(function(el) {
return el.name === name && el.year === year;
});
}
function checkIncomeStatements(name, year) {
var found = incomeStatements.some(function(el) {
return el.name === name && el.year === year;
});
}
I am using the below JavaScript code to get the title of all lists in SharePoint 2013. How can i adapt this to return the URL of each list as well?
I've tired this but it doesn't work:
//listUrl = oList.get_url();
//console.log(listUrl);
Code:
function retrieveAllListProperties() {
var clientContext = new SP.ClientContext('/StrategicProjectOffice');
var oWebsite = clientContext.get_web();
this.collList = oWebsite.get_lists();
clientContext.load(collList);
clientContext.executeQueryAsync(
Function.createDelegate(this, this.onQuerySucceeded)
);
}
function onQuerySucceeded() {
var listTitle = '';
var listEnumerator = collList.getEnumerator();
while (listEnumerator.moveNext()) {
var oList = listEnumerator.get_current();
listTitle = oList.get_title();
//listUrl = oList.get_url();
//console.log(listUrl);
if (listTitle.indexOf("SPO") >= 0) {
getItemsFromView(listTitle, "All Tasks");
}
}
}
List Url could be retrieved via SPList.RootFolder property, in your example the line:
clientContext.load(collList);
needs to be replaced with
clientContext.load(collList,'Include(RootFolder.ServerRelativeUrl)');
which tells to construct a query to return RootFolder.ServerRelativeUrl property of List object.
Example
Here is my version which retrieves lists and prints its url:
var ctx = SP.ClientContext.get_current();
var web = ctx.get_web();
var lists = web.get_lists();
ctx.load(lists,'Include(RootFolder.ServerRelativeUrl)');
ctx.executeQueryAsync(
function () {
for(var i = 0; i < lists.get_count(); i++){
var list = lists.getItemAtIndex(i);
var listUrl = list.get_rootFolder().get_serverRelativeUrl();
console.log(listUrl);
}
},
function(sender,args){
console.log(args.get_message());
}
);
I am encountering a performance issue trying to fill in a grid with some data from a webservice.
Thanks to the Timeline analyzer from Chrome, I have found the function wich is causing my performance issue. It's due to a loop to get all the data from an xml and converting it into a JSON.
I've tried different loop (each, for, while) expecting improvement but didn't find better than each.
Here is an example of the xml with one child (but in my case there can be 10000 childs):
<DockDistrSearchOutput>
<groupRecord>
<uniqueIdentifier><![CDATA[KP789281]]></uniqueIdentifier>
<supplierCode><![CDATA[10003]]></supplierCode>
<supplierDescription><![CDATA[GROSFILLEX]]></supplierDescription>
<supplierCommercialContractCode><![CDATA[CTCOM01]]></supplierCommercialContractCode>
<supplierCommercialContractDescription><![CDATA[STANDARD EN EURO]]></supplierCommercialContractDescription>
<supplierAddressChainCode><![CDATA[1]]></supplierAddressChainCode>
<supplierAddressChainDescription><![CDATA[FIL1]]></supplierAddressChainDescription>
<articleCode><![CDATA[13A42001]]></articleCode>
<articleDescription><![CDATA[Banane]]></articleDescription>
<logisticVariantCode><![CDATA[2]]></logisticVariantCode>
<logisticVariantDescription><![CDATA[VL0 BANANE C G100]]></logisticVariantDescription>
<weightFactor><![CDATA[0]]></weightFactor>
<purchasePricePerKilo><![CDATA[13]]></purchasePricePerKilo>
<amount><![CDATA[0]]></amount>
<startDate><![CDATA[2016-07-29]]></startDate>
<endDate><![CDATA[2016-07-31]]></endDate>
<crossDockDiscount />
<crossDockMargin />
<extraMargin><![CDATA[3]]></extraMargin>
<listPrice />
<costPriceRD />
<oldSupplierCode><![CDATA[10003]]></oldSupplierCode>
<oldSupplierCommercialContractCode><![CDATA[CTCOM01]]></oldSupplierCommercialContractCode>
<oldSupplierAddressChainCode><![CDATA[1]]></oldSupplierAddressChainCode>
<oldArticleCode><![CDATA[13A42001]]></oldArticleCode>
<oldLogisticVariantCode><![CDATA[2]]></oldLogisticVariantCode>
<oldStartDate><![CDATA[2016-07-29]]></oldStartDate>
<oldEndDate><![CDATA[2016-07-31]]></oldEndDate>
</groupRecord>
</DockDistrSearchOutput>
And here is the loop used to get the data (15900ms for my test case) :
function getDataFromXml(xml){
var data = [];
$(xml).children().each( function (index) {
uniqueNumber++;
var recStatus = ORIGINAL;
if(this.getAttribute("recordState")!=null){
recStatus = this.getAttribute("recordState");
}
this.setAttribute("recordState", recStatus);
this.setAttribute("lineNumber", uniqueNumber);
var record = [];
if(this.getAttribute("checkedLine") == "true"){
record["checkedLine"] = true;
}else
record["checkedLine"] = false;
// Build the array by reading XML Tagname/value
record["id"] = "ID"+uniqueNumber;
$(this).children().each(function () {
record[this.tagName] = this.text;
});
record["recordState"] = recStatus;
record["rowXML"] = this;
data.push(record);
record = null;
recStatus = null;
});
return data;
}
Here are other try to get the data faster:
function getDataFromXml(xml){
var data = [];
$(xml).children().each( function (index) {
uniqueNumber++;
var recStatus = ORIGINAL;
if(this.getAttribute("recordState")!=null){
recStatus = this.getAttribute("recordState");
}
this.setAttribute("recordState", recStatus);
this.setAttribute("lineNumber", uniqueNumber);
var record = [];
/* First try
var a = $(this).children();
var l = a.length;
while(l--){
record[a[l].tagName] = a[l].text;
}*/
//Second try
var tmp = "{", cur = null;
for(var i =0, ln=this.childElementCount; i<ln;++i)
{
cur = this.children[i];
tmp += '"'+cur.tagName+'":"'+cur.text.replace('"', '')+'",';
}
tmp +='}';
tmp = tmp.replace(',}', '}');
record=JSON.parse(tmp);
if(this.getAttribute("checkedLine") == "true"){
record["checkedLine"] = true;
}else
record["checkedLine"] = false;
// Build the array by reading XML Tagname/value
record["id"] = "ID"+uniqueNumber;
record["recordState"] = recStatus;
record["rowXML"] = this;
data.push(record);
tmp = null;
cur = null;
record = null;
recStatus = null;
});
return data;
}
A try with only JavaScript (9600ms in my test case):
function getDataFromXml(xml){
var data = [];
var a = xml.getElementsByTagName("groupRecord");
var l = a.length;
while(l--){
uniqueNumber++;
var recStatus = ORIGINAL;
var groupRec = a[l];
if(groupRec.getAttribute("recordState")!=null){
recStatus = groupRec.getAttribute("recordState");
}
groupRec.setAttribute("recordState", recStatus);
groupRec.setAttribute("lineNumber", uniqueNumber);
var record = [];
var b = groupRec.childNodes;
var j = b.length;
while(j--){
record[b[j].tagName] = b[j].text;
}
if(groupRec.getAttribute("checkedLine") == "true"){
record["checkedLine"] = true;
}else
record["checkedLine"] = false;
// Build the array by reading XML Tagname/value
record["id"] = "ID"+uniqueNumber;
record["recordState"] = recStatus;
record["rowXML"] = groupRec;
data.push(record);
tmp = null;
cur = null;
record = null;
recStatus = null;
groupRec = null;
}
return data;
}
If someone has any advice to increase the performance of my loop?
Thanks in advance!
Edit: Thanks to Jaromanda X I have improved the performance by using only JavaScript.
I have a javascript object as below
var mydata=[
{"Club":"Blackburn","EventNo":1,"Pnts":3,"CumPnts":0},
{"Club":"Blackburn","EventNo":2,"Pnts":1,"CumPnts":0},
{"Club":"Blackburn","EventNo":3,"Pnts":4,"CumPnts":0},
{"Club":"Preston","EventNo":1,"Pnts":2,"CumPnts":0},
{"Club":"Preston","EventNo":2,"Pnts":4,"CumPnts":0},
{"Club":"Preston","EventNo":3,"Pnts":2,"CumPnts":0},]
I want to update the object so that CumPnts contains a running points total for each Club as below
{"Club":"Blackburn","EventNo":1,"Pnts":3,"CumPnts":3},
{"Club":"Blackburn","EventNo":2,"Pnts":1,"CumPnts":4},
{"Club":"Blackburn","EventNo":3,"Pnts":4,"CumPnts":8},
{"Club":"Preston","EventNo":1,"Pnts":2,"CumPnts":2},
{"Club":"Preston","EventNo":2,"Pnts":4,"CumPnts":6},
{"Club":"Preston","EventNo":3,"Pnts":1,"CumPnts":7},]
Any help would be much appreciated
Here is a function that loops through the list and updates it after it's been added. But I suspect that the events come in one at a time so there could be another function that can look the cumPtns obj and take from that. Here is for the current list.
var cumData = {};
var mydata=[
{"Club":"Blackburn","EventNo":1,"Pnts":3,"CumPnts":0},
{"Club":"Blackburn","EventNo":2,"Pnts":1,"CumPnts":0},
{"Club":"Blackburn","EventNo":3,"Pnts":4,"CumPnts":0},
{"Club":"Preston","EventNo":1,"Pnts":2,"CumPnts":0},
{"Club":"Preston","EventNo":2,"Pnts":4,"CumPnts":0},
{"Club":"Preston","EventNo":3,"Pnts":2,"CumPnts":0}];
function updateMyData() {
for (var i = 0; i < mydata.length; i++) {
var item = mydata[i];
if(cumData[item.Club] == undefined) {
cumData[item.Club] = {};
cumData[item.Club] = item.Pnts;
} else {
cumData[item.Club] = cumData[item.Club] + item.Pnts;
}
mydata[i].CumPnts = cumData[item.Club];
};
console.log(mydata);
//if you want to return it you can have this line below. Otherwise the object is updated so you'll probably want to do something with it once it's updated. Call back maybe?
return mydata;
}
updateMyData();
The first time it encounters a team it adds it to an array and so does with the corresponding cumPnts, so we can keep track of whether we checked a team earlier or not.
var tmArr = [];
var cumArr = [];
for(var i = 0; i < mydata.length; i++) {
var elm = mydata[i];
var club = elm.Club;
var points = elm.Pnts;
var idx = tmArr.indexOf(club);
if(idx > -1) {
cumArr[idx] += points;
elm.CumPnts = cumArr[idx];
}
else {
elm.CumPnts = points;
tmArr[tmArr.length] = club;
cumArr[cumArr.length] = points;
}
}
jsfiddle DEMO
I'm getting HTML from a forum url, and parsing the post count of the user from their profile page. I don't know how to write the parsed number into the Google spreadsheet.
It should go account by account in column B till last row and update the column A with count.
The script doesn't give me any errors, but it doesn't set the retrieved value into the spreadsheet.
function msg(message){
Browser.msgBox(message);
}
function onOpen() {
var ui = SpreadsheetApp.getUi();
ui.createMenu("Update")
.addItem('Update Table', 'updatePosts')
.addToUi();
}
function getPostCount(profileUrl){
var html = UrlFetchApp.fetch(profileUrl).getContentText();
var sliced = html.slice(0,html.search('Posts Per Day'));
sliced = sliced.slice(sliced.search('<dt>Total Posts</dt>'),sliced.length);
postCount = sliced.slice(sliced.search("<dd> ")+"<dd> ".length,sliced.search("</dd>"));
return postCount;
}
function updatePosts(){
if(arguments[0]===false){
showAlert = false;
} else {
showAlert=true;
}
var spreadSheet = SpreadsheetApp.getActiveSpreadsheet();
var accountSheet = spreadSheet.getSheetByName("account-stats");
var statsLastCol = statsSheet.getLastColumn();
var accountCount = accountSheet.getLastRow();
var newValue = 0;
var oldValue = 0;
var totalNewPosts = 0;
for (var i=2; i<=accountCount; i++){
newValue = parseInt(getPostCount(accountSheet.getRange(i, 9).getValue()));
oldValue = parseInt(accountSheet.getRange(i, 7).getValue());
totalNewPosts = totalNewPosts + newValue - oldValue;
accountSheet.getRange(i, 7).setValue(newValue);
statsSheet.getRange(i,statsLastCol).setValue(newValue-todaysValue);
}
if(showAlert==false){
return 0;
}
msg(totalNewPosts+" new post found!");
}
function valinar(needle, haystack){
haystack = haystack[0];
for (var i in haystack){
if(haystack[i]==needle){
return true;
}
}
return false;
}
The is the first time I'm doing something like this and working from an example from other site.
I have one more question. In function getPostCount I send the function profileurl. Where do I declare that ?
Here is how you get the URL out of the spreadsheet:
function getPostCount(profileUrl){
var ss = SpreadsheetApp.getActiveSpreadsheet();
var thisSheet = ss.getSheetByName("List1");
var getNumberOfRows = thisSheet.getLastRow();
var urlProfile = "";
var sliced = "";
var A_Column = "";
var arrayIndex = 0;
var rngA2Bx = thisSheet.getRange(2, 2, getNumberOfRows, 1).getValues();
for (var i = 2; i < getNumberOfRows + 1; i++) { //Start getting urls from row 2
//Logger.log('count i: ' + i);
arrayIndex = i-2;
urlProfile = rngA2Bx[arrayIndex][0];
//Logger.log('urlProfile: ' + urlProfile);
var html = UrlFetchApp.fetch(urlProfile).getContentText();
sliced = html.slice(0,html.search('Posts Per Day'));
var postCount = sliced.slice(sliced.search("<dd> ")+"<dd> ".length,sliced.search("</dd>"));
sliced = sliced.slice(sliced.search('<dt>Total Posts</dt>'),sliced.length);
postCount = sliced.slice(sliced.search("<dd> ")+"<dd> ".length,sliced.search("</dd>"));
Logger.log('postCount: ' + postCount);
A_Column = thisSheet.getRange(i, 1);
A_Column.setValue(postCount);
};
}
You're missing var in front of one of your variables:
postCount = sliced.slice(sliced.search("<dd> ")+"<dd> ".length,sliced.search("</dd>"));
That won't work. Need to put var in front. var postCount = ....
In this function:
function updatePosts(){
if(arguments[0]===false){
showAlert = false;
} else {
showAlert=true;
}
There is no array named arguments anywhere in your code. Where is arguments defined and how is it getting any values put into it?