We Keep Coding

Web scraping dynamic content with phantomjs

Im trying to scrape data from a site that generates its data via a javascript but I cant seem to locate the invoking script. When I look at the page source there is an in-page script that has a variable which has an array of the data I wish to retrieve but there is another script which contains all the codes for the various companies I wish to retrieve data from. This is what i've tried so far:
var url = 'http://www.asx.com.au/asx/share-price-research/company/ZAM/details';
var page = require('webpage').create();
page.open(url, function (status) {
var digitalData = page.evaluate(function () {
return window.digitalData;
})page.then(function (digitalData) {
console.log
}
('DigtalData is ' + digitalData));
phantom.exit();
});
Unfortunately the above script doesnt output anything. The script which contains the codes for the companies I want to get the data from I think is this:
var locationPath = window.location.pathname.split('/');
var companyCode = locationPath[locationPath.length - 1].trim();
var sectorCodes = {
"MOQ": "soft",
"1PG": "soft",
"ONT": "heal",
"1ST": "heal",
"T3D": "food",
"TGP": "real",
"TIX": "real",
"TDO": "ener",
"DDD": "mate",
"3PL": "cons",
"4DS": "semi"
};
setTimeout(function () {
googletag.cmd.push(function () {
googletag.defineSlot('/76291182/ASX_leaderboard_com_info', [728, 90], 'div-gpt-ad-1450158832871-0').addService(googletag.pubads());
googletag.defineSlot('/76291182/ASX_MREC_com_info', [[300, 250], [300, 600]], 'div-gpt-ad-1450158832871-1').addService(googletag.pubads());
googletag.defineSlot('/76291182/ASX_MREC_lower_com_info', [300, 250], 'div-gpt-ad-1450158832871-2').addService(googletag.pubads());
googletag.defineSlot('/76291182/ASX_skyscraper_com_info', [160, 600], 'div-gpt-ad-1450158832871-3').addService(googletag.pubads());
googletag.defineSlot('/76291182/ASX_half_page_com_info', [300, 600], 'div-gpt-ad-1450158832871-4').addService(googletag.pubads());
googletag.pubads().setTargeting("cc", companyCode);
if (typeof sectorCodes[companyCode] != 'undefined') {
googletag.pubads().setTargeting('sec', sectorCodes[companyCode]);
}
googletag.enableServices();
});
}, 2000);
This script seems to do a lot more than just contain the codes for the companies but its the only JS script I can find that has the company codes. I believe the data I want is from this script:
var currentURL = (document.URL);
var part = currentURL.split("/")[6];
// var dwsDTM = $('#company-code-title').text();
var digitalData = {
"page": {
"pageInfo": {
"pageID": "3345",
"pageName": "Company info " + part,
"pageURL": window.location.href,
"issueDate": "n/a",
"updatedDate": "n/a",
"brand": "ASX",
"generator": "OpenText",
"domain": "Website",
"sysEnv": "",
"delayType": "Normal"
},
"category": {
"primaryCategory": "Prices and research",
"subCategory1": "Company information",
"subCategory2": "Company info " + part,
"subCategory3": "",
"pageType": ""
},
"productInfo": {}
},
"user": {
"profileInfo": {
"memberB2B": "",
"businessMemberID": "",
"memberRetail": "",
"retailMemberID": ""
},
"version": "1.0",
"events": [],
"vendor": {
"GoogleAnalytics": {
"account": "UA-9950793-3",
"eventCategory": ""
}
}
}
};
I believe its generated to this HTML:
<div class="view-content" ui-view></div>
Im new to scraping dynamic content and this looks so hard to do. Could someone point me in the right direction to retrieve this data?
Thanks
UPDATE: So after a few hours of playing on the site I figured out that they use angularJS for their data call backs and upon further playing around I found the links they use to store their data. They have an API but I dont think its publicly available. In any case I think I can write a simple DOM Script to retrieve and format the data. Will keep you all posted

I figured out a solution. Utilising their API and using HTML DOM and PHP I was able to retrieve the data I wanted and then output the data to CSV. I cant discuss my code or the API because it would be in violation of their terms and service.

Proper display of layered json data

I have json that looks like this:
[
{
"Id": 1,
"Name": "Item1",
"Order": 1,
"Categories": [
{
"Id": 1,
"Name": "Item1-Subitem1",
"Order": 1,
"Subcategories": [
{
"Id": 1,
"Name": "Item1-Subitem1-Subsubitem1",
"Order": 2
},
{
"Id": 2,
"Name": "Item1-Subitem1-Subsubitem2",
"Order": 1
},
...
and with angular I need to display on first page (or route) link on main items. For example:Item1Item2And when clicked on them links should be displayed with names from 'Categories'For example:Item1-Subitem1Item1-Subitem2And when clicked on these links there should be displayed links with name from 'Subcategories'For example:Item1-Subitem1-Subsubitem1Item1-Subitem1-Subsubitem2
Now my first links work but I don't know how to get that nested data from same json according to value from url. When I click on links in first view I go to page I need but without data. No errors is displayed. I belive I need to parse id from url but how to achieve that?
It can be checked here http://plnkr.co/edit/GTfLFQepFcXzXYcIHnP0I am using ui-router

In secondList.js, you've defined your scope with the wrong name. It should be:
scope: {
secondList: '='
},
After you fix that, you're also missing the final factory in service.js:
app.factory('singleList', ['$http', function ($http) {
return {
get: function () {
return $http.get('data.json').then(function (re) {
return re.data;
});
}
};
}])

d3pie error: no data supplied on d3pie.js

I am using rails, and gathering some data to make pie charts. I am just using ruby objects (so no JSON), and using d3pie. First, I make a helper function. Then I make a javascript function using that helper, and pass it in the dom. Here's my code;
helper.rb
def options_data_to_d3(options_data)
d3_data = []
options_data.each do |key, value|
d3_data.push( { label: key.option.as_json, value: value.as_json } )
end
return JSON.pretty_generate(d3_data)
end
this takes the ruby hash, and makes it into json
js function
function dataPieChart(id, data) {
var config = {
"header": {
"title": {
"text": "Quiz Questions",
"fontSize": 18,
"font": "verdana"
},
"size": {
"canvasHeight": 400,
"canvasWidth": 500
},
"data": {
"content": data
},
"labels": {
"outer": {
"pieDistance": 32
}
}
}
}
var pie = new d3pie(id, config);
}
passing into the view
<div id="quizQuestionOptionPie<%= question.id %>"></div>
<script type="text/javascript">dataPieChart("quizQuestionOptionPie<%= question.id %>", <%= raw options_data_to_d3(data[:options]) %>);
</script>
when I call a console log in the javascript function to see what data is, I get the correct output that both d3pie and d3 are looking for, yet I am still getting the error
d3pie error: no data supplied.
does anyone see something wrong with my code, or something I am missing? any help is appreciated.

You config option is bracketed incorrectly (everything is a child of "header"). You really meant:
var config = {
"header": {
"title": {
"text": "Quiz Questions",
"fontSize": 18,
"font": "verdana"
},
},
"size": {
"canvasHeight": 400,
"canvasWidth": 500
},
"data": {
"content": data
},
"labels": {
"outer": {
"pieDistance": 32
}
}
};

Javascript/Jquery push json objects into array

I have an json object coming from a $.post in jquery.
In order to loop through and store the data clientside I would like to add it to an array. For each search results that comes back I would like to "append" the array so it grows.
This is my json:
{
"companies": [
{
"companyid": "115",
"saved": false,
"orgnumber": "010101010",
"companyname": "TestCompany",
"header": "header info"
},
{
"companyid": "116",
"saved": false,
"orgnumber": "010101010",
"companyname": "TestCompany",
"header": "header info"
} ]
}
This is what I have come up with so far were data is the json
comming back fron the post ajax request. Obj is just an object holding the array
which I declared further up in my code. obj.companies = new Array();
obj.companies.push(data['companies']);
The next part I need to loop out the array. Trying to do it like this.
$.each(obj.companies, function(i, item) {
// Does not alert correctly.
alert(item.header);
});
So I need to push the full json object into the array. But I cannot alert the item.header within the loop, how can I accomplish this?
EDIT:
Thanks everyone. Sorry if my question wasnt detailed enough.
I ended up doing this:
getcompanies: function() {
obj = this;
$.post('api/finder/result.php', {}, function(data) {
$.each(data.companies, function(i, item) {
obj.companies.push(item);
});
obj.loadcompanies();
}, "json");
},
loadcompanies: function() {
$.each(this.companies, function(i, item) {
alert(item.header);
}
}

I believe there is a issue with your server side code which is responsible for building JSON object which is getting returned via Ajax. The Correct JSON should be as follows:
{
"companies": [
{
"companyid": "115",
"saved": false,
"orgnumber": "010101010",
"companyname": "TestCompany",
"header": "header info"
},
{
"companyid": "116",
"saved": false,
"orgnumber": "010101010",
"companyname": "TestCompany",
"header": "header info"
}
]
}
Please note that there is only single key with name "companies" which holds an array of objects. Please correct your server side code to get such valid JSON. You can use free online JSON validator tools such as http://jsonlint.com/ to validate your JSON objects.
Now once you get such response from server; you just need to do following steps to get the companies array (following code will go into $.post success handler):
var jsonResp = JSON.parse(postResponse); //postResponse is the success resp of $.post
var companiesArray = jsonResp.companies;
$.each(companiesArray , function (index, valueObj){
var compId = valueObj.companyid;
var isSaved = valueObj.saved;
});
I hope this will help you a bit.

if you want to append new company into your companies array you should
var json_obj = {
"companies": [
{
"companyid": "115",
"saved": false,
"orgnumber": "010101010",
"companyname": "TestCompany",
"header": "header info"
},
{
"companyid": "116",
"saved": false,
"orgnumber": "010101010",
"companyname": "TestCompany",
"header": "header info"
} ]
};
//adding new company into your companies array
json_obj.companies.push({
"companyid":"117",
"saved" : false,
"orgnumber": "20120313",
"companyname":"anotherCompany",
"header":"another header info"
});
//if you want to loop through your companies list you can:
json_obj.companies.map(function ( obj ){
console.log(obj.companyid);
console.log(obj.companyname);
etc..
});

How to write array which collects ng-click event in AngularJs?

I am working with a project, where I need to collect multiple items from user and send it to the server. There is list on my view, where user can click and select the items. My HTML looks like this,
HTML
<div ng-repeat="topicList in searchCtrl.topic">
<div ng-repeat="topicTerm in topicList">
<p>{{topicTerm.number}}&nbsp&nbsp{{topicTerm.name}}</p>
<div ng-repeat="subTopic in topicTerm.subTopics">
{{subTopic.number}}&nbsp&nbsp{{subTopic.name}}
</div>
</div>
</div>
I have used anchor tag, there user can click and at the same time I want the clicked items (which have also unique ID) collected in an Array or variable, which I need to send (these selected items) to the server via form submission.
This is how my controller looks like,
JavaScript Controller
angular.module('myApp').controller("searchController", function($log, searchService, $scope){
var self = this;
self.initializeSearch = function(){
self.searchEntry =
{
"contact":{
"person": "",
"organization": ""
},
"request": {
"input": "",
"language": "en"
},
"topicIds": []
};
// The POST request must looks like above
What I want is that the clicked subTopics IDs collects in an Array "topicIds : []" and I could successfully send the POST request mentioned above. The searchService is a Angular service which helps to get Topics from server and also to POST user input to the server.
This is how my JSON looks like,
JSON API
{
"TopicList" :[
{
"id": "798790fa-78c8-4f00-8179-9e70f40adb14",
"name": "Topic1",
"number": 1.0,
"subTopics": [
{
"id": "82c90f2e-deac-4fa4-80f4-d077edacc2dc",
"name": "data1.1",
"number": 1.1
},
{
"id": "0f0c2b89-6dae-4f60-90f8-df49d96b9af9",
"name": "data1.2",
"number": 1.2
},
{
"id": "131b68b6-1f45-477f-9b0f-8ac80c5b4f4e",
"name": "data1.3",
"number": 1.3
},
{
"id": "16c8f46d-d20c-48f9-a0c0-e3989763082b",
"name": "data1.4",
"number": 1.4
}
]
},
{
"id": "9ed3fee0-5347-4f00-9b56-721b61439f88",
"name": "Topic2",
"number": 2.0,
"subTopics": [
{
"id": "eec13511-1408-4f4b-be6f-8b5a8b6ea28b",
"name": "data2.1",
"number": 2.1
},
...
]
},
...
]
}
How to write a function or array which collects the IDs via ng-click event?
Thanks in Advance.

No need to use an $event, simple pass the subTopic.id, or whatever, in your ng-click, like ng-click="searchCtrl.select(subTopic)"
And then in your controller, you could have:
angular.module('myApp').controller("searchController", function($log, searchService, $scope){
var self = this;
var subTopicIds = []; // array to hold subTopicIds
self.select = function(subTopic) {
subTopicIds.push(subTopic.id);
}
self.initializeSearch = function(){
self.searchEntry =
{
"contact":{
"person": "",
"organization": ""
},
"request": {
"input": "",
"language": "en"
},
"topicIds": subTopicIds // use the object created previously
};
...

You can get an ID in angular like this.
<div ng-click="recordClick($event)">Click</div>
That will feed the click event into the recordClick method, where you can then call it's target property (i.e. the div it was invoked on) and push it in the array.
$scope.clickArray = [];
$scope.recordClick = function(event){
clickArray.push(event.target);
}

I solved this problem by passing subTopics ID in ng-click as a parameter. And as per the requirement I need to call also another event while user click, which I passed as a second argument. So, now both the events works as I wanted via single ng-click.
Here is my updated code,
HTML
<div ng-repeat="topicList in searchCtrl.topic">
<div ng-repeat="topicTerm in topicList">
<p>{{topicTerm.number}}&nbsp&nbsp{{topicTerm.name}}</p>
<div ng-repeat="subTopic in topicTerm.subTopics">
{{subTopic.number}}&nbsp&nbsp{{subTopic.name}}
</div>
</div>
</div>
And here is my controller,
Controller
angular.module('myApp').controller("searchController", function($log, searchService, $scope){
var self = this;
var subTopicIDs = [];
self.select = function(TopicIDs, event){
subTopicIDs.push(TopicIDs);
$(event.target).addClass('selor'); // This is class which changes the background color of the clicked item
console.log(TopicIDs);
}
self.initializeSearch = function(){
self.searchEntry =
{
"contact":{
"person": "",
"organization": ""
},
"request": {
"input": "",
"language": "en"
},
"topicIds": subTopicIDs
};
This is how it solved my problem.
Btw, Thank you Tom and OceansOnPluto.