I am having difficulties with collecting data on changes in Spanish municipalities over time. I have been trying to use this answer as a guide, but I think I am running into problems because in order to move from one dropdown menu to another, a button needs to be clicked. I can't figure out where to work the button into the code.
What I have so far:
#access web page
remDr <- remoteDriver(browserName="firefox", port=4444)
remDr$open(silent=T)
remDr$setImplicitWaitTimeout(3000)
remDr$navigate("http://www.ine.es/intercensal/")
I can access the tables I want individually by entering values into each of three elements (Autonomous community, Province, and Municipality) and clicking buttons, for example:
remDr$executeScript("document.getElementById('cmbCCAA').value = 1;")
ConsultProv <- remDr$findElement(using = 'name', value = 'btnBuscarProv')
ConsultProv$clickElement()
remDr$executeScript("document.getElementById('cmbProv').value = 4;")
ConsultMuni <- remDr$findElement(using = 'name', value = 'btnBuscarMuni')
ConsultMuni$clickElement()
remDr$executeScript("document.getElementById('cmbMuni').value = 5861;")
SubmitMuni <- remDr$findElement(using = 'name', value = 'btnBuscarGeo')
SubmitMuni$clickElement()
And while the code may be inelegant (apologies, but I just began learning R), I can pull individual table data with:
readname <- remDr$findElement(using = 'css selector', value = '.TITULOH3')
namedata <- readname$getElementText()
cleanname1 <- gsub("Province: ", "", namedata)
separatename1 <- str_split(cleanname1, "Municipality: ")
separatename2 <- unlist(separatename1, recursive = TRUE, use.names = TRUE)
trimmed <- trimws(separatename2)
names(trimmed) <- data.frame("Province", "Municipality", stringsAsFactors = FALSE)
finalname <- t(trimmed)
table <- htmlParse(remDr$getPageSource()[[1]])
readdfdata <- readHTMLTable(table, skip.rows = c(1, 2, 4, 5), stringsAsFactors = FALSE)
tabledfdata <- as.data.frame(readdfdata[2])
names(tabledfdata) <- c("skip", "dfpop1842", "dfpop1857", "dfpop1860", "dfpop1877", "dfpop1887", "dfpop1897",
"dfpop1900", "dfpop1910", "dfpop1920", "dfpop1930", "dfpop1940", "dfpop1950", "dfpop1960",
"dfpop1970", "dfpop1981", "dfpop1991", "dfpop2001", "dfpop2011")
cleandftable <- subset(tabledfdata, select = -c(skip))
finaldfpop <- cleandftable
readdjdata <- readHTMLTable(table, skip.rows = c(1, 2, 3, 5), stringsAsFactors = FALSE)
tabledjdata <- as.data.frame(readdjdata[2])
names(tabledjdata) <- c("skip", "djpop1842", "djpop1857", "djpop1860", "djpop1877", "djpop1887", "djpop1897",
"djpop1900", "djpop1910", "djpop1920", "djpop1930", "djpop1940", "djpop1950", "djpop1960",
"djpop1970", "djpop1981", "djpop1991", "djpop2001", "djpop2011")
cleandjtable <- subset(tabledjdata, select = -c(skip))
finaldjpop <- cleandjtable
readhhdata <- readHTMLTable(table, skip.rows = c(1, 2, 3, 4), stringsAsFactors = FALSE)
tablehhdata <- as.data.frame(readhhdata[2])
names(tablehhdata) <- c("skip", "hh1842", "hh1857", "hh1860", "hh1877", "hh1887", "hh1897",
"hh1900", "hh1910", "hh1920", "hh1930", "hh1940", "hh1950", "hh1960",
"hh1970", "hh1981", "hh1991", "hh2001", "hh2011")
cleanhhtable <- subset(tablehhdata, select = -c(skip))
finalhhpop <- cleanhhtable
readchange <- remDr$findElement(using = 'css selector', value = '.alteraciones')
changedata <- readchange$getElementText()
separatechange1 <- str_split(changedata, "\\n ")
separatechange2 <- unlist(separatechange1, recursive = TRUE, use.names = TRUE)
finalchange <- t(separatechange2)
complete <- data.frame(finalname, finaldfpop, finaldjpop, finalhhpop, finalchange, stringsAsFactors = FALSE)
But how do I loop this process in order to collect data for all municipalities? I tried using the changeFun from the previous answer while also incorporating the buttons, but the following code gets stuck:
#Pre-existing changeFun
changeFun <- function(value, elementName, targetName){
changeElem <- remDr$findElement(using = "name", elementName)
script <- paste0("arguments[0].value = '", value, "'; arguments[0].onchange();")
remDr$executeScript(script, list(changeElem))
targetCodes <- c()
while(length(targetCodes) == 0){
targetElem <- remDr$findElement(using = "name", targetName)
target <- xmlParse(targetElem$getElementAttribute("outerHTML")[[1]])
targetCodes <- sapply(querySelectorAll(target, "option"), xmlGetAttr, "value")[-1]
target <- sapply(querySelectorAll(target, "option"), xmlValue)[-1]
if(length(targetCodes) == 0){
Sys.sleep(0.5)
}else{
out <- list(target, targetCodes)
}
}
return(out)
}
#This part works fine in getting autonoma names and codes
autonomas <- remDr$findElement(using = "name", "cmbCCAA")
autonomasdata <- autonomas$getElementAttribute("outerHTML")[[1]]
autocodes <- sapply(querySelectorAll(xmlParse(autonomasdata), "option"), xmlGetAttr, "value")[-1]
autos <- sapply(querySelectorAll(xmlParse(autonomasdata), "option"), xmlValue)[-1]
#But this part for getting Province and Municipality names and codes does not run
autonoma <- list()
for(x in seq_along(autocodes)){
ConsultProv <- remDr$findElement(using = 'name', value = 'btnBuscarProv')
ConsultProv$clickElement()
province <- changeFun(autocodes[[x]], "cmbCCAA", "cmbProv")
municipality <- lapply(province[[2]], function(y){
ConsultMuni <- remDr$findElement(using = 'name', value = 'btnBuscarMuni')
ConsultMuni$clickElement()
municipality <- changeFun(y, "cmbProv", "cmbMuni")
municipality}
)
list(municipality)}
)
autonoma[[x]] <- list(province, municipality)
}
Any advice on how to make the loop work would be greatly appreciated. I think the problem I am having is in incorporating the button clicking, but I can't seem to figure out how to do it. Thanks!
Edits
I have done some messing around with this, and I still think the issue is with incorporating the button clicking. I have created a Frankenstein's-monster of code below that does... something. Basically, I think I need to incorporate JavaScript in an executeScript command somewhere, either in the ChangeFun or in the larger code. Not sure if me narrowing it down in this way helps at all, but I thought I would update everyone on my (lack of) progress.
sel_auto <- remDr$findElement(using = 'name', value = 'cmbCCAA')
raw_auto <- sel_auto$getElementAttribute("outerHTML")[[1]]
num_auto <- sapply(querySelectorAll(xmlParse(raw_auto), "option"), xmlGetAttr, "value")[-1]
nam_auto <- sapply(querySelectorAll(xmlParse(raw_auto), "option"), xmlValue)[-1]
for (i in seq_along(num_auto)){
remDr$executeScript("document.getElementById('cmbCCAA').value = 1;")
clk_auto <- remDr$findElement(using = 'name', value = 'btnBuscarProv')
clk_auto$clickElement()
sel_prov <- remDr$findElement(using = 'name', value = 'cmbProv')
raw_prov <- sel_prov$getElementAttribute("outerHTML")[[1]]
num_prov <- sapply(querySelectorAll(xmlParse(raw_prov), "option"), xmlGetAttr, "value")[-1]
nam_prov <- sapply(querySelectorAll(xmlParse(raw_prov), "option"), xmlValue)[-1]
for (j in seq_along(num_prov)){
sel_prov$clickElement()
clk_prov <- remDr$findElement(using = 'name', value = 'btnBuscarMuni')
clk_prov$clickElement()
sel_muni <- remDr$findElement(using = 'name', value = 'cmbMuni')
raw_muni <- sel_muni$getElementAttribute("outerHTML")[[1]]
num_muni <- sapply(querySelectorAll(xmlParse(raw_muni), "option"), xmlGetAttr, "value")[-1]
nam_muni <- sapply(querySelectorAll(xmlParse(raw_muni), "option"), xmlValue)[-1]
}
}
Related
Basically what I want is a combination of these two answered posts
Change backgorund color of cell of data table while its value is edited in Rshiny
Shiny with DT Select rows, keep cell color
so I want the cell color to change after each edit on the client side, but when the rows with an edited cell are selected, I need the selection highlight to not overwrite the cell colorization, (so that it looks like this https://i.stack.imgur.com/K2Gjv.png).
Difference between my problem and the one here
Shiny with DT Select rows, keep cell color
is that in my case the cell colors which need to keep their cell colors can not be hardocoded as they are selected by the client.
code which enables the cell colorization after cell edit (code from #StéphaneLaurent (I only changed a minor thing so that now selections are possible)), however here the selection "overwrites" the yellow cells. Is it even possible to achieve what I want?
library(shiny)
library(shinyjs)
library(DT)
js <- HTML(
"function colorizeCell(i, j){
var selector = '#dtable tr:nth-child(' + i + ') td:nth-child(' + j + ')';
$(selector).css({'background-color': 'yellow'});
}"
)
colorizeCell <- function(i, j){
sprintf("colorizeCell(%d, %d)", i, j)
}
ui <- fluidPage(
useShinyjs(),
tags$head(
tags$script(js)
),
br(),
DTOutput("dtable")
)
dat <- iris[1:5, ]
server <- function(input, output, session){
output[["dtable"]] <- renderDT({
datatable(dat, editable = TRUE)
}, server = FALSE)
observeEvent(input[["dtable_cell_edit"]], {
info <- input[["dtable_cell_edit"]]
i <- info[["row"]]
j <- info[["col"]]
runjs(colorizeCell(i, j+1))
})
}
shinyApp(ui, server)
library(shiny)
library(shinyjs)
library(DT)
css <- HTML(
"table.dataTable tr.selected td.yellow {
background-color: yellow !important
}
td.yellow {
background-color: yellow !important
}"
)
js <- HTML(
"function colorizeCell(i, j){
var selector = '#dtable tr:nth-child(' + i + ') td:nth-child(' + j + ')';
$(selector).addClass('yellow');
}"
)
colorizeCell <- function(i, j){
sprintf("colorizeCell(%d, %d)", i, j)
}
ui <- fluidPage(
useShinyjs(),
tags$head(
tags$style(css),
tags$script(js)
),
br(),
DTOutput("dtable")
)
dat <- iris[1:5, ]
server <- function(input, output, session){
output[["dtable"]] <- renderDT({
datatable(dat, editable = TRUE)
}, server = FALSE)
observeEvent(input[["dtable_cell_edit"]], {
info <- input[["dtable_cell_edit"]]
i <- info[["row"]]
j <- info[["col"]]
runjs(colorizeCell(i, j+1))
})
}
shinyApp(ui, server)
I am trying to work on an R shiny app where I want to show a consolidated table on top and when the user hovers on any line item it would show the detailed section of that table.So here's the code for the first table
library(ggplot2)
ui <- fluidPage(
titlePanel("Basic DataTable"),
# Create a new row for the table.
DT::dataTableOutput("table")
)
server <- function(input, output) {
data <- mpg
data <-data %>% group_by(manufacturer,year) %>%
summarise(cty = round(mean(cty),2),hwy = round(2,mean(hwy)))
# Filter data based on selections
output$table <- DT::renderDataTable(DT::datatable({
data
}))
}
shinyApp(ui,server)
Now when the user hovers on Audi for example it should show a detailed version just forAudi something like this in a table below.Can this be done in shiny with DT on hover or click.
Here is a way. If you prefer to display the child table on click rather than on hover, replace "table.on('mouseover', 'td', function(){" with "table.on('click', 'td', function(){".
library(shiny)
library(DT)
data(mpg, package = "ggplot2")
callback <- c(
"table.on('mouseover', 'td', function(){",
" var index = table.cell(this).index();",
" Shiny.setInputValue('cell', index, {priority: 'event'});",
"});"
)
ui <- fluidPage(
br(),
DTOutput("tbl")
)
server <- function(input, output, session){
dat <- mpg
output[["tbl"]] <- renderDT({
datatable(
dat,
callback = JS(callback)
)
})
filteredData <- eventReactive(input[["cell"]], {
i <- input[["cell"]]$row + 1
j <- input[["cell"]]$column
if(j > 0){
dat[dat[[j]] == dat[i,j], , drop = FALSE]
}else{
NULL
}
})
output[["tblfiltered"]] <- renderDT({
datatable(
filteredData(),
fillContainer = TRUE,
options = list(
pageLength = 5
)
)
})
observeEvent(filteredData(), {
showModal(
modalDialog(
DTOutput("tblfiltered"),
size = "l",
easyClose = TRUE
)
)
})
}
shinyApp(ui, server)
Can any one give me a solution to the following issue where the drop down goes under the plotly hover when going to select the fist item in the drop down.
I used the following code. But didn't work.
in UI,
fluidRow(
tags$hr(style="z-index: 10000;"),
column(width = 3,h2("Device Type")),
column(width = 3,htmlOutput("disselect")),
column(width = 3,htmlOutput("cityeselect")),
column(width = 3,h2("Property Type"))
),
in server
output$disselect <- renderUI({
selectInput("district", "District", c("All",unique(bookings$District)), selected = "All")
})
Any hacks?
Set the z-index of the drop-down so that it's greater than that of the plotly modebar, for instance 1002 would work:
column(width = 3, offset = 9,
selectInput("y", "y", colnames(mtcars)),style="z-index:1002;")
A working example:
library(shiny)
library(plotly)
ui <- fluidPage(
fluidRow(
column(width = 3, offset = 9,
selectInput("y", "y", colnames(mtcars)),style="z-index:1002;")
),
fluidRow(plotlyOutput("plot"))
)
server <- function(input, output, session) {
output$plot <- renderPlotly({
g <- ggplot(mtcars, aes_string("disp", input$y)) +
geom_point()
g <- ggplotly(g) %>%
config(displayModeBar = TRUE)
g
})
}
shinyApp(ui, server)
If you don't need the plotly modebar, you can just remove it.
Here's an example:
library(shiny)
library(plotly)
ui <- fluidPage(
fluidRow(
column(width = 3, offset = 9,
selectInput("y", "y", colnames(mtcars)))
),
fluidRow(plotlyOutput("plot"))
)
server <- function(input, output, session) {
output$plot <- renderPlotly({
g <- ggplot(mtcars, aes_string("disp", input$y)) +
geom_point()
### code to hide the modebar ###
g <- ggplotly(g) %>%
config(displayModeBar = FALSE)
g
})
}
shinyApp(ui, server)
I am trying to put dynamic text in shiny's header, and have managed to put text in it, but cannot get it to update after it has received new data from an reactive expression. To put text in the header, I have used a basic Java call, tags$script.
My concern is that renderUI only renders the 1st time, and does not force render whenever the reactive value (val) is updated, which is exactly what I require.
Apologies for the weird example below, I have a HUGE dashboard which has several chained dependencies, and I have tried to replicate the types of dependencies in the reproducible example below.
library(shiny)
library(shinydashboard)
ui <- fluidPage(
dashboardPage(skin = 'black',
dashboardHeader(title = "test"),
dashboardSidebar(
sidebarMenu(id = 'MenuTabs',
menuItem("dummy", tabName = "rawanalysis", selected = TRUE, icon = icon("dashboard"))
)
),
dashboardBody(
uiOutput(outputId = 'Header'),
fluidRow(
box(
actionButton("change", "Change")
)
))))
server <- function(input, output) {
Go_rv <- reactiveValues(val = 0)
observeEvent(input$change, {
sam <- rnorm(1)
if(sam > 0){
Go_rv$val <- TRUE
} else {
Go_rv$val <- FALSE
}
})
val <- reactive({
print(Go_rv$val)
if(Go_rv$val){
out <- 0
} else {
out <- -5
}
return(out)
})
output$Header <- renderUI({
removeUI(
selector = "div:has(> #Header)"
)
header_text <- paste0('$(document).ready(function() {
$("header").find("nav").append(\'<div class="myClass">', val(), '</div>\');})')
tags$script(HTML(header_text),
id = 'Header'
)
})
}
shinyApp(ui = ui, server = server)
Basically, a reactive text output with observeEvent should do this job!
library(shiny)
library(shinydashboard)
ui <- fluidPage(
dashboardPage(skin = 'black',
dashboardHeader(title = textOutput('test')),
dashboardSidebar(
sidebarMenu(id = 'MenuTabs',
menuItem("dummy", tabName = "rawanalysis", selected = TRUE, icon = icon("dashboard"))
)
),
dashboardBody(
#uiOutput(outputId = 'Header'),
fluidRow(
box(
actionButton("change", "Change")
)
))))
server <- function(input, output,session) {
title_change <- reactive({
input$change
as.character(Sys.time())
})
observeEvent(input$change, { output$test <- renderText({ title_change()
})
})
}
shinyApp(ui = ui, server = server)
I want to pull in data from 538, but I want the full data which is arrived at by clicking on "Show more polls"... Is there any way for the function to access the additional lines of the table?
http://projects.fivethirtyeight.com/2016-election-forecast/national-polls/
The code for pulling in the top level data is:
require(XML)
polls.html <- htmlTreeParse("http://projects.fivethirtyeight.com/2016-election-forecast/national-polls/",
useInternalNodes = TRUE)
parsedDoc <- readHTMLTable(polls.html, stringsAsFactors=FALSE)
pollData <- data.frame(parsedDoc[4])
I'm a little confused as to why this got downvoted... still doesn't seem obvious to me! But for anyone who wants to know the solution, I did figure it out (thanks to some help from #duncantl). (Also, the complete analysis is at: https://github.com/hardin47/prediction2016)
require(XML)
require(dplyr)
require(tidyr)
require(readr)
require(mosaic)
require(RCurl)
require(ggplot2)
require(lubridate)
require(RJSONIO)
url = "http://projects.fivethirtyeight.com/2016-election-forecast/national-polls/"
doc <- htmlParse(url, useInternalNodes = TRUE)
sc = xpathSApply(doc, "//script[contains(., 'race.model')]",
function(x) c(xmlValue(x), xmlAttrs(x)[["href"]]))
jsobj = gsub(".*race.stateData = (.*);race.pathPrefix.*", "\\1", sc)
data = fromJSON(jsobj)
allpolls <- data$polls
#unlisting the whole thing
indx <- sapply(allpolls, length)
pollsdf <- as.data.frame(do.call(rbind, lapply(allpolls, `length<-`, max(indx))))
#unlisting the weights
pollswt <- as.data.frame(t(as.data.frame(do.call(cbind, lapply(pollsdf$weight, data.frame,
stringsAsFactors=FALSE)))))
names(pollswt) <- c("wtpolls", "wtplus", "wtnow")
row.names(pollswt) <- NULL
pollsdf <- cbind(pollsdf, pollswt)
#unlisting the voting
indxv <- sapply(pollsdf$votingAnswers, length)
pollsvot <- as.data.frame(do.call(rbind, lapply(pollsdf$votingAnswers,
`length<-`, max(indxv))))
pollsvot1 <- rbind(as.data.frame(do.call(rbind, lapply(pollsvot$V1, data.frame,
stringsAsFactors=FALSE))))
pollsvot2 <- rbind(as.data.frame(do.call(rbind, lapply(pollsvot$V2, data.frame,
stringsAsFactors=FALSE))))
pollsvot1 <- cbind(polltype = rownames(pollsvot1), pollsvot1,
polltypeA = gsub('[0-9]+', '', rownames(pollsvot1)),
polltype1 = extract_numeric(rownames(pollsvot1)))
pollsvot1$polltype1 <- ifelse(is.na(pollsvot1$polltype1), 1, pollsvot1$polltype1 + 1)
pollsvot2 <- cbind(polltype = rownames(pollsvot2), pollsvot2,
polltypeA = gsub('[0-9]+', '', rownames(pollsvot2)),
polltype1 = extract_numeric(rownames(pollsvot2)))
pollsvot2$polltype1 <- ifelse(is.na(pollsvot2$polltype1), 1, pollsvot2$polltype1 + 1)
pollsdf <- pollsdf %>%
mutate(population = unlist(population),
sampleSize = as.numeric(unlist(sampleSize)),
pollster = unlist(pollster),
startDate = ymd(unlist(startDate)),
endDate = ymd(unlist(endDate)),
pollsterRating = unlist(pollsterRating)) %>%
select(population, sampleSize, pollster, startDate, endDate, pollsterRating,
wtpolls, wtplus, wtnow)
allpolldata <- cbind(rbind(pollsdf[rep(seq_len(nrow(pollsdf)), each=3),],
pollsdf[rep(seq_len(nrow(pollsdf)), each=3),]),
rbind(pollsvot1, pollsvot2))
allpolldata <- allpolldata %>%
arrange(polltype1, choice)