Web Scraping mit rvest und R

Question

Ich versuche, das gesamte Vermögen eines bestimmten Fonds in diesem Fall ADAFX von http://www.morningstar.com/funds/xnas/adafx/quote.html Web kratzen. Aber das Ergebnis ist immer ein Zeichen (leer); Was mache ich falsch?

Ich habe rvest zuvor mit gemischten Ergebnissen verwendet, so habe ich Zeit gefunden, um Experten-Hilfe von der Gemeinschaft der vertrauenswürdigen Gurus zu bekommen (das ist du).

library(rvest)  
Symbol.i ="ADAFX" 
url <-Paste("http://www.morningstar.com/funds/xnas/",Symbol.i,"/quote.html",sep="") 
    tryCatch(NetAssets.i <- url %>% 
      read_html() %>% 
      html_nodes(xpath='//*[@id="gr_total_asset_wrap"]/span/span') %>% 
      html_text(), error = function(e) NetAssets.i = NA) 

Vielen Dank im Voraus, Cheers,

Aaron Soderstrom

Answer 1

Es ist eine dynamische Seite, die Daten für die verschiedenen sectinons über XHR-Anfragen lädt, so dass Sie an den Entwickler-Tools Netzwerk suchen Klicken Sie auf den Tab, um die Ziel-URLs zu erhalten.

library(httr) 
library(rvest) 

res <- GET(url = "http://quotes.morningstar.com/fundq/c-header", 
      query = list(
      t="XNAS:ADAFX", 
      region="usa", 
      culture="en-US", 
      version="RET", 
      test="QuoteiFrame" 
      ) 
) 

content(res) %>% 
    html_nodes("span[vkey='TotalAssets']") %>% 
    html_text() %>% 
    trimws() 
## [1] "20.6 mil" 

Answer 2

Here ist die CSV-Datei es nennt.

library(httr) 
library(rvest) 
library(tm) 
library(plyr) 
require("dplyr") 

MF.List <- read.csv("C:/Users/Aaron/Documents/Investment Committee/Screener/Filtered Funds.csv") 
Category.list <- read.csv("C:/Users/Aaron/Documents/Investment Committee/Screener/Category.csv") 
Category.list <- na.omit(Category.list) 

Category.name <- "Financial" 
MF.Category.List <- filter(MF.List, Category == Category.name) 

morningstar.scrape <- list() 

for(i in 1:nrow(MF.Category.List)){ 

    Symbol.i =as.character(MF.Category.List[i,"Symbol"]) 
    res <- GET(url = "http://quotes.morningstar.com/fundq/c-header", 
      query = list(
       t=paste("XNAS:",Symbol.i,sep=""), 
       region="usa", 
       culture="en-US", 
       version="RET", 
       test="QuoteiFrame" 
      ) 
) 

    tryCatch(
    TTM.Yield <- content(res) %>% 
     html_nodes("span[vkey='ttmYield']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) TTM.Yield<-NA) 

    tryCatch(
    Load <- content(res) %>% 
     html_nodes("span[vkey='Load']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Load = NA) 

    tryCatch(
    Total.Assets <- content(res) %>% 
     html_nodes("span[vkey='TotalAssets']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Total.Assets = NA) 

    tryCatch(
    Expense.Ratio <- content(res) %>% 
     html_nodes("span[vkey='ExpenseRatio']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Expense.Ratio = NA) 

    tryCatch(
    Fee.Level <- content(res) %>% 
     html_nodes("span[vkey='FeeLevel']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Fee.Level = NA) 

    tryCatch(
    Turnover <- content(res) %>% 
     html_nodes("span[vkey='Turnover']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Turnover = NA) 

    tryCatch(
    Status <- content(res) %>% 
     html_nodes("span[vkey='Status']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Status = NA) 

    tryCatch(
    Min.Investment <- content(res) %>% 
     html_nodes("span[vkey='MinInvestment']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Min.Investment = NA) 

    tryCatch(
    Yield.30day <- content(res) %>% 
     html_nodes("span[vkey='Yield']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Yield.30day = NA) 

    tryCatch(
    Investment.Style <- content(res) %>% 
     html_nodes("span[vkey='InvestmentStyle']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Investment.Style = NA) 

    tryCatch(
    Bond.Style <- content(res) %>% 
     html_nodes("span[vkey='BondStyle']") %>% 
     html_text() %>% 
     trimws() 
    , error = function(e) Bond.Style = NA) 

    x.frame <- c(Symbol =as.character(Symbol.i),TTM.Yield = as.character(TTM.Yield), Load = as.character(Load), 
       Total.Assets = as.character(Total.Assets),Expense.Ratio = as.character(Expense.Ratio), 
       Turnover = as.character(Turnover), Status = as.character(Status), 
       Yield.30day = as.character(Yield.30day), 
       Investment.Style = as.character(Investment.Style),Bond.Style = as.character(Bond.Style)) 

    morningstar.scrape[[i]] = x.frame 
    x.frame = NULL 
} 

MS.scrape <- do.call(rbind, morningstar.scrape) 

Answer 3

Arbeitscode,

Ich habe eine Funktion zur Bahnbrecheisen und entfernt die Bibliothek (tm).

library(httr) 
library(rvest) 


    get.morningstar <- function(Symbol.i,htmlnode){ 
     res <- GET(url = "http://quotes.morningstar.com/fundq/c-header", 
       query = list(
        t=paste("XNAS:",Symbol.i,sep=""), 
        region="usa", 
        culture="en-US", 
        version="RET", 
        test="QuoteiFrame" 
       ) 
    ) 

     x <- content(res) %>% 
     html_nodes(htmlnode) %>% 
     html_text() %>% 
     trimws() 

     return(x) 
    } 



    MF.List <- read.csv("C:/Users/Aaron/Documents/Bitrix24/Investment Committee/Screener/Filtered Funds.csv") 
    Category.list <- read.csv("C:/Users/Aaron/Documents/Bitrix24/Investment Committee/Screener/Category.csv") 
    Category.list <- na.omit(Category.list) 

    Category.name <- "Small Growth" 
    MF.Category.List <- MF.List[grepl(Category.name,MF.List$Category), ] 
    morningstar.scrape <- list() 

    for(i in 1:nrow(MF.Category.List)){ 
     Symbol.i =as.character(MF.Category.List[i,"Symbol"]) 
     try(Total.Assets <- get.morningstar(Symbol.i,"span[vkey='TotalAssets']")) 
     print(Total.Assets) 
    }