Ingesting text

This is from Section 15.3 of the Modern Data Science with R book.

Using rvest

Take a look at the Wikipedia List of songs recorded by the Beatles.

In the book the second list of Other songs is used. I have used the Main Songs list.

A great reference for regex (commands like gsub) is the r4ds book, see Chapter 14 about strings

library(rvest) 
library(tidyr) 
library(methods) 
library(mdsr)
library(tm)
url <- "http://en.wikipedia.org/wiki/List_of_songs_recorded_by_the_Beatles" 
tables <- url %>%
  read_html() %>%
  html_nodes(css = "table") 
tables

{xml_nodeset (7)}
[1] <table class="nowraplinks navbox-inner" style="border-spacing:0;background:transparent;color: ...
[2] <table id="toc" class="toc" summary="Class" align="center" style="text-align:center;"><tbody> ...
[3] <table class="wikitable" style="font-size:90%;">\n<caption>Key\n</caption>\n<tbody>\n<tr>\n<t ...
[4] <table class="wikitable sortable plainrowheaders" style="text-align:center">\n<caption>Name o ...
[5] <table class="wikitable sortable plainrowheaders" style="text-align:center">\n<caption>Name o ...
[6] <table class="wikitable sortable plainrowheaders" style="text-align:center">\n<caption>Name o ...
[7] <table class="nowraplinks vcard hlist collapsible autocollapse navbox-inner" style="border-sp ...

songs <- html_table(tables[[4]])
glimpse(songs)

Observations: 213
Variables: 6
$ Song                     <chr> "\"Across the Universe\"[b]", "\"Act Naturally\"", "\"All I've...
$ `Core catalogue release` <chr> "Let It Be", "Help!", "With the Beatles", "With the Beatles", ...
$ `Songwriter(s)`          <chr> "LennonMcCartney", "Johnny RussellVoni Morrison", "LennonMcCar...
$ `Lead vocal(s)[a]`       <chr> "Lennon", "Starr", "Lennon", "McCartney", "McCartney(with Lenn...
$ Year                     <int> 1969, 1965, 1963, 1963, 1969, 1967, 1964, 1966, 1963, 1965, 19...
$ `Ref(s)`                 <chr> "[6]", "[7]", "[8]", "[8]", "[9]", "[10][11]", "[12]", "[13]",...

songs

other <- html_table(tables[[5]])
glimpse(other)

Observations: 43
Variables: 8
$ Song            <chr> "\"12-Bar Original\"", "\"Ain't She Sweet\"", "\"Ain't She Sweet\"", "\...
$ `Release(s)`    <chr> "Anthology 2", "Anthology 1", "Anthology 3", "Anthology 3", "The Beatle...
$ `Songwriter(s)` <chr> "LennonMcCartneyHarrisonStarkey", "Jack YellenMilton Ager", "Jack Yelle...
$ `Lead vocal(s)` <chr> "Instrumental", "Lennon", "McCartney", "Harrison", "Lennon", "McCartney...
$ Yearrecorded    <int> 1965, 1961, 1969, 1969, 1963, 1962, 1968, 1968, 1960, 1968, 1967, 1968,...
$ Yearreleased    <int> 1996, 1995, 1996, 1996, 2013, 1995, 2018, 2018, 1995, 2018, 1995, 2018,...
$ Notes           <chr> "A twelve-bar blues instrumental, recorded on 4 November 1965.[45] An e...
$ Ref.            <chr> "[47]", "[38]", "[48]", "[48]", "[28]", "[38]", "[51]", "[51]", "[38]",...

other

songs <- songs %>% mutate(Song = gsub('\\"', "", Song), Year = as.numeric(Year)) %>% 
  rename(songwriters = `Songwriter(s)`)
songs

other <- other %>% mutate(Song = gsub('\\"', "", Song), Yearrecorded = as.numeric(Yearrecorded)) %>% 
  rename(songwriters = `Songwriter(s)`)
other

tally(~songwriters, data = songs) %>% 
  sort(decreasing = TRUE) %>% 
  head()

songwriters
               LennonMcCartney                       Harrison                 Larry Williams 
                           159                             22                              3 
                  Carl Perkins                    Chuck Berry LennonMcCartneyHarrisonStarkey 
                             2                              2                              2 

length(grep("McCartney", songs$songwriters))

[1] 165

length(grep("Lennon", songs$songwriters))

[1] 165

length(grep("(McCartney|Lennon)", songs$songwriters))

[1] 165

length(grep("(McCartney|Lennon).*(McCartney|Lennon)", songs$songwriters))

[1] 165

songs %>% filter(grepl("(McCartney|Lennon).*(McCartney|Lennon)", songwriters)) %>% 
  select(Song) %>% 
  head()

song_titles <- VCorpus(VectorSource(songs$Song)) %>% 
  tm_map(removeWords, stopwords("english")) %>% 
  DocumentTermMatrix(control = list(weighting = weightTfIdf))
findFreqTerms(song_titles, 14)

[1] "love"       "revolution" "you"       

Using httr

The following code is from Exercise 15.10. The site stackexchange.com displays questions and answers on technical topics.

library(httr) 

Attaching package: ‘httr’

The following object is masked from ‘package:NLP’:

    content

# Find the most recent R questions on stackoverflow 
getresult <- GET("http://api.stackexchange.com",
                 path = "questions",
                 query = list(site = "stackoverflow.com", tagged = "dplyr")) 
stop_for_status(getresult) # Ensure returned without error 
questions <- content(getresult) # Grab content 
names(questions$items[[1]]) # What does the returned data look like?

 [1] "tags"               "owner"              "is_answered"        "view_count"        
 [5] "answer_count"       "score"              "last_activity_date" "creation_date"     
 [9] "question_id"        "link"               "title"             

length(questions$item)

[1] 30

substr(questions$items[[1]]$title, 1, 68)

[1] "R Dplyr solution for summarize_at correlation"

substr(questions$items[[2]]$title, 1, 68)

[1] "Trouble using dplyr::order to rank values from smallest to largest i"

substr(questions$items[[3]]$title, 1, 68)

[1] "Issue with summarizing a field"

The question asked in this Exercise: How many questions were returned? Without using jargon, describe in words what is being displayed and how it might be used.

The next Exercise 15.11 ask for the same, dplyr. Try something else like ggplot2.

library(httr) 
# Find the most recent R questions on stackoverflow 
getresult <- GET("http://api.stackexchange.com",
                 path = "questions",
                 query = list(site = "stackoverflow.com", tagged = "ggplot2")) 
stop_for_status(getresult) # Ensure returned without error 
questions <- content(getresult) # Grab content 
names(questions$items[[1]]) # What does the returned data look like?

 [1] "tags"               "migrated_from"      "owner"              "is_answered"       
 [5] "view_count"         "accepted_answer_id" "answer_count"       "score"             
 [9] "last_activity_date" "creation_date"      "question_id"        "link"              
[13] "title"             

substr(questions$items[[1]]$title, 1, 68)

[1] "ggsave gives blank jpeg image"

substr(questions$items[[2]]$title, 1, 68)

[1] "ggplot facet_wrap reading all input data as NaN (w/in looped filter)"

substr(questions$items[[3]]$title, 1, 68)

[1] "How to add a vertical line to ggplot when the x-axis goes from 1999 "

substr(questions$items[[4]]$title, 1, 68)

[1] "ggplot geom_boxplot rendering extremely slowly"

substr(questions$items[[5]]$title, 1, 68)

[1] "In R, how do I create multiple time series graphs for multiple sites"