-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrapearchive.R
47 lines (35 loc) · 1.47 KB
/
scrapearchive.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
scrapearchive <- function(address) {
#scrapearchive will look at an archived version of the pencil page and will extract pencil names and prices
#exampleuse
# test4 <- scrapearchive("https://web.archive.org/web/20140417064443/http://www.cultpens.com/acatalog/Pencils.html")
library(rvest)
#get date out, ignore time as unlikely for price to have changed so not worth recording this info
archivedate <- regmatches(address, gregexpr("20[0-9]{6}",address))
#pencils <- html("https://web.archive.org/web/20130730001143/http://www.cultpens.com/acatalog/Pencils.html")
pencils <- html(address)
# pencil product names
pnames <-
pencils %>%
html_nodes("p a") %>%
html_text()
# web page formatting will result in empty lines
# remove empty lines
pnames <- grep ("[a-z]", pnames, value=TRUE)
# product names into vector pnames
paragraphs <-
pencils %>%
html_nodes("p") %>%
html_text()
#remove all entries without a pound sign
paragraphs <- grep ("£", paragraphs, value=TRUE)
# only keep prices
t1 <- regmatches(paragraphs, gregexpr("£([0-9])+.[0-9][0-9]",paragraphs))
# only keep first price
price = do.call("rbind", lapply(t1, "[[", 1))
# both vectors into a dataframe
tryCatch({
data.frame(name=pnames, price=price, date=unlist(archivedate))
}, error=function(e) data.frame(name=character(0),
price=numeric(0),
date=character(0)))
}