-
Notifications
You must be signed in to change notification settings - Fork 0
/
bh_wordcloud.R
125 lines (88 loc) · 3.74 KB
/
bh_wordcloud.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#' Wordcloud
#' Scott Cohn
#' Oct 18 // Dec 15
# Libraries ---------------------------------------------------------------
## libs needed
packages = c("tidyverse",
"tokenizers",
"tm",
"SnowballC")
## Now load or install&load all
package.check <- lapply(
packages,
FUN = function(x) {
if (!require(x, character.only = TRUE)) {
install.packages(x, dependencies = TRUE)
library(x, character.only = TRUE)
}
}
)
# Pre-Run -----------------------------------------------------------------
# Clear out existing files
# Run one line at a time
# Delete the txt files from /bh_txts
system("rm bh_txts/*.txt")
system("rm bh_txts_clean/*.txt")
system("rm bh_wordlist/*.csv")
# File Mgmt ---------------------------------------------------------------
# Folder with PDFs
pdf_path <- "bh_pdfs"
txt_path <- "bh_txts"
# Make a vector of PDF file names
my_pdfs <- list.files(path = pdf_path, pattern = "pdf", full.names = TRUE)
# Convert each PDF file that is named in the vector into a text file
lapply(my_pdfs, function(i) system(paste('"pdftotext"',
paste0('"', i, '"')), wait = FALSE) )
# Run one line at a time
# Copy the txt files from /bh_pdf to /bh_txt
system("cp bh_pdfs/*.txt bh_txts")
# Delete the txt files from /bh_pdf
system("rm bh_pdfs/*.txt")
# Make a vector of txt file names
my_txts <- list.files(path = txt_path, pattern = "txt", full.names = TRUE)
print(my_txts) # check
# Functions ---------------------------------------------------------------
# Import cleaning function
source("bh_cleantext.R")
# Import analysis function
source("bh_text_analysis.R")
# Stop words + Completion dict --------------------------------------------------------------
# import wordstop supplement
wordstop <- read.csv("wordstop.csv", header = FALSE)$V1
# append common english stopwords
wordstop <- c(wordstop, stopwords("en"))
# word completion for stemmings
# compdict <- read.csv("compdict.csv", header = FALSE)$V1
# Clean + Analyze Text ----------------------------------------------------
ch01 <- clean_text(my_txts[1], words_to_remove = wordstop)
text_analyze(ch01, "ch01", list_return = TRUE)
ch02 <- clean_text(my_txts[2], words_to_remove = wordstop)
text_analyze(ch02, "ch02", list_return = TRUE)
ch03 <- clean_text(my_txts[3], words_to_remove = wordstop)
text_analyze(ch03, "ch03", list_return = TRUE)
ch04 <- clean_text(my_txts[4], words_to_remove = wordstop)
text_analyze(ch04, "ch04", list_return = TRUE)
ch05 <- clean_text(my_txts[5], words_to_remove = wordstop)
text_analyze(ch05, "ch05", list_return = TRUE)
ch06 <- clean_text(my_txts[6], words_to_remove = wordstop)
text_analyze(ch06, "ch06", list_return = TRUE)
ch07 <- clean_text(my_txts[7], words_to_remove = wordstop)
text_analyze(ch07, "ch07", list_return = TRUE)
ch08 <- clean_text(my_txts[8], words_to_remove = wordstop)
text_analyze(ch08, "ch08", list_return = TRUE)
ch09 <- clean_text(my_txts[9], words_to_remove = wordstop)
text_analyze(ch09, "ch09", list_return = TRUE)
ch10 <- clean_text(my_txts[10], words_to_remove = wordstop)
text_analyze(ch10, "ch10", list_return = TRUE)
ch11 <- clean_text(my_txts[11], words_to_remove = wordstop)
text_analyze(ch11, "ch11", list_return = TRUE)
ch12 <- clean_text(my_txts[12], words_to_remove = wordstop)
text_analyze(ch12, "ch12", list_return = TRUE)
ch13 <- clean_text(my_txts[13], words_to_remove = wordstop)
text_analyze(ch13, "ch13", list_return = TRUE)
ch14 <- clean_text(my_txts[14], words_to_remove = wordstop)
text_analyze(ch14, "ch14", list_return = TRUE)
ch15 <- clean_text(my_txts[15], words_to_remove = wordstop)
text_analyze(ch15, "ch15", list_return = TRUE)
ch16 <- clean_text(my_txts[16], words_to_remove = wordstop)
text_analyze(ch16, "ch16", list_return = TRUE)