Major update to wikiscrape.py, tidied up files in repo and updates ar…

…e listed in readme
kohjiaxuan · Dec 2, 2019 · bfba605 · bfba605
1 parent 632a750
commit bfba605
Show file tree

Hide file tree

Showing 44 changed files with 3,444 additions and 571 deletions.
diff --git a/.ipynb_checkpoints/FINAL. wikiscrape package (updated 26 Nov)-checkpoint.ipynb b/.ipynb_checkpoints/FINAL. wikiscrape package (updated 26 Nov)-checkpoint.ipynb
diff --git a/.ipynb_checkpoints/WIKI10. Added Lemmatizing (shortening of words)-checkpoint.ipynb b/.ipynb_checkpoints/WIKI10. Added Lemmatizing (shortening of words)-checkpoint.ipynb
diff --git a/ArminBuuren.png b/ArminBuuren.png
diff --git a/Armin_top10words_25112019.png b/Armin_top10words_25112019.png
diff --git a/Armin_top30years_25112019.png b/Armin_top30years_25112019.png
diff --git a/Armin_top40words_25112019.png b/Armin_top40words_25112019.png
diff --git a/ColdplayWordCount2.png b/ColdplayWordCount2.png
diff --git a/ColdplayYearCount3.png b/ColdplayYearCount3.png
diff --git a/Donald_Trump_30years.png b/Donald_Trump_30years.png
diff --git a/Donald_Trump_40words.png b/Donald_Trump_40words.png
diff --git a/FINAL. wikiscrape package (updated 26 Nov).ipynb b/FINAL. wikiscrape package (updated 26 Nov).ipynb
@@ -411,8 +411,8 @@
  " self.parashort.append(paragraph)\n",
  " self.noofpara += 1\n",
  "\n",
- " #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful - cleantext_summary\n",
- " self.parashort2 = self.cleantext_summary(self.parashort)\n",
+ " #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful - __cleantext_summary\n",
+ " self.parashort2 = self.__cleantext_summary(self.parashort)\n",
  " \n",
  " #REMOVE UNWANTED ARRAYS\n",
  " self.parashort = [] \n",
@@ -453,7 +453,7 @@
  " print('Other useful information: Enclose title argument with single quotes. Spaces are allowed, and title is case insensitive.')\n",
  " \n",
  " \n",
- " def cleantext_summary(self, corpus):\n",
+ " def __cleantext_summary(self, corpus):\n",
  " '''Gets summary of the text, internal method'''\n",
  " #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful\n",
  " corpus = list(str(corpus)) #chop everything into letters for usage\n",
@@ -747,7 +747,7 @@
  " self.parashort.append(paragraph)\n",
  " self.noofpara += 1\n",
  "\n",
- " self.parashort2 = self.cleantext_summary(self.parashort)\n",
+ " self.parashort2 = self.__cleantext_summary(self.parashort)\n",
  " #REMOVE UNWANTED ARRAYS\n",
  " self.parashort = [] \n",
  " \n",

diff --git a/Images/ArminBuuren.png b/Images/ArminBuuren.png
diff --git a/Images/Armin_top10words_25112019.png b/Images/Armin_top10words_25112019.png
diff --git a/Images/Armin_top30years_25112019.png b/Images/Armin_top30years_25112019.png
diff --git a/Images/Armin_top40words_25112019.png b/Images/Armin_top40words_25112019.png
diff --git a/Images/ColdplayWordCount2.png b/Images/ColdplayWordCount2.png
diff --git a/Images/ColdplayYearCount3.png b/Images/ColdplayYearCount3.png
diff --git a/Images/Donald_Trump_30years.png b/Images/Donald_Trump_30years.png
diff --git a/Images/Donald_Trump_40words.png b/Images/Donald_Trump_40words.png
diff --git a/WikiScrape_Example.PNG → Images/WikiScrape_Example.PNG b/WikiScrape_Example.PNG → Images/WikiScrape_Example.PNG
diff --git a/bar.png → Images/bar.png b/bar.png → Images/bar.png
diff --git a/coldplayyearcount.png → Images/coldplayyearcount.png b/coldplayyearcount.png → Images/coldplayyearcount.png
diff --git a/coldplayyearcount2.jpg → Images/coldplayyearcount2.jpg b/coldplayyearcount2.jpg → Images/coldplayyearcount2.jpg
diff --git a/coldplayyearcount20.jpg → Images/coldplayyearcount20.jpg b/coldplayyearcount20.jpg → Images/coldplayyearcount20.jpg
diff --git a/coldplayyearcount_20.png → Images/coldplayyearcount20.png b/coldplayyearcount_20.png → Images/coldplayyearcount20.png
diff --git a/...racting and cleaning Wikipedia Site.ipynb → ...racting and cleaning Wikipedia Site.ipynb b/...racting and cleaning Wikipedia Site.ipynb → ...racting and cleaning Wikipedia Site.ipynb
diff --git a/...ape (updated 26 May for year graph).ipynb → ...d 26 May for year graph)-checkpoint.ipynb b/...ape (updated 26 May for year graph).ipynb → ...d 26 May for year graph)-checkpoint.ipynb
diff --git a/...ng Year Plot 26 May 2019-checkpoint.ipynb → ...ng Year Plot 26 May 2019-checkpoint.ipynb b/...ng Year Plot 26 May 2019-checkpoint.ipynb → ...ng Year Plot 26 May 2019-checkpoint.ipynb
diff --git a/... Beautiful Soup India Wiki Practice.ipynb → ... Beautiful Soup India Wiki Practice.ipynb b/... Beautiful Soup India Wiki Practice.ipynb → ... Beautiful Soup India Wiki Practice.ipynb
diff --git a/...ass and Function for Wikipedia Page.ipynb → ...ass and Function for Wikipedia Page.ipynb b/...ass and Function for Wikipedia Page.ipynb → ...ass and Function for Wikipedia Page.ipynb
diff --git a/WIKI02. wikiscrape-Copy1.ipynb → Tests/WIKI02. wikiscrape-Copy1.ipynb b/WIKI02. wikiscrape-Copy1.ipynb → Tests/WIKI02. wikiscrape-Copy1.ipynb
diff --git a/... Top 100 words in English from Wiki.ipynb → ... Top 100 words in English from Wiki.ipynb b/... Top 100 words in English from Wiki.ipynb → ... Top 100 words in English from Wiki.ipynb
diff --git a/.... wikiscrape without summary output.ipynb → .... wikiscrape without summary output.ipynb b/.... wikiscrape without summary output.ipynb → .... wikiscrape without summary output.ipynb
diff --git a/WIKI05. importing wikiscrape test.ipynb → ...s/WIKI05. importing wikiscrape test.ipynb b/WIKI05. importing wikiscrape test.ipynb → ...s/WIKI05. importing wikiscrape test.ipynb
diff --git a/WIKI06. Wikiscrape Example (LinkedIn).ipynb → ...KI06. Wikiscrape Example (LinkedIn).ipynb b/WIKI06. Wikiscrape Example (LinkedIn).ipynb → ...KI06. Wikiscrape Example (LinkedIn).ipynb
diff --git a/WIKI07. Tinkering of Wikipedia Scraper.ipynb → ...I07. Tinkering of Wikipedia Scraper.ipynb b/WIKI07. Tinkering of Wikipedia Scraper.ipynb → ...I07. Tinkering of Wikipedia Scraper.ipynb
diff --git a/...d 26 May for year graph)-checkpoint.ipynb → ...ape (updated 26 May for year graph).ipynb b/...d 26 May for year graph)-checkpoint.ipynb → ...ape (updated 26 May for year graph).ipynb
diff --git a/...crape Testing Year Plot 26 May 2019.ipynb → ...crape Testing Year Plot 26 May 2019.ipynb b/...crape Testing Year Plot 26 May 2019.ipynb → ...crape Testing Year Plot 26 May 2019.ipynb
diff --git a/WIKI10. Added Lemmatizing (shortening of words).ipynb b/WIKI10. Added Lemmatizing (shortening of words).ipynb
diff --git a/__pycache__/wikiscrape.cpython-36.pyc b/__pycache__/wikiscrape.cpython-36.pyc
diff --git a/wikiscrape old.txt b/wikiscrape old.txt
diff --git a/wikiscrape.py b/wikiscrape.py
@@ -1,9 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-# In[ ]:
-
-
 import requests #Get the HTML code
 from bs4 import BeautifulSoup #Tidy up the code
 from collections import Counter #Counter to count occurances of each word
@@ -367,8 +364,8 @@ def __init__(self,title,option='Yes',lang='en',checknltk='No',lemmatize='No'):
  self.parashort.append(paragraph)
  self.noofpara += 1
 
- #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful - cleantext_summary
- self.parashort2 = self.cleantext_summary(self.parashort)
+ #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful - __cleantext_summary
+ self.parashort2 = self.__cleantext_summary(self.parashort)
 
  #REMOVE UNWANTED ARRAYS
  self.parashort = [] 
@@ -409,7 +406,7 @@ def __init__(self,title,option='Yes',lang='en',checknltk='No',lemmatize='No'):
  print('Other useful information: Enclose title argument with single quotes. Spaces are allowed, and title is case insensitive.')
 
 
- def cleantext_summary(self, corpus):
+ def __cleantext_summary(self, corpus):
  '''Gets summary of the text, internal method'''
  #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful
  corpus = list(str(corpus)) #chop everything into letters for usage
@@ -703,7 +700,7 @@ def summary(self, paravalue=2, outsummary='no'):
  self.parashort.append(paragraph)
  self.noofpara += 1
 
- self.parashort2 = self.cleantext_summary(self.parashort)
+ self.parashort2 = self.__cleantext_summary(self.parashort)
  #REMOVE UNWANTED ARRAYS
  self.parashort = [] 
 
@@ -743,4 +740,4 @@ def HELP(self):
  print('plotyear accepts 2 optional argument. The first argument is the filename to save as (default: yearcount.png). The second argument (default: 20) is the number of years to plot in the graph. The frequency count of the most common years will be plotted. This allows the user to understand the years of interest for the Wikipedia Topic.\n')
  print('totalwords accepts 0 argument and shows the total word count and unique word count\n')
  print('summary accepts 2 optional arguments, the first one for the number of paragraphs to show (default: 2) and the second one - Yes to output string and No to print text (default: No). It gives a summary of the Wikipedia page\n')
- print('gettext accepts 1 optional argument - Yes to output string and No to print text (default: No). It retrieves the full text of the Wikipedia title\n')
+ print('gettext accepts 1 optional argument - Yes to output string and No to print text (default: No). It retrieves the full text of the Wikipedia title\n')
diff --git a/wordcount.png b/wordcount.png
diff --git a/yearcount.png b/yearcount.png