\n", " self.wordcorpus2 = self.soup.find_all('li') #obtain all paragraphs starting with tag
\n", " self.para.append(paragraph)\n", "\n", " self.relatedtopic = \",*RELATED WIKI TOPIC*\" #Identify topics in Wikipedia with an URL to point out to user\n", " for paragraph in self.wordcorpus2: #append paragraphs starting with
to line break\n", " self.point = 0 #after
paragraph\n", " if str(sentence).find(\"refer to:\") != -1:\n", " self.offsite = 1 \n", " \n", " #Check if Wikipedia article exists\n", " for sentence in self.missing: \n", " if str(sentence) == \"Wikipedia does not have an article with this exact name.\": #CONVERT ELEMENT TO STRING TYPE BEFORE CHECK!!!\n", " self.goodsite = 0 #sentence exists, bad site means counter flips to 0\n", " \n", " #Confirmation for successful Wikipedia site load\n", " if self.goodsite == 1 and self.offsite == 0:\n", " print('Wikipedia page loaded successfully!! Type variablename.HELP() for documentation of functions.')\n", " \n", " #WHEN SEARCH IS SUCCESSFUL, PRINT 2 PARAGRAPHS FOR PREVIEW\n", " self.parashort = []\n", " self.noofpara = 0\n", " for paragraph in self.wordcorpus: #append ONLY 2 paragraphs starting with
\n", " if self.noofpara < 2 and str(paragraph) != '
\\n
' and len(str(paragraph)) > 199: \n", " self.parashort.append(paragraph)\n", " self.noofpara += 1\n", "\n", " #Data cleaning for printing out summary of Wikipedia (2 paragraphs) if search is successful - __cleantext_summary\n", " self.parashort2 = self.__cleantext_summary(self.parashort)\n", " \n", " #REMOVE UNWANTED ARRAYS\n", " self.parashort = [] \n", "\n", " #END OF DATA PROCESSING FOR AUTOMATIC SUMMARY PRINTED WHEN SEARCHING\n", "\n", " #Print two paragraphs when search is successful\n", " print(self.parashort2)\n", " \n", " \n", " #Ambiguous search detecton and suggested article names\n", " elif self.goodsite == 1 and self.offsite == 1:\n", " print('\\nThe title \"'+ self.title.replace(\"_\", \" \") + '\" you specified is ambiguous. As a result, you are linked to a clarification page.\\n\\n')\n", " print('Here are some suggestions to use: \\n')\n", " \n", " self.all_links = self.soup.find_all(\"a\") #ALL HTML TAGS STARTING WITH\n", " if self.noofpara < self.paravalue and str(paragraph) != '
\\n
' and len(str(paragraph)) > 199: \n", " self.parashort.append(paragraph)\n", " self.noofpara += 1\n", "\n", " self.parashort2 = self.__cleantext_summary(self.parashort)\n", " #REMOVE UNWANTED ARRAYS\n", " self.parashort = [] \n", " \n", " #END OF DATA PROCESSING FOR SUMMARY FUNCTION! i.e. 2 paragraph of wiki\n", "\n", " #Checking if you should print or output argument\n", " if isinstance(outsummary, str): #check for string yes, no and other permutations\n", " if outsummary.lower().strip() in {'yes','true','y','t'}:\n", " print(\"Summary is output. To print it instead, put 'no' in argument\")\n", " return self.parashort2\n", " elif outsummary.lower().strip() in {'no','false','n','f','na','n/a','nan'}:\n", " print(\"Summary is printed. To set as output, put 'yes' in 2nd argument\")\n", " print(self.parashort2)\n", " else:\n", " print(\"2nd argument is invalid, summary is printed by default. To set as output, put 'yes' in 2nd argument\")\n", " print(self.parashort2)\n", " elif isinstance(outsummary, bool): #check for boolean yes/no\n", " if outsummary == True:\n", " print(\"Summary is output. To print it instead, put False or 'no' in 2nd argument\")\n", " return self.parashort2\n", " else:\n", " print(\"Summary is printed. To set as output, put True or 'yes' in 2nd argument\")\n", " print(self.parashort2) \n", " else: #run default - print if invalid option\n", " print(\"2nd argument not a string, summary is printed by default. To set as output, put 'yes' in 2nd argument\")\n", " print(self.parashort2)\n", " \n", " #HELP and basic documentation of available functions\n", " def HELP(self):\n", " '''Explains how to use the class object wiki and also retrieves a list of methods with their actions.'''\n", " print('The wiki() class accepts 5 arguments. The first one is a compulsory title of the Wikipedia page. Second is to format the search string to proper/title case (Yes/No, default: Yes).')\n", " print('Third is for language settings (e.g. English, de, francais, etc., default: English).')\n", " print('Fourth is for implementing NLTK stoplist in provided language based on 3rd arg (Yes/No, default: standard stoplist provided).')\n", " print('Fifth is for lemmatizing text (Yes/No, default: No).\\n\\n')\n", " print('Functions/Methods of Wikipedia scraper package: \\n')\n", " print('commonwords accepts 1 optional argument (default: 100) for the number of most common words in the site and their frequencies to show.\\n')\n", " print('commonwordspct accepts 1 optional argument (default: 10) on the percentage threshold of word count to determine the most frequent words to show.\\n')\n", " print('plotwords accepts 4 optional arguments. The first argument is the filename to save as (default: wordcount.png). The second argument (default: 20) is for the number of most frequent words to show as a GRAPH. The third argument is the Nth most frequent word to start plotting from. (default: 1, starting from most frequent word). The third argument is the filename to save as. The fourth argument removes the latest N years from the most frequent words (default: remove latest 10 years)\\n')\n", " print('plotyear accepts 2 optional argument. The first argument is the filename to save as (default: yearcount.png). The second argument (default: 20) is the number of years to plot in the graph. The frequency count of the most common years will be plotted. This allows the user to understand the years of interest for the Wikipedia Topic.\\n')\n", " print('totalwords accepts 0 argument and shows the total word count and unique word count\\n')\n", " print('summary accepts 2 optional arguments, the first one for the number of paragraphs to show (default: 2) and the second one - Yes to output string and No to print text (default: No). It gives a summary of the Wikipedia page\\n')\n", " print('gettext accepts 1 optional argument - Yes to output string and No to print text (default: No). It retrieves the full text of the Wikipedia title\\n')\n", " " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" } }, "nbformat": 4, "nbformat_minor": 2 }