SMLR03. comparewiki.similar time tracking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import time\n",
    "import wikionly #script name is wikionly (no summary), class name is wiki\n",
    "import re as re\n",
    "import nltk\n",
    "# nltk.download('wordnet')\n",
    "from nltk.corpus import wordnet\n",
    "import math\n",
    "\n",
    "#Input two Wikipedia articles to compute similarity percentage\n",
    "class similar:\n",
    "    def __init__(self,text1,text2,verbose=1):\n",
    "        \"\"\"To start, assign var = comparewiki.similar('arg1','arg2', verbose=1). \n",
    "        arg1 and arg2 are names of the wikipedia articles.\n",
    "        verbose=1 prints the probability score and mathematical calculation. \n",
    "        verbose=2 additionally prints array of words for each article\n",
    "        verbose=0 disables any logs.\n",
    "        To get values in a list for storage, use .ans(). To get the 40 common words for comparison, use .words()\"\"\"\n",
    "        time01 = time.time()\n",
    "        self.wn = nltk.corpus.wordnet #the corpus reader\n",
    "        self.verbose = verbose # Verbose/log level of detail\n",
    "        time02 = time.time()\n",
    "        print(\"Initialize NLTK similarity corpus reader: \" + str(time02 - time01) + \" seconds.\")\n",
    "        \n",
    "        #Error handling: check if both arguments input are string format\n",
    "        checkstr = False\n",
    "        if isinstance(text1, str) == True:\n",
    "            if isinstance(text2, str) == True:\n",
    "                self.text1 = text1\n",
    "                self.text2 = text2\n",
    "                time03 = time.time()\n",
    "                print(\"String check of arguments: \" + str(time03 - time02) + \" seconds.\")\n",
    "                checkstr = True\n",
    "            else:\n",
    "                print('Error! The second argument is not a string format!')        \n",
    "        else:\n",
    "            print('Error! The first argument is not a string format!')\n",
    "        \n",
    "        #Run internal wikipedia python file for processing for both wiki titles\n",
    "        if checkstr == True:\n",
    "            self.wiki1 = wikionly.wiki(text1)\n",
    "            time04 = time.time()\n",
    "            print(\"Scrape wikipedia for article 1: \" + str(round(time04 - time03, 4)) + \" seconds.\")\n",
    "            self.wiki2 = wikionly.wiki(text2)\n",
    "            time05 = time.time()\n",
    "            print(\"Scrape wikipedia for article 2: \" + str(round(time05 - time04, 4)) + \" seconds.\")\n",
    "            \n",
    "        #Call the function that calculates percentage\n",
    "        self.percent(self.wiki1,self.wiki2,self.verbose)\n",
    "        \n",
    "        #call the function that shows list of words for both Wiki sites\n",
    "        #Only can be used if self.percent has been called and list/arrays for articles are created\n",
    "        if self.verbose == 2:\n",
    "            print(self.words())\n",
    "\n",
    "    #Retrieve top 40 common words from wiki page, slice up and append .n01 for NLTK usage\n",
    "    def percent(self,input1,input2,verbose):\n",
    "        time06 = time.time()\n",
    "        self.dotn01 = ('.','n','.','0','1')\n",
    "        self.wiki1list = []\n",
    "        for key in self.wiki1.commonwords(40):\n",
    "            self.wiki1slice = list(key)\n",
    "            for letter in self.dotn01:\n",
    "                self.wiki1slice.append(letter)\n",
    "            self.wiki1slice = ''.join(self.wiki1slice)\n",
    "            self.wiki1list.append(self.wiki1slice)\n",
    "\n",
    "        self.wiki2list = []\n",
    "        for key in self.wiki2.commonwords(40):\n",
    "            self.wiki2slice = list(key)\n",
    "            for letter in self.dotn01:\n",
    "                self.wiki2slice.append(letter)\n",
    "            self.wiki2slice = ''.join(self.wiki2slice)\n",
    "            self.wiki2list.append(self.wiki2slice)\n",
    "            \n",
    "        time07 = time.time()\n",
    "        print(\"Get list of 40 words for both articles: \" + str(round(time07 - time06, 4)) + \" seconds.\")\n",
    "        \n",
    "        #count and sum for calculating similarity\n",
    "        self.count = 0\n",
    "        self.sum = 0\n",
    "        #A count for the ranking of the word (how often it appears in both wiki passages)\n",
    "        self.topten1 = 0\n",
    "        self.topten2 = 0\n",
    "\n",
    "        #For words that are 1-10th and 11-21st in popularity, if both wiki pages have the word, they get more points\n",
    "        for word1 in self.wiki1list:\n",
    "            #Reset self.topten2\n",
    "            self.topten2 = 0\n",
    "            self.topten1 += 1\n",
    "            for word2 in self.wiki2list:\n",
    "                self.topten2 += 1\n",
    "                #reinitialize to zero to prevent old sums from going into maxsum\n",
    "                self.sum1 = 0\n",
    "                self.sum2 = 0\n",
    "                self.sum3 = 0\n",
    "                self.sum4 = 0\n",
    "                self.maxsum = 0\n",
    "                \n",
    "                if self.topten1 < 11 and self.topten2 < 11:\n",
    "                    self.expvalue = 4.5\n",
    "                elif self.topten1 < 21 and self.topten2 < 21:\n",
    "                    self.expvalue = 2.5\n",
    "                else:\n",
    "                    self.expvalue = 1.5\n",
    "                \n",
    "                #Main algorithm for calculating score of words\n",
    "                try:\n",
    "                    if re.findall(r\"\\d+.n.01\", word1) == [] and re.findall(r\"\\d+.n.01\", word2) == []: #check both words not numbers\n",
    "                        #since words have many meanings, for every pair of words, use top two meanings n.01 and n.02 for comparison\n",
    "                        #two for loops will check every permutation pair of words between wiki pages, two meanings for each word, \n",
    "                        #Take the max similarity value taken for computation of similarity index\n",
    "                        #e.g. money.n.01 may have highest value with value.n.02 because value.n.01 has the obvious meaning of worth/significance and secondary for money\n",
    "                        word11 = word1.replace('n.01','n.02')\n",
    "                        word22 = word2.replace('n.01','n.02')\n",
    "                        #print(word11,word22)\n",
    "                        self.x = self.wn.synset(word1)\n",
    "                        self.y = self.wn.synset(word2)\n",
    "                        #get default similarity value of 1st definitions of word\n",
    "                        self.sum1 = self.x.path_similarity(self.y) * math.exp(self.expvalue * self.x.path_similarity(self.y)) + 10 * math.log(0.885+self.x.path_similarity(self.y))\n",
    "                        try: #get 2nd definitions of words and their similarity values, if it exist\n",
    "                            self.xx = self.wn.synset(word11)\n",
    "                            self.yy = self.wn.synset(word22)\n",
    "                            self.sum2 = self.xx.path_similarity(self.y) * math.exp(self.expvalue * self.xx.path_similarity(self.y)) + 10 * math.log(0.89+self.xx.path_similarity(self.y))\n",
    "                            self.sum3 = self.x.path_similarity(self.yy) * math.exp(self.expvalue * self.x.path_similarity(self.yy)) + 10 * math.log(0.89+self.x.path_similarity(self.yy))\n",
    "                            self.sum4 = self.xx.path_similarity(self.yy) * math.exp(self.expvalue * self.xx.path_similarity(self.yy)) + 10 * math.log(0.89+self.xx.path_similarity(self.yy))\n",
    "                        except:\n",
    "                            continue\n",
    "                        self.maxsum = max(self.sum1,self.sum2,self.sum3,self.sum4) #get the max similarity value between 2 words x 2 meanings = 4 comparisons\n",
    "                        #print(word1, word2, self.maxsum)\n",
    "                        self.sum += self.maxsum\n",
    "                        self.count += 1\n",
    "                except:\n",
    "                    if word1 == word2 and re.findall(r\"\\d+.n.01\", word1) == []: #remove years/numbers being counted as match yyyy.n.01\n",
    "                        self.sum += math.exp(self.expvalue) + 10 * math.log(1.89)\n",
    "                        self.count += 1\n",
    "                    else:\n",
    "                        continue\n",
    "\n",
    "        time08 = time.time()\n",
    "        print(\"Calculate similarity for both articles: \" + str(round(time08 - time07, 4)) + \" seconds.\")\n",
    "        \n",
    "        #Print the results and implement ceiling if the percent exceeds 100% or drops below 0%\n",
    "        if self.count != 0:\n",
    "            self.pct = round(self.sum/self.count*100)\n",
    "            if self.pct > 100:\n",
    "                self.pct = 100\n",
    "            elif self.pct < 0:\n",
    "                self.pct = 0\n",
    "            if self.verbose >= 1:\n",
    "                print('Probability of topics being related is ' + str(self.pct) + '%')\n",
    "                print('Count is ' + str(self.count) + ' and sum is ' + str(self.sum))\n",
    "        else:\n",
    "            if self.verbose >= 1:\n",
    "                print('No relation index can be calculated as words are all foreign')\n",
    "                \n",
    "        time09 = time.time()\n",
    "        print(\"Print output: \" + str(time09 - time08) + \" seconds.\")\n",
    "        return self.pct\n",
    "        \n",
    "    #Print out list of common words for both Wiki articles\n",
    "    def words(self):\n",
    "        print(self.wiki1list)\n",
    "        print('\\n')\n",
    "        print(self.wiki2list)\n",
    "        \n",
    "    #Outputs list of results [Article 1, Article 2, Percentage, Yes/No] that can be put into a dataframe\n",
    "    def ans(self):\n",
    "        self.listans = [self.text1,self.text2,self.pct]\n",
    "        if self.pct > 49:\n",
    "            self.listans.append('Yes')\n",
    "        else:\n",
    "            self.listans.append('No')\n",
    "        \n",
    "        if self.verbose == 2:\n",
    "            self.listans.append(self.wiki1list)\n",
    "            self.listans.append(self.wiki2list)\n",
    "        \n",
    "        return self.listans\n",
    "    \n",
    "    def help(self):\n",
    "        print(\"To start, assign var = comparewiki.similar('arg1','arg2', verbose=1). arg1 and arg2 are names of the wikipedia articles, while verbose=1 prints the probability score and mathematical calculation. verbose=2 additionally prints array of words for each article, and verbose=0 disables any logs. To get values in a list for storage, use .ans(). To get the 40 common words for comparison, use .words()\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0 seconds.\n",
      "Scrape wikipedia for article 1: 3.8429 seconds.\n",
      "Scrape wikipedia for article 2: 5.8538 seconds.\n",
      "Get list of 40 words for both articles: 0.0036 seconds.\n",
      "Calculate similarity for both articles: 1.428 seconds.\n",
      "Probability of topics being related is 97%\n",
      "Count is 407 and sum is 396.52122327972694\n",
      "Print output: 0.0020904541015625 seconds.\n"
     ]
    }
   ],
   "source": [
    "a = similar('Joe Biden','Donald Trump')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0 seconds.\n",
      "Scrape wikipedia for article 1: 4.5453 seconds.\n",
      "Scrape wikipedia for article 2: 4.9795 seconds.\n",
      "Get list of 40 words for both articles: 0.003 seconds.\n",
      "Calculate similarity for both articles: 1.086 seconds.\n",
      "Probability of topics being related is 77%\n",
      "Count is 349 and sum is 268.68840002368\n",
      "Print output: 0.0019996166229248047 seconds.\n"
     ]
    }
   ],
   "source": [
    "b = similar('John F Kennedy','Donald Trump')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0 seconds.\n",
      "Scrape wikipedia for article 1: 2.5158 seconds.\n",
      "Scrape wikipedia for article 2: 6.3029 seconds.\n",
      "Get list of 40 words for both articles: 0.003 seconds.\n",
      "Calculate similarity for both articles: 1.2261 seconds.\n",
      "Probability of topics being related is 43%\n",
      "Count is 385 and sum is 166.8504644575315\n",
      "Print output: 0.002000570297241211 seconds.\n"
     ]
    }
   ],
   "source": [
    "b = similar('Tony Blair','Donald Trump')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0029976367950439453 seconds.\n",
      "Scrape wikipedia for article 1: 2.27 seconds.\n",
      "\n",
      "The title \"HP\" you specified is ambiguous. As a result, you are linked to a clarification page.\n",
      "\n",
      "\n",
      "Here are some suggestions to use: \n",
      "\n",
      "None\n",
      "None\n",
      "None\n",
      "wiktionary:HP\n",
      "wiktionary:hp\n",
      "HP Inc.\n",
      "Horsepower\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "Harry Pearson (audio critic)\n",
      "Hit points\n",
      "Horse-Power: Ballet Symphony\n",
      "Hewlett-Packard\n",
      "Hewlett Packard Enterprise\n",
      "Penguin Group\n",
      "HP Foods\n",
      "HP Sauce\n",
      "Handley Page\n",
      "Hindustan Petroleum\n",
      "America West Airlines\n",
      "Heart Peaks\n",
      "Himachal Pradesh\n",
      "Hunters Point (San Francisco)\n",
      "HP postcode area\n",
      "Haptoglobin\n",
      "Hypersensitivity pneumonitis\n",
      "Ilford HP\n",
      "High precipitation supercell\n",
      "High pressure\n",
      "Hollow point\n",
      "Horizontal pitch\n",
      "Security hacker\n",
      "Phreaking\n",
      "Half-pay\n",
      "Haltepunkt\n",
      "Hire purchase\n",
      "Howiesons Poort\n",
      "Hybrid perpetual\n",
      "HP Garage\n",
      "None\n",
      "Scrape wikipedia for article 2: 0.1462 seconds.\n",
      "Get list of 40 words for both articles: 0.002 seconds.\n",
      "Calculate similarity for both articles: 1.7934 seconds.\n",
      "Probability of topics being related is 8%\n",
      "Count is 400 and sum is 30.146690798843782\n",
      "Print output: 0.0016491413116455078 seconds.\n"
     ]
    }
   ],
   "source": [
    "x = similar('Tony Blair','HP')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0007612705230712891 seconds.\n",
      "\n",
      "The title \"NP\" you specified is ambiguous. As a result, you are linked to a clarification page.\n",
      "\n",
      "\n",
      "Here are some suggestions to use: \n",
      "\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "NP (novel)\n",
      "Banana Yoshimoto\n",
      "National Party (disambiguation)\n",
      "Ngee Ann Polytechnic\n",
      "Nigeria Police Force\n",
      "Northern Pacific Railway\n",
      "November Project\n",
      "NP postcode area\n",
      "Nepal\n",
      ".np\n",
      "Nucleoside phosphorylase\n",
      "Nurse practitioner\n",
      "Kallikrein 8\n",
      "Neptunium\n",
      "NP (complexity)\n",
      "NP-complete\n",
      "NP-hard\n",
      "Co-NP\n",
      "Numpy\n",
      "Named Pipe\n",
      "P-n junction\n",
      "Neper\n",
      "Neptunium\n",
      "Power number\n",
      "International Organization for Standardization\n",
      "Not populated\n",
      "Not placed\n",
      "Nanoparticle\n",
      "Noun phrase\n",
      "Notary public\n",
      "Enpi (disambiguation)\n",
      "None\n",
      "Scrape wikipedia for article 1: 0.5552 seconds.\n",
      "\n",
      "The title \"HP\" you specified is ambiguous. As a result, you are linked to a clarification page.\n",
      "\n",
      "\n",
      "Here are some suggestions to use: \n",
      "\n",
      "None\n",
      "None\n",
      "None\n",
      "wiktionary:HP\n",
      "wiktionary:hp\n",
      "HP Inc.\n",
      "Horsepower\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "None\n",
      "Harry Pearson (audio critic)\n",
      "Hit points\n",
      "Horse-Power: Ballet Symphony\n",
      "Hewlett-Packard\n",
      "Hewlett Packard Enterprise\n",
      "Penguin Group\n",
      "HP Foods\n",
      "HP Sauce\n",
      "Handley Page\n",
      "Hindustan Petroleum\n",
      "America West Airlines\n",
      "Heart Peaks\n",
      "Himachal Pradesh\n",
      "Hunters Point (San Francisco)\n",
      "HP postcode area\n",
      "Haptoglobin\n",
      "Hypersensitivity pneumonitis\n",
      "Ilford HP\n",
      "High precipitation supercell\n",
      "High pressure\n",
      "Hollow point\n",
      "Horizontal pitch\n",
      "Security hacker\n",
      "Phreaking\n",
      "Half-pay\n",
      "Haltepunkt\n",
      "Hire purchase\n",
      "Howiesons Poort\n",
      "Hybrid perpetual\n",
      "HP Garage\n",
      "None\n",
      "Scrape wikipedia for article 2: 0.1312 seconds.\n",
      "Get list of 40 words for both articles: 0.001 seconds.\n",
      "Calculate similarity for both articles: 2.0871 seconds.\n",
      "Probability of topics being related is 77%\n",
      "Count is 464 and sum is 355.04877498869314\n",
      "Print output: 0.0020055770874023438 seconds.\n"
     ]
    }
   ],
   "source": [
    "z = similar('NP','HP')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0 seconds.\n",
      "Scrape wikipedia for article 1: 0.7086 seconds.\n",
      "Scrape wikipedia for article 2: 0.9174 seconds.\n",
      "Get list of 40 words for both articles: 0.002 seconds.\n",
      "Calculate similarity for both articles: 1.6015 seconds.\n",
      "Probability of topics being related is 95%\n",
      "Count is 511 and sum is 486.3398758941804\n",
      "Print output: 0.0017197132110595703 seconds.\n"
     ]
    }
   ],
   "source": [
    "trance = similar('Armin van Buuren','Tiesto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize NLTK similarity corpus reader: 0.0 seconds.\n",
      "String check of arguments: 0.0009982585906982422 seconds.\n",
      "Scrape wikipedia for article 1: 2.9781 seconds.\n",
      "Scrape wikipedia for article 2: 2.9265 seconds.\n",
      "Get list of 40 words for both articles: 0.003 seconds.\n",
      "Calculate similarity for both articles: 1.3053 seconds.\n",
      "Probability of topics being related is 95%\n",
      "Count is 408 and sum is 387.96742212073457\n",
      "Print output: 0.002003192901611328 seconds.\n"
     ]
    }
   ],
   "source": [
    "country = similar('Singapore','Japan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}