08 Amazon Fine Food Reviews Analysis_Decision Trees.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn import tree\n",
    "iris = load_iris()\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf = clf.fit(iris.data, iris.target)\n",
    "clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ExecutableNotFound",
     "evalue": "failed to execute ['dot', '-Tpdf', '-O', 'iris'], make sure the Graphviz executables are on your systems' PATH",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\backend.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(cmd, input, capture_output, check, quiet, **kwargs)\u001b[0m\n\u001b[0;32m    146\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 147\u001b[1;33m         \u001b[0mproc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstartupinfo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mget_startupinfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    148\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)\u001b[0m\n\u001b[0;32m    768\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 769\u001b[1;33m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[0;32m    770\u001b[0m         \u001b[1;32mexcept\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[1;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)\u001b[0m\n\u001b[0;32m   1171\u001b[0m                                          \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfspath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcwd\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcwd\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1172\u001b[1;33m                                          startupinfo)\n\u001b[0m\u001b[0;32m   1173\u001b[0m             \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 2] The system cannot find the file specified",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mExecutableNotFound\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-21cd8f554c7e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mdot_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_graphviz\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_file\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgraphviz\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSource\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdot_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"iris\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\files.py\u001b[0m in \u001b[0;36mrender\u001b[1;34m(self, filename, directory, view, cleanup, format, renderer, formatter)\u001b[0m\n\u001b[0;32m    186\u001b[0m             \u001b[0mformat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_format\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    187\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 188\u001b[1;33m         \u001b[0mrendered\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilepath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformatter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    189\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    190\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcleanup\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\backend.py\u001b[0m in \u001b[0;36mrender\u001b[1;34m(engine, format, filepath, renderer, formatter, quiet)\u001b[0m\n\u001b[0;32m    181\u001b[0m     \"\"\"\n\u001b[0;32m    182\u001b[0m     \u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrendered\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcommand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilepath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformatter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 183\u001b[1;33m     \u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcapture_output\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcheck\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquiet\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mquiet\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    184\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mrendered\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\backend.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(cmd, input, capture_output, check, quiet, **kwargs)\u001b[0m\n\u001b[0;32m    148\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    149\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mENOENT\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 150\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mExecutableNotFound\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    151\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pragma: no cover\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    152\u001b[0m             \u001b[1;32mraise\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mExecutableNotFound\u001b[0m: failed to execute ['dot', '-Tpdf', '-O', 'iris'], make sure the Graphviz executables are on your systems' PATH"
     ]
    }
   ],
   "source": [
    "import graphviz \n",
    "dot_data = tree.export_graphviz(clf, out_file=None) \n",
    "graph = graphviz.Source(dot_data) \n",
    "graph.render(\"iris\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IXmtUCbYiWM3"
   },
   "source": [
    "# Amazon Fine Food Reviews Analysis\n",
    "\n",
    "\n",
    "Data Source: https://www.kaggle.com/snap/amazon-fine-food-reviews <br>\n",
    "\n",
    "EDA: https://nycdatascience.com/blog/student-works/amazon-fine-foods-visualization/\n",
    "\n",
    "\n",
    "The Amazon Fine Food Reviews dataset consists of reviews of fine foods from Amazon.<br>\n",
    "\n",
    "Number of reviews: 568,454<br>\n",
    "Number of users: 256,059<br>\n",
    "Number of products: 74,258<br>\n",
    "Timespan: Oct 1999 - Oct 2012<br>\n",
    "Number of Attributes/Columns in data: 10 \n",
    "\n",
    "Attribute Information:\n",
    "\n",
    "1. Id\n",
    "2. ProductId - unique identifier for the product\n",
    "3. UserId - unqiue identifier for the user\n",
    "4. ProfileName\n",
    "5. HelpfulnessNumerator - number of users who found the review helpful\n",
    "6. HelpfulnessDenominator - number of users who indicated whether they found the review helpful or not\n",
    "7. Score - rating between 1 and 5\n",
    "8. Time - timestamp for the review\n",
    "9. Summary - brief summary of the review\n",
    "10. Text - text of the review\n",
    "\n",
    "\n",
    "#### Objective:\n",
    "Given a review, determine whether the review is positive (rating of 4 or 5) or negative (rating of 1 or 2).\n",
    "\n",
    "<br>\n",
    "[Q] How to determine if a review is positive or negative?<br>\n",
    "<br> \n",
    "[Ans] We could use Score/Rating. A rating of 4 or 5 can be cosnidered as a positive review. A rating of 1 or 2 can be considered as negative one. A review of rating 3 is considered nuetral and such reviews are ignored from our analysis. This is an approximate and proxy way of determining the polarity (positivity/negativity) of a review.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "syeYHDR_iWM7"
   },
   "source": [
    "# [1]. Reading Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D_eZrQC0iWM9"
   },
   "source": [
    "## [1.1] Loading the data\n",
    "\n",
    "The dataset is available in two forms\n",
    "1. .csv file\n",
    "2. SQLite Database\n",
    "\n",
    "In order to load the data, We have used the SQLITE dataset as it is easier to query the data and visualise the data efficiently.\n",
    "<br> \n",
    "\n",
    "Here as we only want to get the global sentiment of the recommendations (positive or negative), we will purposefully ignore all Scores equal to 3. If the score is above 3, then the recommendation wil be set to \"positive\". Otherwise, it will be set to \"negative\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "53fLFqf8iWNA"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "import re\n",
    "# Tutorial about Python regular expressions: https://pymotw.com/2/re/\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors\n",
    "import pickle\n",
    "\n",
    "from tqdm import tqdm\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j3kshrC-iWNI",
    "outputId": "07826916-2c3f-4a0a-e928-ddbc6d567a11"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data points in our data (5000, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>B001E4KFG0</td>\n",
       "      <td>A3SGXH7AUHU8GW</td>\n",
       "      <td>delmartian</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1303862400</td>\n",
       "      <td>Good Quality Dog Food</td>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>B00813GRG4</td>\n",
       "      <td>A1D87F6ZCVE5NK</td>\n",
       "      <td>dll pa</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1346976000</td>\n",
       "      <td>Not as Advertised</td>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>B000LQOCH0</td>\n",
       "      <td>ABXLMWJIXXAIN</td>\n",
       "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1219017600</td>\n",
       "      <td>\"Delight\" says it all</td>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id   ProductId          UserId                      ProfileName  \\\n",
       "0   1  B001E4KFG0  A3SGXH7AUHU8GW                       delmartian   \n",
       "1   2  B00813GRG4  A1D87F6ZCVE5NK                           dll pa   \n",
       "2   3  B000LQOCH0   ABXLMWJIXXAIN  Natalia Corres \"Natalia Corres\"   \n",
       "\n",
       "   HelpfulnessNumerator  HelpfulnessDenominator  Score        Time  \\\n",
       "0                     1                       1      1  1303862400   \n",
       "1                     0                       0      0  1346976000   \n",
       "2                     1                       1      1  1219017600   \n",
       "\n",
       "                 Summary                                               Text  \n",
       "0  Good Quality Dog Food  I have bought several of the Vitality canned d...  \n",
       "1      Not as Advertised  Product arrived labeled as Jumbo Salted Peanut...  \n",
       "2  \"Delight\" says it all  This is a confection that has been around a fe...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using SQLite Table to read data.\n",
    "con = sqlite3.connect('database.sqlite') \n",
    "\n",
    "# filtering only positive and negative reviews i.e. \n",
    "# not taking into consideration those reviews with Score=3\n",
    "# SELECT * FROM Reviews WHERE Score != 3 LIMIT 500000, will give top 500000 data points\n",
    "# you can change the number to any other number based on your computing power\n",
    "\n",
    "# filtered_data = pd.read_sql_query(\"\"\" SELECT * FROM Reviews WHERE Score != 3 LIMIT 500000\"\"\", con) \n",
    "# for tsne assignment you can take 5k data points\n",
    "\n",
    "filtered_data = pd.read_sql_query(\"\"\" SELECT * FROM Reviews WHERE Score != 3 LIMIT 5000\"\"\", con) \n",
    "\n",
    "# Give reviews with Score>3 a positive rating(1), and reviews with a score<3 a negative rating(0).\n",
    "def partition(x):\n",
    "    if x < 3:\n",
    "        return 0\n",
    "    return 1\n",
    "\n",
    "#changing reviews with score less than 3 to be positive and vice-versa\n",
    "actualScore = filtered_data['Score']\n",
    "positiveNegative = actualScore.map(partition) \n",
    "filtered_data['Score'] = positiveNegative\n",
    "print(\"Number of data points in our data\", filtered_data.shape)\n",
    "filtered_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-7Iy8yWfiWNR"
   },
   "outputs": [],
   "source": [
    "display = pd.read_sql_query(\"\"\"\n",
    "SELECT UserId, ProductId, ProfileName, Time, Score, Text, COUNT(*)\n",
    "FROM Reviews\n",
    "GROUP BY UserId\n",
    "HAVING COUNT(*)>1\n",
    "\"\"\", con)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wuK6OYpOiWNV",
    "outputId": "5cb5082e-47b9-4d4b-cf90-fc6ffe0dceeb",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80668, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>Time</th>\n",
       "      <th>Score</th>\n",
       "      <th>Text</th>\n",
       "      <th>COUNT(*)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#oc-R115TNMSPFT9I7</td>\n",
       "      <td>B005ZBZLT4</td>\n",
       "      <td>Breyton</td>\n",
       "      <td>1331510400</td>\n",
       "      <td>2</td>\n",
       "      <td>Overall its just OK when considering the price...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#oc-R11D9D7SHXIJB9</td>\n",
       "      <td>B005HG9ESG</td>\n",
       "      <td>Louis E. Emory \"hoppy\"</td>\n",
       "      <td>1342396800</td>\n",
       "      <td>5</td>\n",
       "      <td>My wife has recurring extreme muscle spasms, u...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>#oc-R11DNU2NBKQ23Z</td>\n",
       "      <td>B005ZBZLT4</td>\n",
       "      <td>Kim Cieszykowski</td>\n",
       "      <td>1348531200</td>\n",
       "      <td>1</td>\n",
       "      <td>This coffee is horrible and unfortunately not ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>#oc-R11O5J5ZVQE25C</td>\n",
       "      <td>B005HG9ESG</td>\n",
       "      <td>Penguin Chick</td>\n",
       "      <td>1346889600</td>\n",
       "      <td>5</td>\n",
       "      <td>This will be the bottle that you grab from the...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>#oc-R12KPBODL2B5ZD</td>\n",
       "      <td>B007OSBEV0</td>\n",
       "      <td>Christopher P. Presta</td>\n",
       "      <td>1348617600</td>\n",
       "      <td>1</td>\n",
       "      <td>I didnt like this coffee. Instead of telling y...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               UserId   ProductId             ProfileName        Time  Score  \\\n",
       "0  #oc-R115TNMSPFT9I7  B005ZBZLT4                 Breyton  1331510400      2   \n",
       "1  #oc-R11D9D7SHXIJB9  B005HG9ESG  Louis E. Emory \"hoppy\"  1342396800      5   \n",
       "2  #oc-R11DNU2NBKQ23Z  B005ZBZLT4        Kim Cieszykowski  1348531200      1   \n",
       "3  #oc-R11O5J5ZVQE25C  B005HG9ESG           Penguin Chick  1346889600      5   \n",
       "4  #oc-R12KPBODL2B5ZD  B007OSBEV0   Christopher P. Presta  1348617600      1   \n",
       "\n",
       "                                                Text  COUNT(*)  \n",
       "0  Overall its just OK when considering the price...         2  \n",
       "1  My wife has recurring extreme muscle spasms, u...         3  \n",
       "2  This coffee is horrible and unfortunately not ...         2  \n",
       "3  This will be the bottle that you grab from the...         3  \n",
       "4  I didnt like this coffee. Instead of telling y...         2  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(display.shape)\n",
    "display.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OMKVKkH7iWNZ",
    "outputId": "acc9d31a-27a1-48a1-d2f2-ce1f2e511793"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>Time</th>\n",
       "      <th>Score</th>\n",
       "      <th>Text</th>\n",
       "      <th>COUNT(*)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>80638</th>\n",
       "      <td>AZY10LLTJ71NX</td>\n",
       "      <td>B001ATMQK2</td>\n",
       "      <td>undertheshrine \"undertheshrine\"</td>\n",
       "      <td>1296691200</td>\n",
       "      <td>5</td>\n",
       "      <td>I bought this 6 pack because for the price tha...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              UserId   ProductId                      ProfileName        Time  \\\n",
       "80638  AZY10LLTJ71NX  B001ATMQK2  undertheshrine \"undertheshrine\"  1296691200   \n",
       "\n",
       "       Score                                               Text  COUNT(*)  \n",
       "80638      5  I bought this 6 pack because for the price tha...         5  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display[display['UserId']=='AZY10LLTJ71NX']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s37uqS_jiWNe",
    "outputId": "489f08d4-48a2-4e82-bfc1-4f982f35b2e4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "393063"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display['COUNT(*)'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "efnNNqNfiWNh"
   },
   "source": [
    "#  [2] Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Eq0hnbxhiWNi"
   },
   "source": [
    "## [2.1] Data Cleaning: Deduplication\n",
    "\n",
    "It is observed (as shown in the table below) that the reviews data had many duplicate entries. Hence it was necessary to remove duplicates in order to get unbiased results for the analysis of the data.  Following is an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3CKLJ2FFiWNj",
    "outputId": "c995e0c8-7ef1-48a0-d7a9-b0cc50efaa83",
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78445</td>\n",
       "      <td>B000HDL1RQ</td>\n",
       "      <td>AR5J8UI46CURR</td>\n",
       "      <td>Geetha Krishnan</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1199577600</td>\n",
       "      <td>LOACKER QUADRATINI VANILLA WAFERS</td>\n",
       "      <td>DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>138317</td>\n",
       "      <td>B000HDOPYC</td>\n",
       "      <td>AR5J8UI46CURR</td>\n",
       "      <td>Geetha Krishnan</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1199577600</td>\n",
       "      <td>LOACKER QUADRATINI VANILLA WAFERS</td>\n",
       "      <td>DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>138277</td>\n",
       "      <td>B000HDOPYM</td>\n",
       "      <td>AR5J8UI46CURR</td>\n",
       "      <td>Geetha Krishnan</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1199577600</td>\n",
       "      <td>LOACKER QUADRATINI VANILLA WAFERS</td>\n",
       "      <td>DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>73791</td>\n",
       "      <td>B000HDOPZG</td>\n",
       "      <td>AR5J8UI46CURR</td>\n",
       "      <td>Geetha Krishnan</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1199577600</td>\n",
       "      <td>LOACKER QUADRATINI VANILLA WAFERS</td>\n",
       "      <td>DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>155049</td>\n",
       "      <td>B000PAQ75C</td>\n",
       "      <td>AR5J8UI46CURR</td>\n",
       "      <td>Geetha Krishnan</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1199577600</td>\n",
       "      <td>LOACKER QUADRATINI VANILLA WAFERS</td>\n",
       "      <td>DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Id   ProductId         UserId      ProfileName  HelpfulnessNumerator  \\\n",
       "0   78445  B000HDL1RQ  AR5J8UI46CURR  Geetha Krishnan                     2   \n",
       "1  138317  B000HDOPYC  AR5J8UI46CURR  Geetha Krishnan                     2   \n",
       "2  138277  B000HDOPYM  AR5J8UI46CURR  Geetha Krishnan                     2   \n",
       "3   73791  B000HDOPZG  AR5J8UI46CURR  Geetha Krishnan                     2   \n",
       "4  155049  B000PAQ75C  AR5J8UI46CURR  Geetha Krishnan                     2   \n",
       "\n",
       "   HelpfulnessDenominator  Score        Time  \\\n",
       "0                       2      5  1199577600   \n",
       "1                       2      5  1199577600   \n",
       "2                       2      5  1199577600   \n",
       "3                       2      5  1199577600   \n",
       "4                       2      5  1199577600   \n",
       "\n",
       "                             Summary  \\\n",
       "0  LOACKER QUADRATINI VANILLA WAFERS   \n",
       "1  LOACKER QUADRATINI VANILLA WAFERS   \n",
       "2  LOACKER QUADRATINI VANILLA WAFERS   \n",
       "3  LOACKER QUADRATINI VANILLA WAFERS   \n",
       "4  LOACKER QUADRATINI VANILLA WAFERS   \n",
       "\n",
       "                                                Text  \n",
       "0  DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...  \n",
       "1  DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...  \n",
       "2  DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...  \n",
       "3  DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...  \n",
       "4  DELICIOUS WAFERS. I FIND THAT EUROPEAN WAFERS ...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display= pd.read_sql_query(\"\"\"\n",
    "SELECT *\n",
    "FROM Reviews\n",
    "WHERE Score != 3 AND UserId=\"AR5J8UI46CURR\"\n",
    "ORDER BY ProductID\n",
    "\"\"\", con)\n",
    "display.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GmHe67iviWNn"
   },
   "source": [
    "As it can be seen above that same user has multiple reviews with same values for HelpfulnessNumerator, HelpfulnessDenominator, Score, Time, Summary and Text and on doing analysis it was found that <br>\n",
    "<br> \n",
    "ProductId=B000HDOPZG was Loacker Quadratini Vanilla Wafer Cookies, 8.82-Ounce Packages (Pack of 8)<br>\n",
    "<br> \n",
    "ProductId=B000HDL1RQ was Loacker Quadratini Lemon Wafer Cookies, 8.82-Ounce Packages (Pack of 8) and so on<br>\n",
    "\n",
    "It was inferred after analysis that reviews with same parameters other than ProductId belonged to the same product just having different flavour or quantity. Hence in order to reduce redundancy it was decided to eliminate the rows having same parameters.<br>\n",
    "\n",
    "The method used for the same was that we first sort the data according to ProductId and then just keep the first similar product review and delelte the others. for eg. in the above just the review for ProductId=B000HDL1RQ remains. This method ensures that there is only one representative for each product and deduplication without sorting would lead to possibility of different representatives still existing for the same product."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wgmdXIJviWNn"
   },
   "outputs": [],
   "source": [
    "#Sorting data according to ProductId in ascending order\n",
    "sorted_data=filtered_data.sort_values('ProductId', axis=0, ascending=True, inplace=False, kind='quicksort', na_position='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ComjQ0rHiWNp",
    "outputId": "1c155404-4bb7-4078-c46b-1a9a33ac0602"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4986, 10)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Deduplication of entries\n",
    "final=sorted_data.drop_duplicates(subset={\"UserId\",\"ProfileName\",\"Time\",\"Text\"}, keep='first', inplace=False)\n",
    "final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yFrWiU23iWNt",
    "outputId": "b9d26952-d8af-4fbb-eb3d-c6a28abcc029"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99.72"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking to see how much % of data still remains\n",
    "(final['Id'].size*1.0)/(filtered_data['Id'].size*1.0)*100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "g4lvCHn6iWNw"
   },
   "source": [
    "<b>Observation:-</b> It was also seen that in two rows given below the value of HelpfulnessNumerator is greater than HelpfulnessDenominator which is not practically possible hence these two rows too are removed from calcualtions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6ehzwEdgiWNw",
    "outputId": "60e18da4-784f-49d8-84b7-59823cfa3e85"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64422</td>\n",
       "      <td>B000MIDROQ</td>\n",
       "      <td>A161DK06JJMCYF</td>\n",
       "      <td>J. E. Stephens \"Jeanne\"</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1224892800</td>\n",
       "      <td>Bought This for My Son at College</td>\n",
       "      <td>My son loves spaghetti so I didn't hesitate or...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44737</td>\n",
       "      <td>B001EQ55RW</td>\n",
       "      <td>A2V0I904FH7ABY</td>\n",
       "      <td>Ram</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1212883200</td>\n",
       "      <td>Pure cocoa taste with crunchy almonds inside</td>\n",
       "      <td>It was almost a 'love at first bite' - the per...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Id   ProductId          UserId              ProfileName  \\\n",
       "0  64422  B000MIDROQ  A161DK06JJMCYF  J. E. Stephens \"Jeanne\"   \n",
       "1  44737  B001EQ55RW  A2V0I904FH7ABY                      Ram   \n",
       "\n",
       "   HelpfulnessNumerator  HelpfulnessDenominator  Score        Time  \\\n",
       "0                     3                       1      5  1224892800   \n",
       "1                     3                       2      4  1212883200   \n",
       "\n",
       "                                        Summary  \\\n",
       "0             Bought This for My Son at College   \n",
       "1  Pure cocoa taste with crunchy almonds inside   \n",
       "\n",
       "                                                Text  \n",
       "0  My son loves spaghetti so I didn't hesitate or...  \n",
       "1  It was almost a 'love at first bite' - the per...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display= pd.read_sql_query(\"\"\"\n",
    "SELECT *\n",
    "FROM Reviews\n",
    "WHERE Score != 3 AND Id=44737 OR Id=64422\n",
    "ORDER BY ProductID\n",
    "\"\"\", con)\n",
    "\n",
    "display.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EBa1d6gViWNz"
   },
   "outputs": [],
   "source": [
    "final=final[final.HelpfulnessNumerator<=final.HelpfulnessDenominator]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KNQ01asjiWN1",
    "outputId": "0d2433fb-dd33-41cd-a18d-110d776f60b4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4986, 10)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1    4178\n",
       "0     808\n",
       "Name: Score, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Before starting the next phase of preprocessing lets see the number of entries left\n",
    "print(final.shape)\n",
    "\n",
    "#How many positive and negative reviews are present in our dataset?\n",
    "final['Score'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "92IZ5ea0iWN4"
   },
   "source": [
    "#  [3] Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NTwQITJKiWN5"
   },
   "source": [
    "## [3.1].  Preprocessing Review Text\n",
    "\n",
    "Now that we have finished deduplication our data requires some preprocessing before we go on further with analysis and making the prediction model.\n",
    "\n",
    "Hence in the Preprocessing phase we do the following in the order below:-\n",
    "\n",
    "1. Begin by removing the html tags\n",
    "2. Remove any punctuations or limited set of special characters like , or . or # etc.\n",
    "3. Check if the word is made up of english letters and is not alpha-numeric\n",
    "4. Check to see if the length of the word is greater than 2 (as it was researched that there is no adjective in 2-letters)\n",
    "5. Convert the word to lowercase\n",
    "6. Remove Stopwords\n",
    "7. Finally Snowball Stemming the word (it was obsereved to be better than Porter Stemming)<br>\n",
    "\n",
    "After which we collect the words used to describe positive and negative reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kTCxT5aWiWN5",
    "outputId": "64588cda-64b3-47f0-8d01-941e1cc85cc5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Why is this $[...] when the same product is available for $[...] here?<br />https://www.amazon.com/VICTOR-FLY-MAGNET-BAIT-REFILL/dp/B00004RBDY<br /><br />The Victor M380 and M502 traps are unreal, of course -- total fly genocide. Pretty stinky, but only right nearby.\n",
      "==================================================\n",
      "I recently tried this flavor/brand and was surprised at how delicious these chips are.  The best thing was that there were a lot of \"brown\" chips in the bsg (my favorite), so I bought some more through amazon and shared with family and friends.  I am a little disappointed that there are not, so far, very many brown chips in these bags, but the flavor is still very good.  I like them better than the yogurt and green onion flavor because they do not seem to be as salty, and the onion flavor is better.  If you haven't eaten Kettle chips before, I recommend that you try a bag before buying bulk.  They are thicker and crunchier than Lays but just as fresh out of the bag.\n",
      "==================================================\n",
      "Wow.  So far, two two-star reviews.  One obviously had no idea what they were ordering; the other wants crispy cookies.  Hey, I'm sorry; but these reviews do nobody any good beyond reminding us to look  before ordering.<br /><br />These are chocolate-oatmeal cookies.  If you don't like that combination, don't order this type of cookie.  I find the combo quite nice, really.  The oatmeal sort of \"calms\" the rich chocolate flavor and gives the cookie sort of a coconut-type consistency.  Now let's also remember that tastes differ; so, I've given my opinion.<br /><br />Then, these are soft, chewy cookies -- as advertised.  They are not \"crispy\" cookies, or the blurb would say \"crispy,\" rather than \"chewy.\"  I happen to like raw cookie dough; however, I don't see where these taste like raw cookie dough.  Both are soft, however, so is this the confusion?  And, yes, they stick together.  Soft cookies tend to do that.  They aren't individually wrapped, which would add to the cost.  Oh yeah, chocolate chip cookies tend to be somewhat sweet.<br /><br />So, if you want something hard and crisp, I suggest Nabiso's Ginger Snaps.  If you want a cookie that's soft, chewy and tastes like a combination of chocolate and oatmeal, give these a try.  I'm here to place my second order.\n",
      "==================================================\n",
      "love to order my coffee on amazon.  easy and shows up quickly.<br />This k cup is great coffee.  dcaf is very good as well\n",
      "==================================================\n"
     ]
    }
   ],
   "source": [
    "# printing some random reviews\n",
    "sent_0 = final['Text'].values[0]\n",
    "print(sent_0)\n",
    "print(\"=\"*50)\n",
    "\n",
    "sent_1000 = final['Text'].values[1000]\n",
    "print(sent_1000)\n",
    "print(\"=\"*50)\n",
    "\n",
    "sent_1500 = final['Text'].values[1500]\n",
    "print(sent_1500)\n",
    "print(\"=\"*50)\n",
    "\n",
    "sent_4900 = final['Text'].values[4900]\n",
    "print(sent_4900)\n",
    "print(\"=\"*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iX3VndCPiWN8",
    "outputId": "7f8cad02-65a6-4530-e7e1-bb6775877737"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Why is this $[...] when the same product is available for $[...] here?<br /> /><br />The Victor M380 and M502 traps are unreal, of course -- total fly genocide. Pretty stinky, but only right nearby.\n"
     ]
    }
   ],
   "source": [
    "# remove urls from text python: https://stackoverflow.com/a/40823105/4084039\n",
    "sent_0 = re.sub(r\"http\\S+\", \"\", sent_0)\n",
    "sent_1000 = re.sub(r\"http\\S+\", \"\", sent_1000)\n",
    "sent_150 = re.sub(r\"http\\S+\", \"\", sent_1500)\n",
    "sent_4900 = re.sub(r\"http\\S+\", \"\", sent_4900)\n",
    "\n",
    "print(sent_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DyO7iqEriWN_",
    "outputId": "2a010a1c-574d-409f-d725-c726a25d3e70"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Why is this $[...] when the same product is available for $[...] here? />The Victor M380 and M502 traps are unreal, of course -- total fly genocide. Pretty stinky, but only right nearby.\n",
      "==================================================\n",
      "I recently tried this flavor/brand and was surprised at how delicious these chips are.  The best thing was that there were a lot of \"brown\" chips in the bsg (my favorite), so I bought some more through amazon and shared with family and friends.  I am a little disappointed that there are not, so far, very many brown chips in these bags, but the flavor is still very good.  I like them better than the yogurt and green onion flavor because they do not seem to be as salty, and the onion flavor is better.  If you haven't eaten Kettle chips before, I recommend that you try a bag before buying bulk.  They are thicker and crunchier than Lays but just as fresh out of the bag.\n",
      "==================================================\n",
      "Wow.  So far, two two-star reviews.  One obviously had no idea what they were ordering; the other wants crispy cookies.  Hey, I'm sorry; but these reviews do nobody any good beyond reminding us to look  before ordering.These are chocolate-oatmeal cookies.  If you don't like that combination, don't order this type of cookie.  I find the combo quite nice, really.  The oatmeal sort of \"calms\" the rich chocolate flavor and gives the cookie sort of a coconut-type consistency.  Now let's also remember that tastes differ; so, I've given my opinion.Then, these are soft, chewy cookies -- as advertised.  They are not \"crispy\" cookies, or the blurb would say \"crispy,\" rather than \"chewy.\"  I happen to like raw cookie dough; however, I don't see where these taste like raw cookie dough.  Both are soft, however, so is this the confusion?  And, yes, they stick together.  Soft cookies tend to do that.  They aren't individually wrapped, which would add to the cost.  Oh yeah, chocolate chip cookies tend to be somewhat sweet.So, if you want something hard and crisp, I suggest Nabiso's Ginger Snaps.  If you want a cookie that's soft, chewy and tastes like a combination of chocolate and oatmeal, give these a try.  I'm here to place my second order.\n",
      "==================================================\n",
      "love to order my coffee on amazon.  easy and shows up quickly.This k cup is great coffee.  dcaf is very good as well\n"
     ]
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/16206380/python-beautifulsoup-how-to-remove-all-tags-from-an-element\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "soup = BeautifulSoup(sent_0, 'lxml')\n",
    "text = soup.get_text()\n",
    "print(text)\n",
    "print(\"=\"*50)\n",
    "\n",
    "soup = BeautifulSoup(sent_1000, 'lxml')\n",
    "text = soup.get_text()\n",
    "print(text)\n",
    "print(\"=\"*50)\n",
    "\n",
    "soup = BeautifulSoup(sent_1500, 'lxml')\n",
    "text = soup.get_text()\n",
    "print(text)\n",
    "print(\"=\"*50)\n",
    "\n",
    "soup = BeautifulSoup(sent_4900, 'lxml')\n",
    "text = soup.get_text()\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qzIaKh8niWOC"
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/a/47091490/4084039\n",
    "import re\n",
    "\n",
    "def decontracted(phrase):\n",
    "    # specific\n",
    "    phrase = re.sub(r\"won't\", \"will not\", phrase)\n",
    "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
    "\n",
    "    # general\n",
    "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
    "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
    "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
    "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
    "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
    "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
    "    return phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Esa29-JwiWOD",
    "outputId": "be726596-752d-489b-d311-f4180ac83ac4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wow.  So far, two two-star reviews.  One obviously had no idea what they were ordering; the other wants crispy cookies.  Hey, I am sorry; but these reviews do nobody any good beyond reminding us to look  before ordering.<br /><br />These are chocolate-oatmeal cookies.  If you do not like that combination, do not order this type of cookie.  I find the combo quite nice, really.  The oatmeal sort of \"calms\" the rich chocolate flavor and gives the cookie sort of a coconut-type consistency.  Now let is also remember that tastes differ; so, I have given my opinion.<br /><br />Then, these are soft, chewy cookies -- as advertised.  They are not \"crispy\" cookies, or the blurb would say \"crispy,\" rather than \"chewy.\"  I happen to like raw cookie dough; however, I do not see where these taste like raw cookie dough.  Both are soft, however, so is this the confusion?  And, yes, they stick together.  Soft cookies tend to do that.  They are not individually wrapped, which would add to the cost.  Oh yeah, chocolate chip cookies tend to be somewhat sweet.<br /><br />So, if you want something hard and crisp, I suggest Nabiso is Ginger Snaps.  If you want a cookie that is soft, chewy and tastes like a combination of chocolate and oatmeal, give these a try.  I am here to place my second order.\n",
      "==================================================\n"
     ]
    }
   ],
   "source": [
    "sent_1500 = decontracted(sent_1500)\n",
    "print(sent_1500)\n",
    "print(\"=\"*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qf0c7-nXiWOG",
    "outputId": "6104976d-bda8-4de1-9d91-808f3661d53b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Why is this $[...] when the same product is available for $[...] here?<br /> /><br />The Victor  and  traps are unreal, of course -- total fly genocide. Pretty stinky, but only right nearby.\n"
     ]
    }
   ],
   "source": [
    "#remove words with numbers python: https://stackoverflow.com/a/18082370/4084039\n",
    "sent_0 = re.sub(\"\\S*\\d\\S*\", \"\", sent_0).strip()\n",
    "print(sent_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1Z5VK6wxiWOI",
    "outputId": "d3405347-aa6a-4df5-c3d9-2c11441c2385"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wow So far two two star reviews One obviously had no idea what they were ordering the other wants crispy cookies Hey I am sorry but these reviews do nobody any good beyond reminding us to look before ordering br br These are chocolate oatmeal cookies If you do not like that combination do not order this type of cookie I find the combo quite nice really The oatmeal sort of calms the rich chocolate flavor and gives the cookie sort of a coconut type consistency Now let is also remember that tastes differ so I have given my opinion br br Then these are soft chewy cookies as advertised They are not crispy cookies or the blurb would say crispy rather than chewy I happen to like raw cookie dough however I do not see where these taste like raw cookie dough Both are soft however so is this the confusion And yes they stick together Soft cookies tend to do that They are not individually wrapped which would add to the cost Oh yeah chocolate chip cookies tend to be somewhat sweet br br So if you want something hard and crisp I suggest Nabiso is Ginger Snaps If you want a cookie that is soft chewy and tastes like a combination of chocolate and oatmeal give these a try I am here to place my second order \n"
     ]
    }
   ],
   "source": [
    "#remove spacial character: https://stackoverflow.com/a/5843547/4084039\n",
    "sent_1500 = re.sub('[^A-Za-z0-9]+', ' ', sent_1500)\n",
    "print(sent_1500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MfzZtIP1iWOL"
   },
   "outputs": [],
   "source": [
    "# https://gist.github.com/sebleier/554280\n",
    "# we are removing the words from the stop words list: 'no', 'nor', 'not'\n",
    "# <br /><br /> ==> after the above steps, we are getting \"br br\"\n",
    "# we are including them into stop words list\n",
    "# instead of <br /> if we have <br/> these tags would have revmoved in the 1st step\n",
    "\n",
    "stopwords= set(['br', 'the', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\",\\\n",
    "            \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \\\n",
    "            'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\\\n",
    "            'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', \\\n",
    "            'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \\\n",
    "            'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \\\n",
    "            'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\\\n",
    "            'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\\\n",
    "            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\\\n",
    "            'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \\\n",
    "            's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', \\\n",
    "            've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn',\\\n",
    "            \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn',\\\n",
    "            \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", \\\n",
    "            'won', \"won't\", 'wouldn', \"wouldn't\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g1C43tjNiWOM",
    "outputId": "a997afc9-894c-4a56-d538-dadea8a21e6d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 4986/4986 [00:02<00:00, 2010.35it/s]\n"
     ]
    }
   ],
   "source": [
    "# Combining all the above stundents \n",
    "from tqdm import tqdm\n",
    "preprocessed_reviews = []\n",
    "# tqdm is for printing the status bar\n",
    "for sentance in tqdm(final['Text'].values):\n",
    "    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n",
    "    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n",
    "    sentance = decontracted(sentance)\n",
    "    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n",
    "    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n",
    "    # https://gist.github.com/sebleier/554280\n",
    "    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n",
    "    preprocessed_reviews.append(sentance.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MWDrCFxViWOP",
    "outputId": "5618c9c3-a86a-42a4-e9c4-d24d93fd5a7a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'wow far two two star reviews one obviously no idea ordering wants crispy cookies hey sorry reviews nobody good beyond reminding us look ordering chocolate oatmeal cookies not like combination not order type cookie find combo quite nice really oatmeal sort calms rich chocolate flavor gives cookie sort coconut type consistency let also remember tastes differ given opinion soft chewy cookies advertised not crispy cookies blurb would say crispy rather chewy happen like raw cookie dough however not see taste like raw cookie dough soft however confusion yes stick together soft cookies tend not individually wrapped would add cost oh yeah chocolate chip cookies tend somewhat sweet want something hard crisp suggest nabiso ginger snaps want cookie soft chewy tastes like combination chocolate oatmeal give try place second order'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessed_reviews[1500]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fEu8sShciWOS"
   },
   "source": [
    "<h2><font color='red'>[3.2] Preprocessing Review Summary</font></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XP1URIERiWOU"
   },
   "outputs": [],
   "source": [
    "## Similartly you can do preprocessing for review summary also."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Important Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing all relevant libraries\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier \n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Splitting\n",
    "# Train Data :  60%\n",
    "# CV Data    :  20%\n",
    "# Test Data  :  20%\n",
    "\n",
    "X=preprocessed_reviews\n",
    "y=np.array(final['Score'])\n",
    "\n",
    "#####  SPLIT  #######\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4)\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X_train, y_train, test_size=0.25, random_state=4)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p2lyiwJ_iWOW"
   },
   "source": [
    "# [4] Featurization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "T_LFck8iiWOX"
   },
   "source": [
    "## [4.1] BAG OF WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OVHZ9f8biWOX",
    "outputId": "b121f6f4-e30a-4184-b083-a77cdab3ca19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "some feature names  ['aa', 'aahhhs', 'aback', 'abandon', 'abates', 'abby', 'ability', 'able', 'aboulutely', 'absence']\n",
      "==================================================\n",
      "the type of count vectorizer  <class 'scipy.sparse.csr.csr_matrix'>\n",
      "the shape of out text BOW vectorizer  (2991, 9983)\n",
      "the number of unique words  9983\n"
     ]
    }
   ],
   "source": [
    "#BoW\n",
    "X_train_bow, X_test_bow, X_cv_bow = X_train, X_test, X_cv\n",
    "y_train_bow, y_test_bow, y_cv_bow = y_train, y_test, y_cv\n",
    "\n",
    "count_vect = CountVectorizer() #in scikit-learn\n",
    "count_vect.fit(X_train_bow)\n",
    "print(\"some feature names \", count_vect.get_feature_names()[:10])\n",
    "print('='*50)\n",
    "\n",
    "bow_counts_train = count_vect.transform(X_train_bow)\n",
    "bow_counts_cv = count_vect.transform(X_cv_bow)\n",
    "bow_counts_test = count_vect.transform(X_test_bow)\n",
    "\n",
    "print(\"the type of count vectorizer \",type(bow_counts_train))\n",
    "print(\"the shape of out text BOW vectorizer \",bow_counts_train.get_shape())\n",
    "print(\"the number of unique words \", bow_counts_train.get_shape()[1])\n",
    "\n",
    "# bow_counts_test=bow_counts_test.toarray()\n",
    "# bow_counts_train=bow_counts_train.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0tcbbZxSiWOd"
   },
   "source": [
    "## [4.3] TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mQ3EDEsoiWOd",
    "outputId": "b305ed29-861d-40e9-c1df-b12c4e6492a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "some sample features(unique words in the corpus) ['able', 'able find', 'absolute', 'absolutely', 'absolutely love', 'acid', 'across', 'actual', 'actually', 'add']\n",
      "==================================================\n",
      "the type of count vectorizer  <class 'scipy.sparse.csr.csr_matrix'>\n",
      "the shape of out text TFIDF vectorizer  (2991, 1945)\n",
      "the number of unique words including both unigrams and bigrams  1945\n"
     ]
    }
   ],
   "source": [
    "X_train_tf, X_test_tf, X_cv_tf = X_train, X_test, X_cv\n",
    "y_train_tf, y_test_tf, y_cv_tf = y_train, y_test, y_cv\n",
    "\n",
    "tf_idf_vect = TfidfVectorizer(ngram_range=(1,2), min_df=10)\n",
    "final_tf_idf_train = tf_idf_vect.fit_transform(X_train_tf)\n",
    "print(\"some sample features(unique words in the corpus)\",tf_idf_vect.get_feature_names()[0:10])\n",
    "print('='*50)\n",
    "\n",
    "\n",
    "final_tf_idf_cv = tf_idf_vect.transform(X_cv_tf)\n",
    "final_tf_idf_test = tf_idf_vect.transform(X_test_tf)\n",
    "print(\"the type of count vectorizer \",type(final_tf_idf_train))\n",
    "print(\"the shape of out text TFIDF vectorizer \",final_tf_idf_train.get_shape())\n",
    "print(\"the number of unique words including both unigrams and bigrams \", final_tf_idf_train.get_shape()[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u566lt93iWOf"
   },
   "source": [
    "## [4.4] Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pxlbMwFjiWOg"
   },
   "outputs": [],
   "source": [
    "# Train your own Word2Vec model using your own text corpus\n",
    "\n",
    "X_train_w2v, X_test_w2v, X_cv_w2v = X_train, X_test, X_cv\n",
    "y_train_w2v, y_test_w2v, y_cv_w2v = y_train, y_test, y_cv\n",
    "\n",
    "\n",
    "i=0\n",
    "list_of_sentance_test=[]\n",
    "for sentance in X_test_w2v:\n",
    "    list_of_sentance_test.append(sentance.split())\n",
    "\n",
    "i=0\n",
    "list_of_sentance_cv=[]\n",
    "for sentance in X_cv_w2v:\n",
    "    list_of_sentance_cv.append(sentance.split())\n",
    "\n",
    "i=0\n",
    "list_of_sentance=[]\n",
    "for sentance in X_train_w2v:\n",
    "    list_of_sentance.append(sentance.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NOd5wp0yiWOh",
    "outputId": "ae28439d-438c-4d88-ed12-db5f3317218f",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('think', 0.9996241331100464), ('garlic', 0.9996083974838257), ('tasty', 0.9995932579040527), ('course', 0.9995924234390259), ('cookies', 0.9995923042297363), ('awesome', 0.9995913505554199), ('must', 0.9995895624160767), ('absolutely', 0.9995788931846619), ('nice', 0.9995783567428589), ('nothing', 0.9995714426040649)]\n",
      "==================================================\n",
      "[('together', 0.999501645565033), ('style', 0.999495267868042), ('similar', 0.9994926452636719), ('cereal', 0.999476432800293), ('enjoyed', 0.9994719624519348), ('sometimes', 0.9994703531265259), ('cost', 0.9994683265686035), ('including', 0.9994651079177856), ('school', 0.9994620084762573), ('work', 0.9994588494300842)]\n"
     ]
    }
   ],
   "source": [
    "# Using Google News Word2Vectors\n",
    "\n",
    "is_your_ram_gt_16g=False\n",
    "want_to_use_google_w2v = False\n",
    "want_to_train_w2v = True\n",
    "\n",
    "if want_to_train_w2v:\n",
    "    # min_count = 5 considers only words that occured atleast 5 times\n",
    "    w2v_model=Word2Vec(list_of_sentance,min_count=5,size=50, workers=4)\n",
    "    print(w2v_model.wv.most_similar('great'))\n",
    "    print('='*50)\n",
    "    print(w2v_model.wv.most_similar('worst'))\n",
    "    \n",
    "elif want_to_use_google_w2v and is_your_ram_gt_16g:\n",
    "    if os.path.isfile('GoogleNews-vectors-negative300.bin'):\n",
    "        w2v_model=KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)\n",
    "        print(w2v_model.wv.most_similar('great'))\n",
    "        print(w2v_model.wv.most_similar('worst'))\n",
    "    else:\n",
    "        print(\"you don't have gogole's word2vec file, keep want_to_train_w2v = True, to train your own w2v \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "teE01Qk2iWOj",
    "outputId": "f063436c-8b62-49d3-e093-723395e66ccd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of words that occured minimum 5 times  2791\n",
      "sample words  ['got', 'warehouse', 'deal', 'not', 'expecting', 'much', 'however', 'get', 'no', 'regular', 'donut', 'shop', 'instead', 'receive', 'half', 'box', 'decaf', 'love', 'flavors', 'though', 'ok', 'think', 'hard', 'getting', 'another', 'variety', 'pack', 'tea', 'anything', 'steal', 'issues', 'itching', 'throat', 'ear', 'everything', 'else', 'enjoy', 'smell', 'say', 'last', 'hours', 'fruit', 'gift', 'really', 'perfect', 'fresh', 'well', 'packed', 'tasty', 'looking']\n"
     ]
    }
   ],
   "source": [
    "w2v_words = list(w2v_model.wv.vocab)\n",
    "print(\"number of words that occured minimum 5 times \",len(w2v_words))\n",
    "print(\"sample words \", w2v_words[0:50])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mUNnof40iWOl"
   },
   "source": [
    "## [4.4.1] Converting text into vectors using Avg W2V, TFIDF-W2V"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "heHLB_FCiWOm"
   },
   "source": [
    "#### [4.4.1.1] Avg W2v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iCYoNRANiWOm",
    "outputId": "96ebb104-44d7-4b7d-aa45-62614bbfcf44"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 2991/2991 [00:02<00:00, 1263.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2991\n",
      "50\n"
     ]
    }
   ],
   "source": [
    "# average Word2Vec\n",
    "# compute average word2vec for each review.\n",
    "sent_vectors = []; # the avg-w2v for each sentence/review is stored in this list\n",
    "for sent in tqdm(list_of_sentance): # for each review/sentence\n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length 50, you might need to change this to 300 if you use google's w2v\n",
    "    cnt_words =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        if word in w2v_words:\n",
    "            vec = w2v_model.wv[word]\n",
    "            sent_vec += vec\n",
    "            cnt_words += 1\n",
    "    if cnt_words != 0:\n",
    "        sent_vec /= cnt_words\n",
    "    sent_vectors.append(sent_vec)\n",
    "print(len(sent_vectors))\n",
    "print(len(sent_vectors[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 997/997 [00:00<00:00, 1074.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "997 997\n",
      "50\n"
     ]
    }
   ],
   "source": [
    "# CV W2V DATA VECTORS\n",
    "\n",
    "sent_vectors_cv = []; # the avg-w2v for each sentence/review is stored in this list\n",
    "for sent in tqdm(list_of_sentance_cv): # for each review/sentence\n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length 50, you might need to change this to 300 if you use google's w2v\n",
    "    cnt_words =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        if word in w2v_words:\n",
    "            vec = w2v_model.wv[word]\n",
    "            sent_vec += vec\n",
    "            cnt_words += 1\n",
    "    if cnt_words != 0:\n",
    "        sent_vec /= cnt_words\n",
    "    sent_vectors_cv.append(sent_vec)\n",
    "print(len(sent_vectors_cv), len(y_cv_w2v))\n",
    "print(len(sent_vectors_cv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 998/998 [00:00<00:00, 1065.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "998\n",
      "50\n"
     ]
    }
   ],
   "source": [
    "# TEST W2V DATA VECTORS\n",
    "\n",
    "sent_vectors_test = []; # the avg-w2v for each sentence/review is stored in this list\n",
    "for sent in tqdm(list_of_sentance_test): # for each review/sentence\n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length 50, you might need to change this to 300 if you use google's w2v\n",
    "    cnt_words =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        if word in w2v_words:\n",
    "            vec = w2v_model.wv[word]\n",
    "            sent_vec += vec\n",
    "            cnt_words += 1\n",
    "    if cnt_words != 0:\n",
    "        sent_vec /= cnt_words\n",
    "    sent_vectors_test.append(sent_vec)\n",
    "print(len(sent_vectors_test))\n",
    "print(len(sent_vectors_test[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ylTn3LgiiWOo"
   },
   "source": [
    "#### [4.4.1.2] TFIDF weighted W2v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XBxf_qKSiWOo"
   },
   "outputs": [],
   "source": [
    "# S = [\"abc def pqr\", \"def def def abc\", \"pqr pqr def\"]\n",
    "model = TfidfVectorizer()\n",
    "tf_idf_matrix = model.fit_transform(X_train_w2v)\n",
    "# we are converting a dictionary with word as a key, and the idf as a value\n",
    "dictionary = dict(zip(model.get_feature_names(), list(model.idf_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5FHQMSyUiWOp",
    "outputId": "6b5b71be-e3f7-4933-cd74-f78fb703132d",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 2991/2991 [00:13<00:00, 218.13it/s]\n"
     ]
    }
   ],
   "source": [
    "# TF-IDF weighted Word2Vec\n",
    "tfidf_feat = model.get_feature_names() # tfidf words/col-names\n",
    "# final_tf_idf is the sparse matrix with row= sentence, col=word and cell_val = tfidf\n",
    "\n",
    "tfidf_sent_vectors_train = []; # the tfidf-w2v for each sentence/review is stored in this list\n",
    "row=0;\n",
    "for sent in tqdm(list_of_sentance): # for each review/sentence \n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length\n",
    "    weight_sum =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        if word in w2v_words and word in tfidf_feat:\n",
    "            vec = w2v_model.wv[word]\n",
    "#             tf_idf = tf_idf_matrix[row, tfidf_feat.index(word)]\n",
    "            # to reduce the computation we are \n",
    "            # dictionary[word] = idf value of word in whole courpus\n",
    "            # sent.count(word) = tf valeus of word in this review\n",
    "            tf_idf = dictionary[word]*(sent.count(word)/len(sent))\n",
    "            sent_vec += (vec * tf_idf)\n",
    "            weight_sum += tf_idf\n",
    "    if weight_sum != 0:\n",
    "        sent_vec /= weight_sum\n",
    "    tfidf_sent_vectors_train.append(sent_vec)\n",
    "    row += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 997/997 [00:05<00:00, 176.75it/s]\n"
     ]
    }
   ],
   "source": [
    "tfidf_sent_vectors_cv = []; # the tfidf-w2v for each sentence/review is stored in this list\n",
    "row=0;\n",
    "for sent in tqdm(list_of_sentance_cv): # for each review/sentence \n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length\n",
    "    weight_sum =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        if word in w2v_words and word in tfidf_feat:\n",
    "            vec = w2v_model.wv[word]\n",
    "#             tf_idf = tf_idf_matrix[row, tfidf_feat.index(word)]\n",
    "            # to reduce the computation we are \n",
    "            # dictionary[word] = idf value of word in whole courpus\n",
    "            # sent.count(word) = tf valeus of word in this review\n",
    "            tf_idf = dictionary[word]*(sent.count(word)/len(sent))\n",
    "            sent_vec += (vec * tf_idf)\n",
    "            weight_sum += tf_idf\n",
    "    if weight_sum != 0:\n",
    "        sent_vec /= weight_sum\n",
    "    tfidf_sent_vectors_cv.append(sent_vec)\n",
    "    row += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 998/998 [00:05<00:00, 180.67it/s]\n"
     ]
    }
   ],
   "source": [
    "tfidf_sent_vectors_test = []; # the tfidf-w2v for each sentence/review is stored in this list\n",
    "row=0;\n",
    "for sent in tqdm(list_of_sentance_test): # for each review/sentence \n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length\n",
    "    weight_sum =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        if word in w2v_words and word in tfidf_feat:\n",
    "            vec = w2v_model.wv[word]\n",
    "#             tf_idf = tf_idf_matrix[row, tfidf_feat.index(word)]\n",
    "            # to reduce the computation we are \n",
    "            # dictionary[word] = idf value of word in whole courpus\n",
    "            # sent.count(word) = tf valeus of word in this review\n",
    "            tf_idf = dictionary[word]*(sent.count(word)/len(sent))\n",
    "            sent_vec += (vec * tf_idf)\n",
    "            weight_sum += tf_idf\n",
    "    if weight_sum != 0:\n",
    "        sent_vec /= weight_sum\n",
    "    tfidf_sent_vectors_test.append(sent_vec)\n",
    "    row += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IwObBT2UiWOr"
   },
   "source": [
    "# [5] Assignment 8: Decision Trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "k_mwmumRiWOr"
   },
   "source": [
    "<ol>\n",
    "    <li><strong>Apply Decision Trees on these feature sets</strong>\n",
    "        <ul>\n",
    "            <li><font color='red'>SET 1:</font>Review text, preprocessed one converted into vectors using (BOW)</li>\n",
    "            <li><font color='red'>SET 2:</font>Review text, preprocessed one converted into vectors using (TFIDF)</li>\n",
    "            <li><font color='red'>SET 3:</font>Review text, preprocessed one converted into vectors using (AVG W2v)</li>\n",
    "            <li><font color='red'>SET 4:</font>Review text, preprocessed one converted into vectors using (TFIDF W2v)</li>\n",
    "        </ul>\n",
    "    </li>\n",
    "    <br>\n",
    "    <li><strong>The hyper paramter tuning (best `depth` in range [1, 5, 10, 50, 100, 500, 100], and the best `min_samples_split` in range [5, 10, 100, 500])</strong>\n",
    "        <ul>\n",
    "    <li>Find the best hyper parameter which will give the maximum <a href='https://www.appliedaicourse.com/course/applied-ai-course-online/lessons/receiver-operating-characteristic-curve-roc-curve-and-auc-1/'>AUC</a> value</li>\n",
    "    <li>Find the best hyper paramter using k-fold cross validation or simple cross validation data</li>\n",
    "    <li>Use gridsearch cv or randomsearch cv or you can also write your own for loops to do this task of hyperparameter tuning</li>          \n",
    "        </ul>\n",
    "    </li>\n",
    "    <br>\n",
    "    <li><strong>Graphviz</strong>\n",
    "        <ul>\n",
    "    <li>Visualize your decision tree with Graphviz. It helps you to understand how a decision is being made, given a new vector.</li>\n",
    "    <li>Since feature names are not obtained from word2vec related models, visualize only BOW & TFIDF decision trees using Graphviz</li>\n",
    "    <li>Make sure to print the words in each node of the decision tree instead of printing its index.</li>\n",
    "    <li>Just for visualization purpose, limit max_depth to 2 or 3 and either embed the generated images of graphviz in your notebook, or directly upload them as .png files.</li>                \n",
    "        </ul>\n",
    "    </li>\n",
    "    <br>\n",
    "    <li><strong>Feature importance</strong>\n",
    "        <ul>\n",
    "    <li>Find the top 20 important features from both feature sets <font color='red'>Set 1</font> and <font color='red'>Set 2</font> using `feature_importances_` method of <a href='https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html'>Decision Tree Classifier</a> and print their corresponding feature names</li>\n",
    "        </ul>\n",
    "    </li>\n",
    "    <br>\n",
    "    <li><strong>Feature engineering</strong>\n",
    "        <ul>\n",
    "    <li>To increase the performance of your model, you can also experiment with with feature engineering like :</li>\n",
    "            <ul>\n",
    "            <li>Taking length of reviews as another feature.</li>\n",
    "            <li>Considering some features from review summary as well.</li>\n",
    "        </ul>\n",
    "        </ul>\n",
    "    </li>\n",
    "    <br>\n",
    "    <li><strong>Representation of results</strong>\n",
    "        <ul>\n",
    "    <li>You need to plot the performance of model both on train data and cross validation data for each hyper parameter, like shown in the figure.\n",
    "    <img src='train_cv_auc.JPG' width=300px></li>\n",
    "    <li>Once after you found the best hyper parameter, you need to train your model with it, and find the AUC on test data and plot the ROC curve on both train and test.\n",
    "    <img src='train_test_auc.JPG' width=300px></li>\n",
    "    <li>Along with plotting ROC curve, you need to print the <a href='https://www.appliedaicourse.com/course/applied-ai-course-online/lessons/confusion-matrix-tpr-fpr-fnr-tnr-1/'>confusion matrix</a> with predicted and original labels of test data points. Please visualize your confusion matrices using <a href='https://seaborn.pydata.org/generated/seaborn.heatmap.html'>seaborn heatmaps.\n",
    "    <img src='confusion_matrix.png' width=300px></li>\n",
    "        </ul>\n",
    "    </li>\n",
    "    <br>\n",
    "    <li><strong>Conclusion</strong>\n",
    "        <ul>\n",
    "    <li>You need to summarize the results at the end of the notebook, summarize it in the table format. To print out a table please refer to this prettytable library<a href='https://zetcode.com/python/prettytable/'>  link</a> \n",
    "        <img src='summary.JPG' width=400px>\n",
    "    </li>\n",
    "        </ul>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uqhvv_IiiWOs"
   },
   "source": [
    "<h4><font color='red'>Note: Data Leakage</font></h4>\n",
    "\n",
    "1. There will be an issue of data-leakage if you vectorize the entire data and then split it into train/cv/test.\n",
    "2. To avoid the issue of data-leakag, make sure to split your data first and then vectorize it. \n",
    "3. While vectorizing your data, apply the method fit_transform() on you train data, and apply the method transform() on cv/test data.\n",
    "4. For more details please go through this <a href='https://soundcloud.com/applied-ai-course/leakage-bow-and-tfidf'>link.</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ayLYlocviWOs"
   },
   "source": [
    "# Applying Decision Trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fvKQwJMWiWOt"
   },
   "source": [
    "## [5.1] Applying Decision Trees on BOW,<font color='red'> SET 1</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vnSi4W-iiWOt"
   },
   "outputs": [
    {
     "ename": "ExecutableNotFound",
     "evalue": "failed to execute ['dot', '-Tpdf', '-O', 'iris'], make sure the Graphviz executables are on your systems' PATH",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\backend.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(cmd, input, capture_output, check, quiet, **kwargs)\u001b[0m\n\u001b[0;32m    146\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 147\u001b[1;33m         \u001b[0mproc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstartupinfo\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mget_startupinfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    148\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)\u001b[0m\n\u001b[0;32m    768\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 769\u001b[1;33m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[0;32m    770\u001b[0m         \u001b[1;32mexcept\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[1;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)\u001b[0m\n\u001b[0;32m   1171\u001b[0m                                          \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfspath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcwd\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcwd\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1172\u001b[1;33m                                          startupinfo)\n\u001b[0m\u001b[0;32m   1173\u001b[0m             \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 2] The system cannot find the file specified",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mExecutableNotFound\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-64-3b71c31f0397>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mdot_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_graphviz\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_file\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgraphviz\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSource\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdot_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"iris\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\files.py\u001b[0m in \u001b[0;36mrender\u001b[1;34m(self, filename, directory, view, cleanup, format, renderer, formatter)\u001b[0m\n\u001b[0;32m    186\u001b[0m             \u001b[0mformat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_format\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    187\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 188\u001b[1;33m         \u001b[0mrendered\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilepath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformatter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    189\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    190\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcleanup\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\backend.py\u001b[0m in \u001b[0;36mrender\u001b[1;34m(engine, format, filepath, renderer, formatter, quiet)\u001b[0m\n\u001b[0;32m    181\u001b[0m     \"\"\"\n\u001b[0;32m    182\u001b[0m     \u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrendered\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcommand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilepath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformatter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 183\u001b[1;33m     \u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcapture_output\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcheck\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquiet\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mquiet\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    184\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mrendered\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\graphviz\\backend.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(cmd, input, capture_output, check, quiet, **kwargs)\u001b[0m\n\u001b[0;32m    148\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    149\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mENOENT\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 150\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mExecutableNotFound\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    151\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pragma: no cover\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    152\u001b[0m             \u001b[1;32mraise\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mExecutableNotFound\u001b[0m: failed to execute ['dot', '-Tpdf', '-O', 'iris'], make sure the Graphviz executables are on your systems' PATH"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "iris = load_iris()\n",
    "clf = DecisionTreeClassifier()\n",
    "clf = clf.fit(iris.data, iris.target)\n",
    "import graphviz \n",
    "dot_data = tree.export_graphviz(clf, out_file=None) \n",
    "graph = graphviz.Source(dot_data) \n",
    "graph.render(\"iris\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jm4vkPwMiWOu"
   },
   "source": [
    "### [5.1.1] Top 20 important features from<font color='red'> SET 1</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "puCcjOAkiWOv"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7JyitDTTiWOx"
   },
   "source": [
    "### [5.1.2] Graphviz visualization of Decision Tree on BOW,<font color='red'> SET 1</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-jxU7zrAiWOy"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5przjMTbiWOz"
   },
   "source": [
    "## [5.2] Applying Decision Trees on TFIDF,<font color='red'> SET 2</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bm3mfI7miWOz"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PIKUCldjiWO0"
   },
   "source": [
    "### [5.2.1] Top 20 important features from<font color='red'> SET 2</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p2P0iHQciWO1"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RTZB9ipUiWO3"
   },
   "source": [
    "### [5.2.2] Graphviz visualization of Decision Tree on TFIDF,<font color='red'> SET 2</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DMbZoPoTiWO4"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NVhl_V6MiWO5"
   },
   "source": [
    "## [5.3] Applying Decision Trees on AVG W2V,<font color='red'> SET 3</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uYDXnzMmiWO5"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PnwVzXytiWO7"
   },
   "source": [
    "## [5.4] Applying Decision Trees on TFIDF W2V,<font color='red'> SET 4</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YqODkVCgiWO7"
   },
   "outputs": [],
   "source": [
    "# Please write all the code with proper documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Em-rF6u2iWO8"
   },
   "source": [
    "# [6] Conclusions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3Hnu2QziiWO9"
   },
   "outputs": [],
   "source": [
    "# Please compare all your models using Prettytable library"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "NTwQITJKiWN5",
    "T_LFck8iiWOX",
    "mjfrB76yiWOa",
    "0tcbbZxSiWOd",
    "u566lt93iWOf",
    "mUNnof40iWOl",
    "heHLB_FCiWOm",
    "ylTn3LgiiWOo",
    "NVhl_V6MiWO5",
    "PnwVzXytiWO7"
   ],
   "name": "08 Amazon Fine Food Reviews Analysis_Decision Trees.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}