preprocessing_of_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "preprocessing_of_data.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "t0tsZrJjepLA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "28912c74-791b-4b7c-f3bb-99ab325d0a1b"
      },
      "source": [
        "import pandas as pd\n",
        "import string\n",
        "import os\n",
        "import numpy as np\n",
        "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
        "from PIL import Image\n",
        "import urllib\n",
        "import requests\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import train_test_split,cross_val_score\n",
        "from sklearn.metrics import roc_curve, auc, classification_report\n",
        "from sklearn.preprocessing import label_binarize\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.model_selection import cross_val_score\n",
        "import time\n",
        "import glob\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import operator\n",
        "import folium\n",
        "from itertools import cycle, islice\n",
        "from pandas import options\n",
        "import warnings\n",
        "import pickle\n",
        "import re\n",
        "import nltk\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch\n",
        "from matplotlib.pyplot import figure\n",
        "from nltk.corpus import stopwords \n",
        "import nltk\n",
        "\n",
        "\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XAZWn7e3PNJg"
      },
      "source": [
        "###Here is a Notebook that do the preprocess of our data just like in the first project. I save the splitted data in my drive so as to load them from the other Notebooks to save time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uc_QqeaakAqw",
        "outputId": "edabecf9-9398-49b2-8ece-40a290fa3777"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kmH3AzpufR1q"
      },
      "source": [
        "## *Step 1:* Load and Read Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8qwNPUpwfLrD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "7a0b645b-b881-4d86-8a71-925d0b3ae96e"
      },
      "source": [
        "path = \"/content/drive/My Drive/\"\n",
        "\n",
        "#Read data from csv file \n",
        "# df = pd.read_csv(path + 'SentimentTweets.csv') \n",
        "df = pd.read_csv(path + 'SentimentTweets.csv',nrows=500000) \n",
        "df = df.dropna()\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>target</th>\n",
              "      <th>id</th>\n",
              "      <th>date</th>\n",
              "      <th>flag</th>\n",
              "      <th>user</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>680949</td>\n",
              "      <td>0</td>\n",
              "      <td>2249621587</td>\n",
              "      <td>Fri Jun 19 22:41:08 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>sukumarpant</td>\n",
              "      <td>#brokenpromises...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>406741</td>\n",
              "      <td>0</td>\n",
              "      <td>2059003515</td>\n",
              "      <td>Sat Jun 06 16:03:21 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>MTMSparrow</td>\n",
              "      <td>David Carradine  so sad. Thai's law not sure i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1337108</td>\n",
              "      <td>4</td>\n",
              "      <td>2017466467</td>\n",
              "      <td>Wed Jun 03 08:26:14 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>itsmemcee</td>\n",
              "      <td>A @ 415 B @ 425. Tell your bro i say congrats!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1560887</td>\n",
              "      <td>4</td>\n",
              "      <td>2186457254</td>\n",
              "      <td>Mon Jun 15 18:52:04 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>jdfreivald</td>\n",
              "      <td>@littlefluffycat  Indeed.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1466295</td>\n",
              "      <td>4</td>\n",
              "      <td>2064458395</td>\n",
              "      <td>Sun Jun 07 06:19:20 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>CrazyHan</td>\n",
              "      <td>Completed Race 4 Life in 58mins with girlies f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499995</th>\n",
              "      <td>644302</td>\n",
              "      <td>0</td>\n",
              "      <td>2235992375</td>\n",
              "      <td>Fri Jun 19 02:35:10 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>mojitoCyprus</td>\n",
              "      <td>@SkyNews they've finally managed to kill F1!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499996</th>\n",
              "      <td>528609</td>\n",
              "      <td>0</td>\n",
              "      <td>2195149293</td>\n",
              "      <td>Tue Jun 16 10:44:42 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>rorossetto</td>\n",
              "      <td>@elvensapphire I'm brazilian so brazilian are ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499997</th>\n",
              "      <td>1159941</td>\n",
              "      <td>4</td>\n",
              "      <td>1979388849</td>\n",
              "      <td>Sun May 31 02:50:09 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>Princes_Doll</td>\n",
              "      <td>@ScherziNicole Aww thats so sweet! I joined a ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499998</th>\n",
              "      <td>495969</td>\n",
              "      <td>0</td>\n",
              "      <td>2185406246</td>\n",
              "      <td>Mon Jun 15 17:20:03 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>teagi_weegi</td>\n",
              "      <td>Woke up kinda depressed and sad</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499999</th>\n",
              "      <td>787714</td>\n",
              "      <td>0</td>\n",
              "      <td>2324965812</td>\n",
              "      <td>Thu Jun 25 04:27:53 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>GemizzleBizzle</td>\n",
              "      <td>Wish I had something to play my dad's old Clas...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500000 rows × 7 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        Unnamed: 0  ...                                               text\n",
              "0           680949  ...                                #brokenpromises... \n",
              "1           406741  ...  David Carradine  so sad. Thai's law not sure i...\n",
              "2          1337108  ...    A @ 415 B @ 425. Tell your bro i say congrats! \n",
              "3          1560887  ...                          @littlefluffycat  Indeed.\n",
              "4          1466295  ...  Completed Race 4 Life in 58mins with girlies f...\n",
              "...            ...  ...                                                ...\n",
              "499995      644302  ...      @SkyNews they've finally managed to kill F1! \n",
              "499996      528609  ...  @elvensapphire I'm brazilian so brazilian are ...\n",
              "499997     1159941  ...  @ScherziNicole Aww thats so sweet! I joined a ...\n",
              "499998      495969  ...                   Woke up kinda depressed and sad \n",
              "499999      787714  ...  Wish I had something to play my dad's old Clas...\n",
              "\n",
              "[500000 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "79FuCriQfZ-g"
      },
      "source": [
        "#### Take the columns that we want for Sentiment Analysis (text and target columns)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "umD-LURxfa1u",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "3137a090-011f-48e0-b811-cc6f7183f71a"
      },
      "source": [
        "df.drop(df.columns[0], axis=1, inplace=True)\n",
        "df.drop(df.columns[1], axis=1, inplace=True)\n",
        "df.drop(df.columns[1], axis=1, inplace=True)\n",
        "df.drop(df.columns[1], axis=1, inplace=True)\n",
        "df.drop(df.columns[1], axis=1, inplace=True)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>#brokenpromises...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>David Carradine  so sad. Thai's law not sure i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>A @ 415 B @ 425. Tell your bro i say congrats!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>@littlefluffycat  Indeed.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Completed Race 4 Life in 58mins with girlies f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499995</th>\n",
              "      <td>0</td>\n",
              "      <td>@SkyNews they've finally managed to kill F1!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499996</th>\n",
              "      <td>0</td>\n",
              "      <td>@elvensapphire I'm brazilian so brazilian are ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499997</th>\n",
              "      <td>4</td>\n",
              "      <td>@ScherziNicole Aww thats so sweet! I joined a ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499998</th>\n",
              "      <td>0</td>\n",
              "      <td>Woke up kinda depressed and sad</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499999</th>\n",
              "      <td>0</td>\n",
              "      <td>Wish I had something to play my dad's old Clas...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        target                                               text\n",
              "0            0                                #brokenpromises... \n",
              "1            0  David Carradine  so sad. Thai's law not sure i...\n",
              "2            4    A @ 415 B @ 425. Tell your bro i say congrats! \n",
              "3            4                          @littlefluffycat  Indeed.\n",
              "4            4  Completed Race 4 Life in 58mins with girlies f...\n",
              "...        ...                                                ...\n",
              "499995       0      @SkyNews they've finally managed to kill F1! \n",
              "499996       0  @elvensapphire I'm brazilian so brazilian are ...\n",
              "499997       4  @ScherziNicole Aww thats so sweet! I joined a ...\n",
              "499998       0                   Woke up kinda depressed and sad \n",
              "499999       0  Wish I had something to play my dad's old Clas...\n",
              "\n",
              "[500000 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YAfL5UPFfwUe"
      },
      "source": [
        "## *Step 2:* Data Cleaning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08A8vhKQf0Q6"
      },
      "source": [
        "### a) Lowercase of text set and get rid of the punctiations and symbols.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xeuOWzEofdIM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "3e011ec5-7850-415d-f3bb-3208f93b25ba"
      },
      "source": [
        "df['text'] = df['text'].str.lower()\n",
        "\n",
        "#Define function for removing special characters\n",
        "def remove_special_characters(text, remove_digits=True):\n",
        "    text=re.sub(r'[^a-zA-z0-9\\s]','',text)\n",
        "    return text\n",
        "\n",
        "df['text'] = df['text'].apply(lambda x:''.join([i for i in x if i not in string.punctuation]))\n",
        "\n",
        "#Apply function on review column\n",
        "df['text']=df['text'].apply(remove_special_characters)\n",
        "\n",
        "\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>brokenpromises</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>david carradine  so sad thais law not sure if ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>a  415 b  425 tell your bro i say congrats</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>littlefluffycat  indeed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>completed race 4 life in 58mins with girlies f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499995</th>\n",
              "      <td>0</td>\n",
              "      <td>skynews theyve finally managed to kill f1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499996</th>\n",
              "      <td>0</td>\n",
              "      <td>elvensapphire im brazilian so brazilian are an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499997</th>\n",
              "      <td>4</td>\n",
              "      <td>scherzinicole aww thats so sweet i joined a wh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499998</th>\n",
              "      <td>0</td>\n",
              "      <td>woke up kinda depressed and sad</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499999</th>\n",
              "      <td>0</td>\n",
              "      <td>wish i had something to play my dads old clash...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        target                                               text\n",
              "0            0                                    brokenpromises \n",
              "1            0  david carradine  so sad thais law not sure if ...\n",
              "2            4        a  415 b  425 tell your bro i say congrats \n",
              "3            4                            littlefluffycat  indeed\n",
              "4            4  completed race 4 life in 58mins with girlies f...\n",
              "...        ...                                                ...\n",
              "499995       0         skynews theyve finally managed to kill f1 \n",
              "499996       0  elvensapphire im brazilian so brazilian are an...\n",
              "499997       4  scherzinicole aww thats so sweet i joined a wh...\n",
              "499998       0                   woke up kinda depressed and sad \n",
              "499999       0  wish i had something to play my dads old clash...\n",
              "\n",
              "[500000 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zoaMla1Hf5hf"
      },
      "source": [
        "Observe how many positive and negative comments we have"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ssgV9SeWf2V7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc04cddd-b381-46a0-c885-545e62a47a93"
      },
      "source": [
        "df['target'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4    250800\n",
              "0    249200\n",
              "Name: target, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dzALpCIMIv1J"
      },
      "source": [
        "Replace the negative label of 4 to 1\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HdQEDJ-oHOjN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "outputId": "ea54323e-4a97-4a05-a548-5e5c41bb62de"
      },
      "source": [
        "df['target'] = df['target'].replace(4,1)\n",
        "display(df['target'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "0         0\n",
              "1         0\n",
              "2         1\n",
              "3         1\n",
              "4         1\n",
              "         ..\n",
              "499995    0\n",
              "499996    0\n",
              "499997    1\n",
              "499998    0\n",
              "499999    0\n",
              "Name: target, Length: 500000, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9knsUxCpgAY_"
      },
      "source": [
        "### 2) Remove tags and links "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ukyRSgQSf7v1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "f9db4ea3-65a1-481d-b592-54dd745e5341"
      },
      "source": [
        "import re\n",
        "#Remove tags and links \n",
        "tag = re.compile(r'<[^>]+>')\n",
        "\n",
        "df['text'] = df['text'].apply(lambda x: tag.sub('', x)) #removing html labels\n",
        "\n",
        "df['text'] = df['text'].replace(r'http\\S+', '', regex=True).replace(r'www.\\S+', '', regex=True).replace(r'http\\S+', '', regex=True).replace(r'\"', '', regex=True)\n",
        "\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>brokenpromises</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>david carradine  so sad thais law not sure if ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>a  415 b  425 tell your bro i say congrats</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>littlefluffycat  indeed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>completed race 4 life in 58mins with girlies f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499995</th>\n",
              "      <td>0</td>\n",
              "      <td>skynews theyve finally managed to kill f1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499996</th>\n",
              "      <td>0</td>\n",
              "      <td>elvensapphire im brazilian so brazilian are an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499997</th>\n",
              "      <td>1</td>\n",
              "      <td>scherzinicole aww thats so sweet i joined a wh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499998</th>\n",
              "      <td>0</td>\n",
              "      <td>woke up kinda depressed and sad</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499999</th>\n",
              "      <td>0</td>\n",
              "      <td>wish i had something to play my dads old clash...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        target                                               text\n",
              "0            0                                    brokenpromises \n",
              "1            0  david carradine  so sad thais law not sure if ...\n",
              "2            1        a  415 b  425 tell your bro i say congrats \n",
              "3            1                            littlefluffycat  indeed\n",
              "4            1  completed race 4 life in 58mins with girlies f...\n",
              "...        ...                                                ...\n",
              "499995       0         skynews theyve finally managed to kill f1 \n",
              "499996       0  elvensapphire im brazilian so brazilian are an...\n",
              "499997       1  scherzinicole aww thats so sweet i joined a wh...\n",
              "499998       0                   woke up kinda depressed and sad \n",
              "499999       0  wish i had something to play my dads old clash...\n",
              "\n",
              "[500000 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lGJizkwygFQi"
      },
      "source": [
        "### 3) Tokenization and Lemmanization\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xxGQ2ksmgCoD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "f2edfec8-b228-441a-8e13-673926d4f1e7"
      },
      "source": [
        "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
        "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
        "\n",
        "def lemmatize_text(text):\n",
        "    return \" \".join([lemmatizer.lemmatize(w, pos=\"v\") for w in w_tokenizer.tokenize(text)])\n",
        "\n",
        "df['text'] = df.text.apply(lemmatize_text).copy()\n",
        "\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>brokenpromises</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>david carradine so sad thais law not sure if i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>a 415 b 425 tell your bro i say congrats</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>littlefluffycat indeed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>complete race 4 life in 58mins with girlies fr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499995</th>\n",
              "      <td>0</td>\n",
              "      <td>skynews theyve finally manage to kill f1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499996</th>\n",
              "      <td>0</td>\n",
              "      <td>elvensapphire im brazilian so brazilian be any...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499997</th>\n",
              "      <td>1</td>\n",
              "      <td>scherzinicole aww thats so sweet i join a whil...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499998</th>\n",
              "      <td>0</td>\n",
              "      <td>wake up kinda depress and sad</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499999</th>\n",
              "      <td>0</td>\n",
              "      <td>wish i have something to play my dads old clas...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        target                                               text\n",
              "0            0                                     brokenpromises\n",
              "1            0  david carradine so sad thais law not sure if i...\n",
              "2            1           a 415 b 425 tell your bro i say congrats\n",
              "3            1                             littlefluffycat indeed\n",
              "4            1  complete race 4 life in 58mins with girlies fr...\n",
              "...        ...                                                ...\n",
              "499995       0           skynews theyve finally manage to kill f1\n",
              "499996       0  elvensapphire im brazilian so brazilian be any...\n",
              "499997       1  scherzinicole aww thats so sweet i join a whil...\n",
              "499998       0                      wake up kinda depress and sad\n",
              "499999       0  wish i have something to play my dads old clas...\n",
              "\n",
              "[500000 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q-2r_VXsgOI1"
      },
      "source": [
        "## *Step 3*: Split dataset into train, test and validation set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NXVYcupAd2Ba"
      },
      "source": [
        "I choose to split the dataset to 99%-1% because is too big"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0-Xlw49gJr8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d9fbbc1-bbc6-4570-a2c1-cf27b0958643"
      },
      "source": [
        "x = pd.DataFrame(df, columns = ['text']) \n",
        "y = pd.DataFrame(df, columns = ['target']) \n",
        "\n",
        "# Split dataset to train and test set.\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(x, y, test_size=0.005, random_state=42)\n",
        "\n",
        "# Split train dataset to train and validation set.\n",
        "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.005, random_state=42)\n",
        "\n",
        "print(\"Shape of x_train: \", X_train.shape)\n",
        "print(\"Shape of y_train: \", Y_train.shape)\n",
        "print(\"Shape of x_test:  \", X_test.shape)\n",
        "print(\"Shape of y_test:  \", Y_test.shape)\n",
        "print(\"Shape of x_val:  \", X_val.shape)\n",
        "print(\"Shape of y_val:  \", Y_val.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of x_train:  (495012, 1)\n",
            "Shape of y_train:  (495012, 1)\n",
            "Shape of x_test:   (2500, 1)\n",
            "Shape of y_test:   (2500, 1)\n",
            "Shape of x_val:   (2488, 1)\n",
            "Shape of y_val:   (2488, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a2hHFo9DwAIX"
      },
      "source": [
        "## *Step 4*: Save the data in order to use them in other Notebooks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWvVhlLglbt5"
      },
      "source": [
        "x_train_file = open('/content/drive/My Drive/X_train.pkl', 'wb')\n",
        "pickle.dump(X_train,x_train_file)\n",
        "x_train_file.close()\n",
        "\n",
        "x_test_file = open('/content/drive/My Drive/X_test.pkl', 'wb')\n",
        "pickle.dump(X_test,x_test_file)\n",
        "x_test_file.close()\n",
        "\n",
        "y_train_file = open('/content/drive/My Drive/Y_train.pkl', 'wb')\n",
        "pickle.dump(Y_train,y_train_file)\n",
        "y_train_file.close()\n",
        "\n",
        "y_test_file = open('/content/drive/My Drive/Y_test.pkl', 'wb')\n",
        "pickle.dump(Y_test,y_test_file)\n",
        "y_test_file.close()\n",
        "\n",
        "x_val_file = open('/content/drive/My Drive/X_val.pkl', 'wb')\n",
        "pickle.dump(X_val,x_val_file)\n",
        "x_val_file.close()\n",
        "\n",
        "y_val_file = open('/content/drive/My Drive/Y_val.pkl', 'wb')\n",
        "pickle.dump(Y_val,y_val_file)\n",
        "y_val_file.close()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}