ml/Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import bcolz\n",
    "import pickle\n",
    "import nltk\n",
    "import re\n",
    "import xgboost\n",
    "import m2cgen as m2c\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "STOP_WORDS = pd.read_csv(\"stopwords.csv\", header=0)\n",
    "STOP_WORDS = list(STOP_WORDS['STOP_WORD'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the [Sentiment140](http:https://help.sentiment140.com/for-students/) labelled dataset to train our classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"trainingandtestdata/training.1600000.processed.noemoticon.csv\", encoding=\"latin-1\", header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get DataFrame with only 2 columns: Text and Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http:https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/usr/local/lib/python3.7/site-packages/pandas/core/frame.py:4223: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http:https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return super().rename(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@switchfoot http:https://twitpic.com/2y1zl - Awww, t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1599995</td>\n",
       "      <td>Just woke up. Having no school is the best fee...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1599996</td>\n",
       "      <td>TheWDB.com - Very cool to hear old Walt interv...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1599997</td>\n",
       "      <td>Are you ready for your MoJo Makeover? Ask me f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1599998</td>\n",
       "      <td>Happy 38th Birthday to my boo of alll time!!! ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1599999</td>\n",
       "      <td>happy #charitytuesday @theNSPCC @SparksCharity...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1600000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Text Sentiment\n",
       "0        @switchfoot http:https://twitpic.com/2y1zl - Awww, t...         0\n",
       "1        is upset that he can't update his Facebook by ...         0\n",
       "2        @Kenichan I dived many times for the ball. Man...         0\n",
       "3          my whole body feels itchy and like its on fire          0\n",
       "4        @nationwideclass no, it's not behaving at all....         0\n",
       "...                                                    ...       ...\n",
       "1599995  Just woke up. Having no school is the best fee...         1\n",
       "1599996  TheWDB.com - Very cool to hear old Walt interv...         1\n",
       "1599997  Are you ready for your MoJo Makeover? Ask me f...         1\n",
       "1599998  Happy 38th Birthday to my boo of alll time!!! ...         1\n",
       "1599999  happy #charitytuesday @theNSPCC @SparksCharity...         1\n",
       "\n",
       "[1600000 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_texts_pd = df[[5]]\n",
    "tweet_texts_pd['Sentiment'] = np.where(df[[0]] == 4, \"1\", \"0\")\n",
    "tweet_texts_pd.rename(columns = {5: \"Text\"}, inplace=True)\n",
    "tweet_texts_pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For simplicity take n positive and n negative tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Text</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>800000</td>\n",
       "      <td>I LOVE @Health4UandPets u guys r the best!!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>800001</td>\n",
       "      <td>im meeting up with one of my besties tonight! ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>800002</td>\n",
       "      <td>@DaRealSunisaKim Thanks for the Twitter add, S...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>800003</td>\n",
       "      <td>Being sick can be really cheap when it hurts t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>800004</td>\n",
       "      <td>@LovesBrooklyn2 he has that effect on everyone</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9995</td>\n",
       "      <td>4995</td>\n",
       "      <td>long day today</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9996</td>\n",
       "      <td>4996</td>\n",
       "      <td>a friend broke his promises..</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9997</td>\n",
       "      <td>4997</td>\n",
       "      <td>@gjarnling I am fine thanks - tired</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9998</td>\n",
       "      <td>4998</td>\n",
       "      <td>trying to keep my eyes open..damn baking</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9999</td>\n",
       "      <td>4999</td>\n",
       "      <td>why the hell is it snowing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       index                                               Text Sentiment\n",
       "0     800000       I LOVE @Health4UandPets u guys r the best!!          1\n",
       "1     800001  im meeting up with one of my besties tonight! ...         1\n",
       "2     800002  @DaRealSunisaKim Thanks for the Twitter add, S...         1\n",
       "3     800003  Being sick can be really cheap when it hurts t...         1\n",
       "4     800004    @LovesBrooklyn2 he has that effect on everyone          1\n",
       "...      ...                                                ...       ...\n",
       "9995    4995                                    long day today          0\n",
       "9996    4996                     a friend broke his promises..          0\n",
       "9997    4997               @gjarnling I am fine thanks - tired          0\n",
       "9998    4998          trying to keep my eyes open..damn baking          0\n",
       "9999    4999                        why the hell is it snowing          0\n",
       "\n",
       "[10000 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = 5000\n",
    "tweet_texts_pd = pd.concat([tweet_texts_pd[tweet_texts_pd['Sentiment'] == \"1\"][:n], tweet_texts_pd[tweet_texts_pd['Sentiment'] == \"0\"][:n]])\n",
    "tweet_texts_pd.reset_index(inplace=True)\n",
    "tweet_texts_pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unpack and preprcoess glove word vectors (no need to run this)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_path = \"glove\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = []\n",
    "idx = 0\n",
    "word2idx = {}\n",
    "vectors = bcolz.carray(np.zeros(1), rootdir=f'{glove_path}/6B.50.dat', mode='w')\n",
    "\n",
    "with open(f'{glove_path}/glove.6B.50d.txt', 'rb') as f:\n",
    "    for l in f:\n",
    "        line = l.decode().split()\n",
    "        word = line[0]\n",
    "        words.append(word)\n",
    "        word2idx[word] = idx\n",
    "        idx += 1\n",
    "        vect = np.array(line[1:]).astype(np.float)\n",
    "        vectors.append(vect)\n",
    "    \n",
    "vectors = bcolz.carray(vectors[1:].reshape((400000, 50)), rootdir=f'{glove_path}/6B.50.dat', mode='w')\n",
    "vectors.flush()\n",
    "pickle.dump(words, open(f'{glove_path}/6B.50_words.pkl', 'wb'))\n",
    "pickle.dump(word2idx, open(f'{glove_path}/6B.50_idx.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use [glove](https://nlp.stanford.edu/projects/glove/) as a pretrained word vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = bcolz.open(f'{glove_path}/6B.50.dat')[:]\n",
    "words = pickle.load(open(f'{glove_path}/6B.50_words.pkl', 'rb'))\n",
    "word2idx = pickle.load(open(f'{glove_path}/6B.50_idx.pkl', 'rb'))\n",
    "\n",
    "glove = {w: vectors[word2idx[w]] for w in words}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define a tokenizer to transform tweet texts into sequences of words characterizing them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(s):\n",
    "    series = pd.Series(re.split(\"\\W\", s)).str.lower()\n",
    "    return series[(series != \"\") & (series.str.len() > 2) & ~(series.isin(STOP_WORDS))].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenize tweet texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_pd = tweet_texts_pd[\"Text\"].apply(tokenize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform tweets into vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 50)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_vectorized = []\n",
    "for l in words_pd:\n",
    "    arr = []\n",
    "    for w in l:\n",
    "        if w in glove:\n",
    "            arr.append(glove[w])\n",
    "    if not arr:\n",
    "        res = np.zeros((50))\n",
    "    else:\n",
    "        res = np.mean(np.asarray(arr), axis=0)\n",
    "    words_vectorized.append(res)\n",
    "words_vectorized = np.asarray(words_vectorized)\n",
    "words_vectorized.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train xgboost classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=5,\n",
       "              min_child_weight=1, missing=None, n_estimators=5, n_jobs=1,\n",
       "              nthread=None, num_leaves=5, objective='binary:logistic',\n",
       "              random_state=7, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "              seed=None, silent=None, subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = xgboost.XGBClassifier(n_estimators=5, random_state=7, max_depth=5, num_leaves=5)\n",
    "clf.fit(words_vectorized, tweet_texts_pd[\"Sentiment\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export classifier using [m2cgen](https://github.com/BayesWitnesses/m2cgen) library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = m2c.export_to_java(clf, package_name=\"realtime_events.ml\", class_name=\"SentimentPredictor\")\n",
    "with open('Model.java', 'w') as f:\n",
    "    f.write(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export word2vec into json to use in flink pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def jsonify_values(d):\n",
    "    res = {}\n",
    "    for k, v in d.items():\n",
    "        res[k] = v.tolist()\n",
    "    return res\n",
    "json.dump(jsonify_values(glove), open(\"word2vec.json\", \"w\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}