04_nlp_stem_lemmatize_text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP: Stemming & Lemmatizing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import nltk\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stem_lem_text(s, type='Lancaster'):\n",
    "    words = s.split()\n",
    "    \n",
    "    if type == 'Porter':\n",
    "        choice = PorterStemmer()\n",
    "        reformed = [choice.stem(word) for word in words]\n",
    "    elif type == 'Snowball':\n",
    "        choice = SnowballStemmer('english')\n",
    "        reformed = [choice.stem(word) for word in words]\n",
    "    elif type == 'Lemmatize':\n",
    "        choice = WordNetLemmatizer()\n",
    "        reformed = [choice.lemmatize(word) for word in words]\n",
    "    else:\n",
    "        choice = LancasterStemmer()\n",
    "        reformed = [choice.stem(word) for word in words]\n",
    "        \n",
    "    reformed = \" \".join(reformed)\n",
    "    return reformed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('news_articles_cleaned_trimmed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 84155 entries, 0 to 84154\n",
      "Data columns (total 6 columns):\n",
      "article_text            84155 non-null object\n",
      "datetime                84155 non-null object\n",
      "source                  84155 non-null object\n",
      "title                   84155 non-null object\n",
      "url                     84155 non-null object\n",
      "cleaned_article_text    84155 non-null object\n",
      "dtypes: object(6)\n",
      "memory usage: 3.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_text</th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>cleaned_article_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31653</th>\n",
       "      <td>FRANKFURT, July 5 (Reuters) - European stocks...</td>\n",
       "      <td>2007-07-05 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>European shares end down on Vodafone, rate wor...</td>\n",
       "      <td>http:https://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European stocks led telecommunications heavywe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57149</th>\n",
       "      <td>(Changes entry on Porsche to be filed under Vo...</td>\n",
       "      <td>2016-12-23</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>REFILE-German stocks - Factors to watch on Dec...</td>\n",
       "      <td>http:https://uk.reuters.com/article/germany-stocks-f...</td>\n",
       "      <td>The DAX looked set according premarket brokera...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79304</th>\n",
       "      <td>As dawn breaks in Asia traders are readying th...</td>\n",
       "      <td>2017-02-14 23:00:00</td>\n",
       "      <td>FT</td>\n",
       "      <td>Fast Asia Open: Between two Yellens</td>\n",
       "      <td>https://www.ft.com/content/78392d38-8e4b-32d4-...</td>\n",
       "      <td>As dawn breaks Asia traders readying purgatory...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5334</th>\n",
       "      <td>\\nMy father was born in 1919 and died in 2002....</td>\n",
       "      <td>2015-03-22 00:05:05</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Enough of the dry politics of numbers. We need...</td>\n",
       "      <td>https://www.theguardian.com/commentisfree/2015...</td>\n",
       "      <td>My father born died For first years life stock...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42951</th>\n",
       "      <td>Sept 4 (Reuters) - Hong Kong stock erased earl...</td>\n",
       "      <td>2015-09-04</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Hong Kong stocks fall on nervousness ahead of ...</td>\n",
       "      <td>http:https://uk.reuters.com/article/markets-hongkong...</td>\n",
       "      <td>Hong Kong stock erased early gains investors l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21594</th>\n",
       "      <td>* FTSEurofirst 300 falls 0.5 pct; Euro STOXX 5...</td>\n",
       "      <td>2014-07-08 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Banks lead European shares lower as US fine co...</td>\n",
       "      <td>http:https://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>Banking stocks led European shares German bank...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54009</th>\n",
       "      <td>TOKYO, Sept 16 (Reuters) - The Nikkei benchma...</td>\n",
       "      <td>2011-09-15</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Nikkei set to rise, financials may outperform</td>\n",
       "      <td>http:https://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>The Nikkei benchmark may climb helped gains Wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55917</th>\n",
       "      <td>* Nikkei briefly pares gains after N.Korean n...</td>\n",
       "      <td>2009-05-25</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Nikkei up 1.3 pct, shrugs off N. Korea nuclear...</td>\n",
       "      <td>http:https://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>Japan Nikkei stock mostly shrugged North Korea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58650</th>\n",
       "      <td>BANGKOK, Aug 5 (Reuters) - Indonesian shares e...</td>\n",
       "      <td>2014-08-05</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>SE Asia Stocks -Indonesian, Philippine shares ...</td>\n",
       "      <td>http:https://uk.reuters.com/article/markets-southeas...</td>\n",
       "      <td>Indonesian shares eased weaker expected econom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12142</th>\n",
       "      <td>\\nA political tussle over Indonesia's formidab...</td>\n",
       "      <td>2014-01-09 15:54:00</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Natural resources and sensible leaders bring h...</td>\n",
       "      <td>https://www.theguardian.com/world/2014/jan/09/...</td>\n",
       "      <td>A political tussle Indonesia formidable minera...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_text             datetime  \\\n",
       "31653   FRANKFURT, July 5 (Reuters) - European stocks...  2007-07-05 00:00:00   \n",
       "57149  (Changes entry on Porsche to be filed under Vo...           2016-12-23   \n",
       "79304  As dawn breaks in Asia traders are readying th...  2017-02-14 23:00:00   \n",
       "5334   \\nMy father was born in 1919 and died in 2002....  2015-03-22 00:05:05   \n",
       "42951  Sept 4 (Reuters) - Hong Kong stock erased earl...           2015-09-04   \n",
       "21594  * FTSEurofirst 300 falls 0.5 pct; Euro STOXX 5...  2014-07-08 00:00:00   \n",
       "54009   TOKYO, Sept 16 (Reuters) - The Nikkei benchma...           2011-09-15   \n",
       "55917   * Nikkei briefly pares gains after N.Korean n...           2009-05-25   \n",
       "58650  BANGKOK, Aug 5 (Reuters) - Indonesian shares e...           2014-08-05   \n",
       "12142  \\nA political tussle over Indonesia's formidab...  2014-01-09 15:54:00   \n",
       "\n",
       "         source                                              title  \\\n",
       "31653   Reuters  European shares end down on Vodafone, rate wor...   \n",
       "57149   Reuters  REFILE-German stocks - Factors to watch on Dec...   \n",
       "79304        FT                Fast Asia Open: Between two Yellens   \n",
       "5334   Guardian  Enough of the dry politics of numbers. We need...   \n",
       "42951   Reuters  Hong Kong stocks fall on nervousness ahead of ...   \n",
       "21594   Reuters  Banks lead European shares lower as US fine co...   \n",
       "54009   Reuters      Nikkei set to rise, financials may outperform   \n",
       "55917   Reuters  Nikkei up 1.3 pct, shrugs off N. Korea nuclear...   \n",
       "58650   Reuters  SE Asia Stocks -Indonesian, Philippine shares ...   \n",
       "12142  Guardian  Natural resources and sensible leaders bring h...   \n",
       "\n",
       "                                                     url  \\\n",
       "31653  http:https://uk.reuters.com/article/markets-europe-s...   \n",
       "57149  http:https://uk.reuters.com/article/germany-stocks-f...   \n",
       "79304  https://www.ft.com/content/78392d38-8e4b-32d4-...   \n",
       "5334   https://www.theguardian.com/commentisfree/2015...   \n",
       "42951  http:https://uk.reuters.com/article/markets-hongkong...   \n",
       "21594  http:https://uk.reuters.com/article/markets-europe-s...   \n",
       "54009  http:https://uk.reuters.com/article/markets-japan-st...   \n",
       "55917  http:https://uk.reuters.com/article/markets-japan-st...   \n",
       "58650  http:https://uk.reuters.com/article/markets-southeas...   \n",
       "12142  https://www.theguardian.com/world/2014/jan/09/...   \n",
       "\n",
       "                                    cleaned_article_text  \n",
       "31653  European stocks led telecommunications heavywe...  \n",
       "57149  The DAX looked set according premarket brokera...  \n",
       "79304  As dawn breaks Asia traders readying purgatory...  \n",
       "5334   My father born died For first years life stock...  \n",
       "42951  Hong Kong stock erased early gains investors l...  \n",
       "21594  Banking stocks led European shares German bank...  \n",
       "54009  The Nikkei benchmark may climb helped gains Wa...  \n",
       "55917  Japan Nikkei stock mostly shrugged North Korea...  \n",
       "58650  Indonesian shares eased weaker expected econom...  \n",
       "12142  A political tussle Indonesia formidable minera...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stem / Lemmatize Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['text_lancaster'] = df['cleaned_article_text'].apply(stem_lem_text, type='Lancaster')\n",
    "df['text_porter'] = df['cleaned_article_text'].apply(stem_lem_text, type='Porter')\n",
    "df['text_snowball'] = df['cleaned_article_text'].apply(stem_lem_text, type='Snowball')\n",
    "df['text_lemmatize'] = df['cleaned_article_text'].apply(stem_lem_text, type='Lemmatize')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('news_articles_stemmed_lemmatized_no_extra_stops.csv', index=None, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}