Chapter 2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploring and processing text data "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-1. Converting Text Data to Lowercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=['This is introduction to NLP','It is likely to be useful, to people ','Machine learning is the new electrcity','There would be less hype around AI and more action going forward','python is the best tool!','R is good langauage','I like this book','I want more books like this']\n",
    "\n",
    "#convert list to data frame\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "print(df)\n",
    "\n",
    "x = 'Testing'\n",
    "x2 = x.lower()\n",
    "print(x2)\n",
    "\n",
    "\n",
    "df['tweet'] = df['tweet'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "df['tweet']\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-2. Removing Punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=['This is introduction to NLP','It is likely to be useful, to people ','Machine learning is the new electrcity','There would be less hype around AI and more action going forward','python is the best tool!','R is good langauage','I like this book','I want more books like this']\n",
    "#convert list to dataframe\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "print(df)\n",
    "\n",
    "import re\n",
    "\n",
    "s = \"I. like. This book!\"\n",
    "s1 = re.sub(r'[^\\w\\s]','',s)\n",
    "s1\n",
    "\n",
    "df['tweet'] = df['tweet'].str.replace('[^\\w\\s]','')\n",
    "df['tweet']\n",
    "\n",
    "import string\n",
    "\n",
    "s = \"I. like. This book!\"\n",
    "\n",
    "for c in string.punctuation:\n",
    "s= s.replace(c,\"\")\n",
    "s\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-3. Removing Stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=['This is introduction to NLP','It is likely to be useful, to people ','Machine learning is the new electrcity','There would be less hype around AI and more action going forward','python is the best tool!','R is good langauage','I like this book','I want more books like this']\n",
    "\n",
    "#convert list to data frame\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "print(df)\n",
    "\n",
    "\n",
    "#install and import libraries\n",
    "\n",
    "!pip install nltk\n",
    "import nltk\n",
    "nltk.download()\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#remove stop words\n",
    "\n",
    "stop = stopwords.words('english')\n",
    "df['tweet'] = df['tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\n",
    "df['tweet']\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-4. Standardizing Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lookup_dict = {'nlp':'natural language processing', 'ur':'your', \"wbu\" : \"what about you\"}\n",
    "\n",
    "import re\n",
    "\n",
    "def text_std(input_text):\n",
    " words = input_text.split()\n",
    " new_words = []\n",
    " for word in words:\n",
    "     word = re.sub(r'[^\\w\\s]','',word)\n",
    "     if word.lower() in lookup_dict:\n",
    "         word = lookup_dict[word.lower()]\n",
    "         new_words.append(word)\n",
    "         new_text = \" \".join(new_words)\n",
    " return new_text\n",
    "\n",
    "text_std(\"I like nlp it's ur choice\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-5. Correcting Spelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=['Introduction to NLP','It is likely to be useful, to people ','Machine learning is the new electrcity','R is good langauage','I like this book','I want more books like this']\n",
    "\n",
    "\n",
    "#convert list to dataframe\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "print(df)\n",
    "\n",
    "#Install textblob library\n",
    "!pip install textblob\n",
    "\n",
    "#import libraries and use 'correct' function \n",
    "\n",
    "from textblob import TextBlob\n",
    "\n",
    "df['tweet'].apply(lambda x: str(TextBlob(x).correct()))\n",
    "\n",
    "#You can also use autocorrect library as shown below\n",
    "\n",
    "#install autocorrect\n",
    "\n",
    "!pip install autocorrect\n",
    "\n",
    "from autocorrect import spell\n",
    "print(spell(u'mussage'))\n",
    "print(spell(u'sirvice'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-6. Tokenizing Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's create a list of strings and assign it to a variable. \n",
    "text=['This is introduction to NLP','It is likely to be useful, to people ','Machine learning is the new electrcity','There would be less hype around AI and more action going forward','python is the best tool!','R is good langauage','I like this book','I want more books like this']\n",
    "\n",
    "#convert list to dataframe\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "print(df)\n",
    "\n",
    "#Using textblob\n",
    "from textblob import TextBlob\n",
    "TextBlob(df['tweet'][3]).words\n",
    "\n",
    "#output\n",
    "WordList(['would', 'less', 'hype', 'around', 'ai', 'action', 'going', 'forward'])\n",
    "\n",
    "#using NLTK\n",
    "import nltk\n",
    "\n",
    "#create data\n",
    "mystring = \"My favorite animal is cat\"\n",
    "\n",
    "nltk.word_tokenize(mystring)\n",
    "\n",
    "#output\n",
    "['My', 'favorite', 'animal', 'is', 'cat']\n",
    "\n",
    "#using split function from python\n",
    "mystring.split()\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-7. Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=['I like fishing','I eat fish','There are many fishes in pound']\n",
    "\n",
    "#convert list to dataframe\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "print(df)\n",
    "\n",
    "#Import library\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "st = PorterStemmer()\n",
    "\n",
    "df['tweet'][:5].apply(lambda x: \" \".join([st.stem(word) for word in x.split()]))\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-8. Lemmatizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=['I like fishing','I eat fish','There are many fishes in pound', 'leaves and leaf']\n",
    "\n",
    "#convert list to dataframe\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'tweet':text})\n",
    "\n",
    "print(df)\n",
    "\n",
    "#Import library\n",
    "from textblob import Word\n",
    "\n",
    "#Code for lemmatize\n",
    "df['tweet'] = df['tweet'].apply(lambda x: \" \".join([Word(word).lemmatize() for word in x.split()]))\n",
    "\n",
    "df['tweet']\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-9. Exploring Text Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Execute below code to download dataset, if you haven’t already nltk.download().\n",
    "\n",
    "#Importing data\n",
    "import nltk\n",
    "from nltk.corpus import webtext\n",
    "nltk.download('webtext')\n",
    "wt_sentences = webtext.sents('firefox.txt')\n",
    "wt_words = webtext.words('firefox.txt')\n",
    "\n",
    "# Import Library for computing frequency \n",
    "from nltk.probability import FreqDist\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "\n",
    "\n",
    "# Count the number of words \n",
    "\n",
    "len(wt_sentences)\n",
    "len(wt_words)\n",
    "\n",
    "frequency_dist = nltk.FreqDist(wt_words) \n",
    "frequency_dist\n",
    "\n",
    "sorted_frequency_dist =sorted(frequency_dist,key=frequency_dist.__getitem__, reverse=True)\n",
    "sorted_frequency_dist\n",
    "\n",
    "large_words = dict([(k,v) for k,v in frequency_dist.items() if len(k)>3])\n",
    "\n",
    "frequency_dist = nltk.FreqDist(large_words)\n",
    "frequency_dist.plot(50,cumulative=False)\n",
    "\n",
    "#install library\n",
    "!pip install wordcloud\n",
    "\n",
    "#build wordcloud\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "wcloud = WordCloud().generate_from_frequencies(frequency_dist)\n",
    "\n",
    "#plotting the wordcloud\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(wcloud, interpolation='bilinear')\n",
    "plt.axis(\"off\")\n",
    "(-0.5, 399.5, 199.5, -0.5)\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Recipe 2-10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweet_sample= \"How to take control of your #debt https://personal.vanguard.com/us/insights/saving-investing/debt-management.#Best advice for #family #financial #success (@PrepareToWin)\"\n",
    "\n",
    "def processRow(row):\n",
    "    \n",
    "    import re\n",
    "    import nltk\n",
    "    from textblob import TextBlob\n",
    "    from nltk.corpus import stopwords\n",
    "    from nltk.stem import PorterStemmer\n",
    "    from textblob import Word\n",
    "    from nltk.util import ngrams\n",
    "    import re\n",
    "    from wordcloud import WordCloud, STOPWORDS\n",
    "    from nltk.tokenize import word_tokenize\n",
    "    \n",
    "    tweet = row\n",
    "    #Lower case\n",
    "    tweet.lower()\n",
    "    #Removes unicode strings like \"\\u002c\" and \"x96\" \n",
    "    tweet = re.sub(r'(\\\\u[0-9A-Fa-f]+)',r'', tweet)       \n",
    "    tweet = re.sub(r'[^\\x00-\\x7f]',r'',tweet)\n",
    "    #convert any url to URL\n",
    "    tweet = re.sub('((www\\.[^\\s]+)|(https?:https://[^\\s]+))','URL',tweet)\n",
    "    #Convert any @Username to \"AT_USER\"\n",
    "    tweet = re.sub('@[^\\s]+','AT_USER',tweet)\n",
    "    #Remove additional white spaces\n",
    "    tweet = re.sub('[\\s]+', ' ', tweet)\n",
    "    tweet = re.sub('[\\n]+', ' ', tweet)\n",
    "    #Remove not alphanumeric symbols white spaces\n",
    "    tweet = re.sub(r'[^\\w]', ' ', tweet)\n",
    "    #Removes hastag in front of a word \"\"\"\n",
    "    tweet = re.sub(r'#([^\\s]+)', r'\\1', tweet)\n",
    "    #Replace #word with word\n",
    "    tweet = re.sub(r'#([^\\s]+)', r'\\1', tweet)\n",
    "    #Remove :( or :)\n",
    "    tweet = tweet.replace(':)','')\n",
    "    tweet = tweet.replace(':(','')\n",
    "    #remove numbers\n",
    "    tweet = ''.join([i for i in tweet if not i.isdigit()]) \n",
    "    #remove multiple exclamation\n",
    "    tweet = re.sub(r\"(\\!)\\1+\", ' ', tweet)\n",
    "    #remove multiple question marks\n",
    "    tweet = re.sub(r\"(\\?)\\1+\", ' ', tweet)\n",
    "    #remove multistop\n",
    "    tweet = re.sub(r\"(\\.)\\1+\", ' ', tweet)\n",
    "    #lemma\n",
    "    from textblob import Word\n",
    "    tweet =\" \".join([Word(word).lemmatize() for word in tweet.split()])\n",
    "    #stemmer\n",
    "    #st = PorterStemmer()\n",
    "    #tweet=\" \".join([st.stem(word) for word in tweet.split()])\n",
    "    #Removes emoticons from text \n",
    "    tweet = re.sub(':\\)|;\\)|:-\\)|\\(-:|:-D|=D|:P|xD|X-p|\\^\\^|:-*|\\^\\.\\^|\\^\\-\\^|\\^\\_\\^|\\,-\\)|\\)-:|:\\'\\(|:\\(|:-\\(|:\\S|T\\.T|\\.\\_\\.|:<|:-\\S|:-<|\\*\\-\\*|:O|=O|=\\-O|O\\.o|XO|O\\_O|:-\\@|=/|:/|X\\-\\(|>\\.<|>=\\(|D:', '', tweet)\n",
    "    #trim\n",
    "    tweet = tweet.strip('\\'\"')\n",
    "\n",
    "    row = tweet\n",
    "\n",
    "    return row\n",
    "\n",
    "#call the function with your data\n",
    "processRow(tweet_sample)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}