11.8 crawler/Web Crawler.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Crawler Implementation on TAU courses syllabus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import dateutil.parser as dparser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### using beautiful soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "אלגוריתמי קירוב לאופטימיזציה קומבינטורית 0365-4150-01 מדעים מדויקים ב' 1200 - 0900 ה' 209  שרייבר מתמטי שיעור ד\"ר בוכבינדר ניב\n"
     ]
    }
   ],
   "source": [
    "page = requests.get('http:https://yedion.tau.ac.il/yed/yednew.dll?MfcISAPICommand=but&year=2014&course_num=03654150')\n",
    "soup = BeautifulSoup(page.content, 'html.parser')\n",
    "\n",
    "tables = soup.findAll(\"table\", {\"class\" : \"printtable\"})\n",
    "records = [] # store all of the records in this list\n",
    "for table in tables:\n",
    "    rec = []\n",
    "    heads = table.findAll('th')\n",
    "    rows = table.findAll('tr', {\"dir\" : \"rtl\"})\n",
    "    for h in heads:\n",
    "        rec.append(h.contents[0])\n",
    "    for d in rows[0].findAll('td'):\n",
    "        rec.append(d.contents[0])\n",
    "    records.append(rec)\n",
    "  \n",
    "for i in records: \n",
    "    for j in i:\n",
    "        print j,\n",
    "    print\n",
    "\n",
    "# tabulka = soup.find(\"table\", {\"class\" : \"printtable\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse to df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_to_df_html(html_add):\n",
    "    df = pd.read_html(html_add)\n",
    "    df = df[0].dropna(how = 'all')\n",
    "    df = df.reset_index()\n",
    "    df.drop('index',axis=1, inplace=True)\n",
    "    \n",
    "    if df.shape[0] == 1:\n",
    "        return\n",
    "    # Create the answer df\n",
    "    data = pd.DataFrame(columns=('LastUpdate', 'CourseName', 'CourseNumber', 'Faculty','LecturerName','ClassType','Building','Room','Day','Hour','Semester'))\n",
    "\n",
    "    #convert df to a dictionary\n",
    "    dic = df.to_dict()\n",
    "\n",
    "    #static field - lastUpdate\n",
    "    lastUpdate = dparser.parse(dic[0][0],fuzzy=True).strftime('%m/%d/%Y')\n",
    "\n",
    "    #looping over each course\n",
    "    i = 1\n",
    "    j= 0\n",
    "    while i< df.shape[0]:\n",
    "        k=1\n",
    "        while i+k < df.shape[0]:\n",
    "            if pd.isnull(df.loc[i+k,1]) == False and contain_courseNum(df.loc[i+k,1]):\n",
    "                break\n",
    "            k +=1\n",
    "\n",
    "        main = df.loc[i+2,]\n",
    "        data.loc[j] = [lastUpdate,dic[0][i],dic[1][i],dic[0][i+1],main[6],main[5], main[4], main[3], main[2], main[1], main[0]]\n",
    "        j +=1\n",
    "        i = i+k\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def contain_courseNum(num):    \n",
    "    return len([i for i, ltr in enumerate(num) if ltr == '-']) ==2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get course links\n",
    "page = requests.get('https://engineering.tau.ac.il/yedion/2015-16/14')\n",
    "soup = BeautifulSoup(page.content, 'html.parser')\n",
    "divs = soup.find('div', {'id':'block-system-main'})\n",
    "courses = set()\n",
    "for a in divs.findAll('a', {'class':None}):\n",
    "    href =  a.attrs['href']\n",
    "    if href[:23]=='http:https://yedion.tau.ac.il':\n",
    "        courses.add(href)\n",
    "        \n",
    "# create a df for all cuorses\n",
    "ans = pd.DataFrame()\n",
    "\n",
    "for i in courses:\n",
    "    toAdd = parse_to_df_html(i)\n",
    "    ans = pd.concat([ans, toAdd])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = parse_to_df_html('http:https://yedion.tau.ac.il/yed/yednew.dll?MfcISAPICommand=but&year=2015&course_num=05714177')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LastUpdate</th>\n",
       "      <th>CourseName</th>\n",
       "      <th>CourseNumber</th>\n",
       "      <th>Faculty</th>\n",
       "      <th>LecturerName</th>\n",
       "      <th>ClassType</th>\n",
       "      <th>Building</th>\n",
       "      <th>Room</th>\n",
       "      <th>Day</th>\n",
       "      <th>Hour</th>\n",
       "      <th>Semester</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01/06/2017</td>\n",
       "      <td>מערכות לניהול קשרי לקוחות Crm  Customer Relati...</td>\n",
       "      <td>0571-4177-01</td>\n",
       "      <td>הנדסה</td>\n",
       "      <td>מר עסיס אברהם</td>\n",
       "      <td>שיעור</td>\n",
       "      <td>ולפסון הנדסה</td>\n",
       "      <td>134.0</td>\n",
       "      <td>ה'</td>\n",
       "      <td>1700 - 1500</td>\n",
       "      <td>ב'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01/06/2017</td>\n",
       "      <td>מערכות לניהול קשרי לקוחות Crm  Customer Relati...</td>\n",
       "      <td>0571-4177-02</td>\n",
       "      <td>הנדסה</td>\n",
       "      <td>מר עסיס אברהם</td>\n",
       "      <td>תרגיל</td>\n",
       "      <td>ולפסון הנדסה</td>\n",
       "      <td>134.0</td>\n",
       "      <td>ה'</td>\n",
       "      <td>1800 - 1700</td>\n",
       "      <td>ב'</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LastUpdate                                         CourseName  \\\n",
       "0  01/06/2017  מערכות לניהול קשרי לקוחות Crm  Customer Relati...   \n",
       "1  01/06/2017  מערכות לניהול קשרי לקוחות Crm  Customer Relati...   \n",
       "\n",
       "   CourseNumber Faculty   LecturerName ClassType      Building   Room Day  \\\n",
       "0  0571-4177-01   הנדסה  מר עסיס אברהם     שיעור  ולפסון הנדסה  134.0  ה'   \n",
       "1  0571-4177-02   הנדסה  מר עסיס אברהם     תרגיל  ולפסון הנדסה  134.0  ה'   \n",
       "\n",
       "          Hour Semester  \n",
       "0  1700 - 1500       ב'  \n",
       "1  1800 - 1700       ב'  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}