Breast Cancer Predicition - Classification Algorithms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import NullFormatter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "# Code to reduce notebook warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sklearn Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import jaccard_similarity_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the Metric Reporting Dictionary\n",
    "metric_report_dict = {'Algorithm':[],\n",
    "                      'Log Loss':[],\n",
    "                      'Jaccard Index':[],\n",
    "                      'F1 Score':[],\n",
    "                      'Accuracy Score':[]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analyzing Credit Card Data.ipynb\n",
      "Breast Cancer Predicition - Classification Algorithms.ipynb\n",
      "credit_card_data.csv\n",
      "Customer Gas Geospatial Analysis.ipynb\n",
      "customer_gas_survey.csv\n",
      "Dr nyawate\n",
      "ecommerce_data.csv\n",
      "README.md\n",
      "so_cancer_detection\n",
      "tpot_breast_cancer_pipepline.py\n",
      "tpot_cleaned_data.csv\n",
      "wbc_cancer_data.csv\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID number</th>\n",
       "      <th>mean_radius</th>\n",
       "      <th>mean_texture</th>\n",
       "      <th>mean_perimeter</th>\n",
       "      <th>mean_area</th>\n",
       "      <th>mean_smoothness</th>\n",
       "      <th>mean_compactness</th>\n",
       "      <th>mean_concavity</th>\n",
       "      <th>mean_concave_points</th>\n",
       "      <th>mean_symmetry</th>\n",
       "      <th>...</th>\n",
       "      <th>worst_texture</th>\n",
       "      <th>worst_perimeter</th>\n",
       "      <th>worst_area</th>\n",
       "      <th>worst_smoothness</th>\n",
       "      <th>worst_compactness</th>\n",
       "      <th>worst_concavity</th>\n",
       "      <th>worst_concave_points</th>\n",
       "      <th>worst_symmetry</th>\n",
       "      <th>worst_fractal_dimension</th>\n",
       "      <th>Diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>842302</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>842517</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84300903</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84348301</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84358402</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID number  mean_radius   mean_texture   mean_perimeter  mean_area  \\\n",
       "0     842302         17.99          10.38          122.80     1001.0   \n",
       "1     842517         20.57          17.77          132.90     1326.0   \n",
       "2   84300903         19.69          21.25          130.00     1203.0   \n",
       "3   84348301         11.42          20.38           77.58      386.1   \n",
       "4   84358402         20.29          14.34          135.10     1297.0   \n",
       "\n",
       "   mean_smoothness   mean_compactness   mean_concavity   mean_concave_points   \\\n",
       "0           0.11840            0.27760           0.3001               0.14710   \n",
       "1           0.08474            0.07864           0.0869               0.07017   \n",
       "2           0.10960            0.15990           0.1974               0.12790   \n",
       "3           0.14250            0.28390           0.2414               0.10520   \n",
       "4           0.10030            0.13280           0.1980               0.10430   \n",
       "\n",
       "   mean_symmetry      ...      worst_texture   worst_perimeter  worst_area  \\\n",
       "0          0.2419     ...               17.33           184.60      2019.0   \n",
       "1          0.1812     ...               23.41           158.80      1956.0   \n",
       "2          0.2069     ...               25.53           152.50      1709.0   \n",
       "3          0.2597     ...               26.50            98.87       567.7   \n",
       "4          0.1809     ...               16.67           152.20      1575.0   \n",
       "\n",
       "   worst_smoothness   worst_compactness   worst_concavity   \\\n",
       "0             0.1622              0.6656            0.7119   \n",
       "1             0.1238              0.1866            0.2416   \n",
       "2             0.1444              0.4245            0.4504   \n",
       "3             0.2098              0.8663            0.6869   \n",
       "4             0.1374              0.2050            0.4000   \n",
       "\n",
       "   worst_concave_points   worst_symmetry   worst_fractal_dimension   Diagnosis  \n",
       "0                 0.2654           0.4601                  0.11890           1  \n",
       "1                 0.1860           0.2750                  0.08902           1  \n",
       "2                 0.2430           0.3613                  0.08758           1  \n",
       "3                 0.2575           0.6638                  0.17300           1  \n",
       "4                 0.1625           0.2364                  0.07678           1  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer_df = pd.read_csv(\"wbc_cancer_data.csv\")\n",
    "cancer_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID number                           int64\n",
       "mean_radius                       float64\n",
       "mean_texture                      float64\n",
       "mean_perimeter                    float64\n",
       "mean_area                         float64\n",
       "mean_smoothness                   float64\n",
       "mean_compactness                  float64\n",
       "mean_concavity                    float64\n",
       "mean_concave_points               float64\n",
       "mean_symmetry                     float64\n",
       "mean_fractal_dimension            float64\n",
       "Standard_dev_radius               float64\n",
       "Standard_dev_texture              float64\n",
       "Standard_dev_perimeter            float64\n",
       "Standard_dev_area                 float64\n",
       "Standard_dev_smoothness           float64\n",
       "Standard_dev_compactness          float64\n",
       "Standard_dev_concavity            float64\n",
       "Standard_dev_concave_points       float64\n",
       "Standard_dev_symmetry             float64\n",
       "Standard_dev_fractal_dimension    float64\n",
       "worst_radius                      float64\n",
       "worst_texture                     float64\n",
       "worst_perimeter                   float64\n",
       "worst_area                        float64\n",
       "worst_smoothness                  float64\n",
       "worst_compactness                 float64\n",
       "worst_concavity                   float64\n",
       "worst_concave_points              float64\n",
       "worst_symmetry                    float64\n",
       "worst_fractal_dimension           float64\n",
       " Diagnosis                          int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID number</th>\n",
       "      <th>mean_radius</th>\n",
       "      <th>mean_texture</th>\n",
       "      <th>mean_perimeter</th>\n",
       "      <th>mean_area</th>\n",
       "      <th>mean_smoothness</th>\n",
       "      <th>mean_compactness</th>\n",
       "      <th>mean_concavity</th>\n",
       "      <th>mean_concave_points</th>\n",
       "      <th>mean_symmetry</th>\n",
       "      <th>...</th>\n",
       "      <th>worst_texture</th>\n",
       "      <th>worst_perimeter</th>\n",
       "      <th>worst_area</th>\n",
       "      <th>worst_smoothness</th>\n",
       "      <th>worst_compactness</th>\n",
       "      <th>worst_concavity</th>\n",
       "      <th>worst_concave_points</th>\n",
       "      <th>worst_symmetry</th>\n",
       "      <th>worst_fractal_dimension</th>\n",
       "      <th>Diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>842302</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>842517</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84300903</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84348301</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84358402</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID number  mean_radius   mean_texture   mean_perimeter  mean_area  \\\n",
       "0     842302         17.99          10.38          122.80     1001.0   \n",
       "1     842517         20.57          17.77          132.90     1326.0   \n",
       "2   84300903         19.69          21.25          130.00     1203.0   \n",
       "3   84348301         11.42          20.38           77.58      386.1   \n",
       "4   84358402         20.29          14.34          135.10     1297.0   \n",
       "\n",
       "   mean_smoothness   mean_compactness   mean_concavity   mean_concave_points   \\\n",
       "0           0.11840            0.27760           0.3001               0.14710   \n",
       "1           0.08474            0.07864           0.0869               0.07017   \n",
       "2           0.10960            0.15990           0.1974               0.12790   \n",
       "3           0.14250            0.28390           0.2414               0.10520   \n",
       "4           0.10030            0.13280           0.1980               0.10430   \n",
       "\n",
       "   mean_symmetry      ...      worst_texture   worst_perimeter  worst_area  \\\n",
       "0          0.2419     ...               17.33           184.60      2019.0   \n",
       "1          0.1812     ...               23.41           158.80      1956.0   \n",
       "2          0.2069     ...               25.53           152.50      1709.0   \n",
       "3          0.2597     ...               26.50            98.87       567.7   \n",
       "4          0.1809     ...               16.67           152.20      1575.0   \n",
       "\n",
       "   worst_smoothness   worst_compactness   worst_concavity   \\\n",
       "0             0.1622              0.6656            0.7119   \n",
       "1             0.1238              0.1866            0.2416   \n",
       "2             0.1444              0.4245            0.4504   \n",
       "3             0.2098              0.8663            0.6869   \n",
       "4             0.1374              0.2050            0.4000   \n",
       "\n",
       "   worst_concave_points   worst_symmetry   worst_fractal_dimension   Diagnosis  \n",
       "0                 0.2654           0.4601                  0.11890           1  \n",
       "1                 0.1860           0.2750                  0.08902           1  \n",
       "2                 0.2430           0.3613                  0.08758           1  \n",
       "3                 0.2575           0.6638                  0.17300           1  \n",
       "4                 0.1625           0.2364                  0.07678           1  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_radius</th>\n",
       "      <th>mean_texture</th>\n",
       "      <th>mean_perimeter</th>\n",
       "      <th>mean_area</th>\n",
       "      <th>mean_smoothness</th>\n",
       "      <th>mean_compactness</th>\n",
       "      <th>mean_concavity</th>\n",
       "      <th>mean_concave_points</th>\n",
       "      <th>mean_symmetry</th>\n",
       "      <th>mean_fractal_dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst_texture</th>\n",
       "      <th>worst_perimeter</th>\n",
       "      <th>worst_area</th>\n",
       "      <th>worst_smoothness</th>\n",
       "      <th>worst_compactness</th>\n",
       "      <th>worst_concavity</th>\n",
       "      <th>worst_concave_points</th>\n",
       "      <th>worst_symmetry</th>\n",
       "      <th>worst_fractal_dimension</th>\n",
       "      <th>Diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_radius   mean_texture   mean_perimeter  mean_area  mean_smoothness   \\\n",
       "0         17.99          10.38          122.80     1001.0           0.11840   \n",
       "1         20.57          17.77          132.90     1326.0           0.08474   \n",
       "2         19.69          21.25          130.00     1203.0           0.10960   \n",
       "3         11.42          20.38           77.58      386.1           0.14250   \n",
       "4         20.29          14.34          135.10     1297.0           0.10030   \n",
       "\n",
       "   mean_compactness   mean_concavity   mean_concave_points   mean_symmetry   \\\n",
       "0            0.27760           0.3001               0.14710          0.2419   \n",
       "1            0.07864           0.0869               0.07017          0.1812   \n",
       "2            0.15990           0.1974               0.12790          0.2069   \n",
       "3            0.28390           0.2414               0.10520          0.2597   \n",
       "4            0.13280           0.1980               0.10430          0.1809   \n",
       "\n",
       "   mean_fractal_dimension     ...      worst_texture   worst_perimeter  \\\n",
       "0                 0.07871     ...               17.33           184.60   \n",
       "1                 0.05667     ...               23.41           158.80   \n",
       "2                 0.05999     ...               25.53           152.50   \n",
       "3                 0.09744     ...               26.50            98.87   \n",
       "4                 0.05883     ...               16.67           152.20   \n",
       "\n",
       "   worst_area  worst_smoothness   worst_compactness   worst_concavity   \\\n",
       "0      2019.0             0.1622              0.6656            0.7119   \n",
       "1      1956.0             0.1238              0.1866            0.2416   \n",
       "2      1709.0             0.1444              0.4245            0.4504   \n",
       "3       567.7             0.2098              0.8663            0.6869   \n",
       "4      1575.0             0.1374              0.2050            0.4000   \n",
       "\n",
       "   worst_concave_points   worst_symmetry   worst_fractal_dimension   Diagnosis  \n",
       "0                 0.2654           0.4601                  0.11890           1  \n",
       "1                 0.1860           0.2750                  0.08902           1  \n",
       "2                 0.2430           0.3613                  0.08758           1  \n",
       "3                 0.2575           0.6638                  0.17300           1  \n",
       "4                 0.1625           0.2364                  0.07678           1  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer_df = cancer_df.drop(columns=[\"ID number\"])\n",
    "cancer_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_radius</th>\n",
       "      <th>mean_texture</th>\n",
       "      <th>mean_perimeter</th>\n",
       "      <th>mean_area</th>\n",
       "      <th>mean_smoothness</th>\n",
       "      <th>mean_compactness</th>\n",
       "      <th>mean_concavity</th>\n",
       "      <th>mean_concave_points</th>\n",
       "      <th>mean_symmetry</th>\n",
       "      <th>mean_fractal_dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst_radius</th>\n",
       "      <th>worst_texture</th>\n",
       "      <th>worst_perimeter</th>\n",
       "      <th>worst_area</th>\n",
       "      <th>worst_smoothness</th>\n",
       "      <th>worst_compactness</th>\n",
       "      <th>worst_concavity</th>\n",
       "      <th>worst_concave_points</th>\n",
       "      <th>worst_symmetry</th>\n",
       "      <th>worst_fractal_dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>25.38</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>24.99</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>23.57</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>22.54</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_radius   mean_texture   mean_perimeter  mean_area  mean_smoothness   \\\n",
       "0         17.99          10.38          122.80     1001.0           0.11840   \n",
       "1         20.57          17.77          132.90     1326.0           0.08474   \n",
       "2         19.69          21.25          130.00     1203.0           0.10960   \n",
       "3         11.42          20.38           77.58      386.1           0.14250   \n",
       "4         20.29          14.34          135.10     1297.0           0.10030   \n",
       "\n",
       "   mean_compactness   mean_concavity   mean_concave_points   mean_symmetry   \\\n",
       "0            0.27760           0.3001               0.14710          0.2419   \n",
       "1            0.07864           0.0869               0.07017          0.1812   \n",
       "2            0.15990           0.1974               0.12790          0.2069   \n",
       "3            0.28390           0.2414               0.10520          0.2597   \n",
       "4            0.13280           0.1980               0.10430          0.1809   \n",
       "\n",
       "   mean_fractal_dimension           ...             worst_radius   \\\n",
       "0                 0.07871           ...                     25.38   \n",
       "1                 0.05667           ...                     24.99   \n",
       "2                 0.05999           ...                     23.57   \n",
       "3                 0.09744           ...                     14.91   \n",
       "4                 0.05883           ...                     22.54   \n",
       "\n",
       "   worst_texture   worst_perimeter  worst_area  worst_smoothness   \\\n",
       "0           17.33           184.60      2019.0             0.1622   \n",
       "1           23.41           158.80      1956.0             0.1238   \n",
       "2           25.53           152.50      1709.0             0.1444   \n",
       "3           26.50            98.87       567.7             0.2098   \n",
       "4           16.67           152.20      1575.0             0.1374   \n",
       "\n",
       "   worst_compactness   worst_concavity   worst_concave_points   \\\n",
       "0              0.6656            0.7119                 0.2654   \n",
       "1              0.1866            0.2416                 0.1860   \n",
       "2              0.4245            0.4504                 0.2430   \n",
       "3              0.8663            0.6869                 0.2575   \n",
       "4              0.2050            0.4000                 0.1625   \n",
       "\n",
       "   worst_symmetry   worst_fractal_dimension  \n",
       "0           0.4601                  0.11890  \n",
       "1           0.2750                  0.08902  \n",
       "2           0.3613                  0.08758  \n",
       "3           0.6638                  0.17300  \n",
       "4           0.2364                  0.07678  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract the X and Y\n",
    "X = cancer_df.iloc[:,:-1]\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    1\n",
       "3    1\n",
       "4    1\n",
       "Name:  Diagnosis, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = cancer_df.iloc[:,-1]\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standardizing/Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.09706398e+00, -2.07333501e+00,  1.26993369e+00,\n",
       "         9.84374905e-01,  1.56846633e+00,  3.28351467e+00,\n",
       "         2.65287398e+00,  2.53247522e+00,  2.21751501e+00,\n",
       "         2.25574689e+00,  2.48973393e+00, -5.65265059e-01,\n",
       "         2.83303087e+00,  2.48757756e+00, -2.14001647e-01,\n",
       "         1.31686157e+00,  7.24026158e-01,  6.60819941e-01,\n",
       "         1.14875667e+00,  9.07083081e-01,  1.88668963e+00,\n",
       "        -1.35929347e+00,  2.30360062e+00,  2.00123749e+00,\n",
       "         1.30768627e+00,  2.61666502e+00,  2.10952635e+00,\n",
       "         2.29607613e+00,  2.75062224e+00,  1.93701461e+00],\n",
       "       [ 1.82982061e+00, -3.53632408e-01,  1.68595471e+00,\n",
       "         1.90870825e+00, -8.26962447e-01, -4.87071673e-01,\n",
       "        -2.38458552e-02,  5.48144156e-01,  1.39236330e-03,\n",
       "        -8.68652457e-01,  4.99254601e-01, -8.76243603e-01,\n",
       "         2.63326966e-01,  7.42401948e-01, -6.05350847e-01,\n",
       "        -6.92926270e-01, -4.40780058e-01,  2.60162067e-01,\n",
       "        -8.05450380e-01, -9.94437403e-02,  1.80592744e+00,\n",
       "        -3.69203222e-01,  1.53512599e+00,  1.89048899e+00,\n",
       "        -3.75611957e-01, -4.30444219e-01, -1.46748968e-01,\n",
       "         1.08708430e+00, -2.43889668e-01,  2.81189987e-01],\n",
       "       [ 1.57988811e+00,  4.56186952e-01,  1.56650313e+00,\n",
       "         1.55888363e+00,  9.42210440e-01,  1.05292554e+00,\n",
       "         1.36347845e+00,  2.03723076e+00,  9.39684817e-01,\n",
       "        -3.98007910e-01,  1.22867595e+00, -7.80083377e-01,\n",
       "         8.50928301e-01,  1.18133606e+00, -2.97005012e-01,\n",
       "         8.14973504e-01,  2.13076435e-01,  1.42482747e+00,\n",
       "         2.37035535e-01,  2.93559404e-01,  1.51187025e+00,\n",
       "        -2.39743838e-02,  1.34747521e+00,  1.45628455e+00,\n",
       "         5.27407405e-01,  1.08293217e+00,  8.54973944e-01,\n",
       "         1.95500035e+00,  1.15225500e+00,  2.01391209e-01]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Scale the Values of X\n",
    "X = StandardScaler().fit(X).transform(X)\n",
    "X[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(569, 30) (569,)\n"
     ]
    }
   ],
   "source": [
    "# Saving untampered with X and Y values for TPOT Optimization\n",
    "X_tpot = X\n",
    "y_tpot = y\n",
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf = ExtraTreesClassifier(n_estimators=50)\n",
    "# clf = clf.fit(X, y)\n",
    "lsvc = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(X, y)\n",
    "# clf.feature_importances_  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(569, 7)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = SelectFromModel(lsvc, prefit=True)\n",
    "X_new = model.transform(X)\n",
    "X_new.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Optimized Value of X after Feature Selection\n",
    "X = X_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    1\n",
       "Name:  Diagnosis, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Split for Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set: (455, 7) (455,)\n",
      "Test Set: (114, 7) (114,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4)\n",
    "\n",
    "print(\"Train Set:\", X_train.shape, y_train.shape)\n",
    "print(\"Test Set:\", X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import all the Classification Algorithms\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import metrics\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Support Vector Machine Accuracy before Parameter Tuning is:  0.9912280701754386\n"
     ]
    }
   ],
   "source": [
    "# Import support vector machine\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "svm_model = svm.SVC(kernel=\"rbf\")\n",
    "svm_model.fit(X_train, y_train)\n",
    "yhat = svm_model.predict(X_test)\n",
    "\n",
    "print(\"Support Vector Machine Accuracy before Parameter Tuning is: \", metrics.accuracy_score(y_test, yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# svm parameter selection\n",
    "def svc_param_selection(X, y, nfolds):\n",
    "    Cs = [0.001, 0.01, 0.1, 1, 10]\n",
    "    gammas = [0.001, 0.01, 0.1, 1]\n",
    "    param_grid = {'C': Cs, 'gamma' : gammas}\n",
    "    grid_search = GridSearchCV(svm.SVC(kernel='rbf'), param_grid, cv=nfolds)\n",
    "    grid_search.fit(X, y)\n",
    "    grid_search.best_params_\n",
    "    return grid_search.best_params_\n",
    "\n",
    "svm_model = svm.SVC(kernel=\"rbf\")\n",
    "svm_model.fit(X_train, y_train)\n",
    "yhat = svm_model.predict(X_test)\n",
    "svc_param = svc_param_selection(X_train, y_train, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Support Vector Machine Accuracy after Parameter Tuning is:  0.9912280701754386\n"
     ]
    }
   ],
   "source": [
    "svm_model = svm.SVC(kernel=\"rbf\", C=svc_param['C'], gamma=svc_param['gamma'])\n",
    "svm_model.fit(X_train, y_train)\n",
    "yhat = svm_model.predict(X_test)\n",
    "\n",
    "print(\"Support Vector Machine Accuracy after Parameter Tuning is: \", metrics.accuracy_score(y_test, yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy:  0.9912280701754386\n",
      "Jaccard Similarity :  0.9912280701754386\n",
      "F1 Score :  0.9912640501993294\n"
     ]
    }
   ],
   "source": [
    "metric_report_dict[\"Algorithm\"].append(\"Support Vector Machine\")\n",
    "metric_report_dict[\"Log Loss\"].append(np.nan)\n",
    "metric_report_dict[\"Jaccard Index\"].append(jaccard_similarity_score(y_test, yhat))\n",
    "metric_report_dict[\"F1 Score\"].append(f1_score(y_test, yhat, average='weighted'))\n",
    "metric_report_dict[\"Accuracy Score\"].append(metrics.accuracy_score(y_test, yhat))\n",
    "\n",
    "print(\"Decision Tree Accuracy: \", metrics.accuracy_score(y_test, yhat))\n",
    "print(\"Jaccard Similarity : \", jaccard_similarity_score(y_test, yhat))\n",
    "print(\"F1 Score : \", f1_score(y_test, yhat, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K Nearest Neighbours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value of K is:  1\n",
      "Test set Accuracy for K :  0.9649122807017544\n",
      "Value of K is:  3\n",
      "Test set Accuracy for K :  0.9736842105263158\n",
      "Value of K is:  5\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  7\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  9\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  11\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  13\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  15\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  17\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  19\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  21\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  23\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  25\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  27\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  29\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  31\n",
      "Test set Accuracy for K :  0.9736842105263158\n",
      "Value of K is:  33\n",
      "Test set Accuracy for K :  0.9736842105263158\n",
      "Value of K is:  35\n",
      "Test set Accuracy for K :  0.9824561403508771\n",
      "Value of K is:  37\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  39\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  41\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  43\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  45\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  47\n",
      "Test set Accuracy for K :  0.9912280701754386\n",
      "Value of K is:  49\n",
      "Test set Accuracy for K :  0.9912280701754386\n"
     ]
    }
   ],
   "source": [
    "# Function to test different values of K and get optimum K value\n",
    "myList = list(range(1,50))\n",
    "\n",
    "# subsetting just the odd ones\n",
    "neighbors = filter(lambda x: x % 2 != 0, myList)\n",
    "\n",
    "for k in neighbors:\n",
    "    neigh = KNeighborsClassifier(n_neighbors = k).fit(X_train, y_train)\n",
    "    yhat = neigh.predict(X_test)\n",
    "    print(\"Value of K is: \", k)\n",
    "    print(\"Test set Accuracy for K : \", metrics.accuracy_score(y_test, yhat))\n",
    "    \n",
    "# From this list the best value of K is 35 we will therefore use 35 to create our KNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value of K is:  13\n",
      "Test set Accuracy for K :  0.9912280701754386\n"
     ]
    }
   ],
   "source": [
    "# Our selected K values is K Nearest Neighbour\n",
    "k = 13\n",
    "neigh = KNeighborsClassifier(n_neighbors = k).fit(X_train, y_train)\n",
    "yhat = neigh.predict(X_test)\n",
    "print(\"Value of K is: \", k)\n",
    "print(\"Test set Accuracy for K : \", metrics.accuracy_score(y_test, yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy:  0.9912280701754386\n",
      "Jaccard Similarity :  0.9912280701754386\n",
      "F1 Score :  0.9912640501993294\n"
     ]
    }
   ],
   "source": [
    "metric_report_dict[\"Algorithm\"].append(\"K-Nearest Neighbour\")\n",
    "metric_report_dict[\"Log Loss\"].append(np.nan)\n",
    "metric_report_dict[\"Jaccard Index\"].append(jaccard_similarity_score(y_test, yhat))\n",
    "metric_report_dict[\"F1 Score\"].append(f1_score(y_test, yhat, average='weighted'))\n",
    "metric_report_dict[\"Accuracy Score\"].append(metrics.accuracy_score(y_test, yhat))\n",
    "\n",
    "print(\"Decision Tree Accuracy: \", metrics.accuracy_score(y_test, yhat))\n",
    "print(\"Jaccard Similarity : \", jaccard_similarity_score(y_test, yhat))\n",
    "print(\"F1 Score : \", f1_score(y_test, yhat, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy before Parameter Tuning:  0.8947368421052632\n"
     ]
    }
   ],
   "source": [
    "cancer_tree = DecisionTreeClassifier(criterion=\"entropy\", max_depth=6)\n",
    "cancer_tree.fit(X_train, y_train)\n",
    "yhat = cancer_tree.predict(X_test)\n",
    "print(\"Decision Tree Accuracy before Parameter Tuning: \", metrics.accuracy_score(y_test, yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# using grid search for best parameters\n",
    "def dtree_param_selection(X, y, nfolds):\n",
    "    parameters = {'min_samples_split': range(10,500,20), 'max_depth': range(1,20,2), 'criterion':['gini','entropy']}\n",
    "    tree = DecisionTreeClassifier()\n",
    "    grid_search = GridSearchCV(tree, parameters)\n",
    "    grid_search.fit(X,y)\n",
    "    grid_search.best_params_\n",
    "    return grid_search.best_params_\n",
    "\n",
    "dtree_param = dtree_param_selection(X_train, y_train, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy after GridCV Parameter Tuning:  0.8771929824561403\n"
     ]
    }
   ],
   "source": [
    "cancer_tree = DecisionTreeClassifier(criterion=dtree_param['criterion'], max_depth=dtree_param['max_depth'], min_samples_split=dtree_param['min_samples_split'])\n",
    "cancer_tree.fit(X_train, y_train)\n",
    "yhat = cancer_tree.predict(X_test)\n",
    "print(\"Decision Tree Accuracy after GridCV Parameter Tuning: \", metrics.accuracy_score(y_test, yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy:  0.8771929824561403\n",
      "Jaccard Similarity :  0.8771929824561403\n",
      "F1 Score :  0.8814248458985301\n"
     ]
    }
   ],
   "source": [
    "metric_report_dict[\"Algorithm\"].append(\"Decision Tree Classifier\")\n",
    "metric_report_dict[\"Log Loss\"].append(np.nan)\n",
    "metric_report_dict[\"Jaccard Index\"].append(jaccard_similarity_score(y_test, yhat))\n",
    "metric_report_dict[\"F1 Score\"].append(f1_score(y_test, yhat, average='weighted'))\n",
    "metric_report_dict[\"Accuracy Score\"].append(metrics.accuracy_score(y_test, yhat))\n",
    "\n",
    "print(\"Decision Tree Accuracy: \", metrics.accuracy_score(y_test, yhat))\n",
    "print(\"Jaccard Similarity : \", jaccard_similarity_score(y_test, yhat))\n",
    "print(\"F1 Score : \", f1_score(y_test, yhat, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistics Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Accuracy before Parameter Tuning is:  0.9824561403508771\n"
     ]
    }
   ],
   "source": [
    "LR = LogisticRegression(solver='liblinear').fit(X_train, y_train)\n",
    "yhat = LR.predict(X_test)\n",
    "accuracy_before_tuning = metrics.accuracy_score(y_test, yhat)\n",
    "print(\"LR Accuracy before Parameter Tuning is: \", accuracy_before_tuning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logistics regression = finding the best parameter\n",
    "def logistics_param_selection(X, y, nfolds):\n",
    "    param_grid = {'C': [0.01, 0.1, 1, 10, 100], 'penalty': ['l1', 'l2'], 'solver':['liblinear']}\n",
    "    gridsearch = GridSearchCV(LogisticRegression(), param_grid, cv=nfolds)\n",
    "    gridsearch.fit(X, y)\n",
    "    return gridsearch.best_params_\n",
    "\n",
    "\n",
    "logistics_param = logistics_param_selection(X_train, y_train, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying optimum parameters\n",
    "# LR_tuned = LogisticRegression(solver=logistics_param['solver'], C=logistics_param['C'], penalty=logistics_param['penalty']).fit(X_train, y_train)\n",
    "# yhat = LR_tuned.predict(X_test)\n",
    "# yhat_proba = LR_tuned.predict_proba(X_test)\n",
    "# accuracy_after_tuning = metrics.accuracy_score(y_test, yhat)\n",
    "# print(\"LR Accuracy after parameter tuning is: \", accuracy_after_tuning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Accuracy after parameter tuning is:  0.9824561403508771\n"
     ]
    }
   ],
   "source": [
    "LR_tuned = LogisticRegression(solver='liblinear').fit(X_train, y_train)\n",
    "yhat = LR_tuned.predict(X_test)\n",
    "yhat_proba = LR_tuned.predict_proba(X_test)\n",
    "accuracy_after_tuning = metrics.accuracy_score(y_test, yhat)\n",
    "print(\"LR Accuracy after parameter tuning is: \", accuracy_after_tuning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy:  0.9824561403508771\n",
      "Jaccard Similarity :  0.9824561403508771\n",
      "F1 Score :  0.9825957298309063\n"
     ]
    }
   ],
   "source": [
    "metric_report_dict[\"Algorithm\"].append(\"Logistics Regression\")\n",
    "metric_report_dict[\"Log Loss\"].append(log_loss(y_test, yhat_proba))\n",
    "metric_report_dict[\"Jaccard Index\"].append(jaccard_similarity_score(y_test, yhat))\n",
    "metric_report_dict[\"F1 Score\"].append(f1_score(y_test, yhat, average='weighted'))\n",
    "metric_report_dict[\"Accuracy Score\"].append(metrics.accuracy_score(y_test, yhat))\n",
    "\n",
    "print(\"Decision Tree Accuracy: \", metrics.accuracy_score(y_test, yhat))\n",
    "print(\"Jaccard Similarity : \", jaccard_similarity_score(y_test, yhat))\n",
    "print(\"F1 Score : \", f1_score(y_test, yhat, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation based on Different Accuracy Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# metrics_df = pd.DataFrame(metric_report_dict)\n",
    "# metrics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using TPOT for the most optimum Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\botieno\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\deap\\tools\\_hypervolume\\pyhv.py:33: ImportWarning: Falling back to the python version of hypervolume module. Expect this to be very slow.\n",
      "  \"module. Expect this to be very slow.\", ImportWarning)\n",
      "c:\\users\\botieno\\appdata\\local\\programs\\python\\python37-32\\lib\\importlib\\_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n",
      "  return f(*args, **kwds)\n",
      "c:\\users\\botieno\\appdata\\local\\programs\\python\\python37-32\\lib\\importlib\\_bootstrap_external.py:434: ImportWarning: Not importing directory c:\\users\\botieno\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\mpl_toolkits: missing __init__\n",
      "  _warnings.warn(msg.format(portions[0]), ImportWarning)\n"
     ]
    }
   ],
   "source": [
    "from tpot import TPOTClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X_tpot, y_tpot, test_size=0.2, random_state=4)\n",
    "# # X_train.shape, X_test.shape, y_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tpot = TPOTClassifier(verbosity=2, max_time_mins=20)\n",
    "# tpot.fit(X_train, y_train)\n",
    "# print(tpot.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tpot.export('tpot_breast_cancer_pipepline.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load tpot_breast_cancer_pipepline.py\n",
    "# import numpy as np\n",
    "# import pandas as pd\n",
    "# from sklearn.ensemble import GradientBoostingClassifier\n",
    "# from sklearn.linear_model import LogisticRegression\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.pipeline import make_pipeline, make_union\n",
    "# from tpot.builtins import StackingEstimator\n",
    "\n",
    "# # NOTE: Make sure that the class is labeled 'target' in the data file\n",
    "# tpot_data = pd.read_csv('PATH/TO/DATA/FILE', sep='COLUMN_SEPARATOR', dtype=np.float64)\n",
    "# features = tpot_data.drop('target', axis=1).values\n",
    "# training_features, testing_features, training_target, testing_target = \\\n",
    "#             train_test_split(features, tpot_data['target'].values, random_state=None)\n",
    "\n",
    "# # Average CV score on the training set was:0.9889616180920529\n",
    "# exported_pipeline = make_pipeline(\n",
    "#     StackingEstimator(estimator=LogisticRegression(C=0.5, dual=False, penalty=\"l2\")),\n",
    "#     StackingEstimator(estimator=GradientBoostingClassifier(learning_rate=0.1, max_depth=8, max_features=0.2, min_samples_leaf=10, min_samples_split=5, n_estimators=100, subsample=0.6000000000000001)),\n",
    "#     LogisticRegression(C=0.5, dual=True, penalty=\"l2\")\n",
    "# )\n",
    "\n",
    "# exported_pipeline.fit(training_features, training_target)\n",
    "# results = exported_pipeline.predict(testing_features)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_radius</th>\n",
       "      <th>mean_texture</th>\n",
       "      <th>mean_perimeter</th>\n",
       "      <th>mean_area</th>\n",
       "      <th>mean_smoothness</th>\n",
       "      <th>mean_compactness</th>\n",
       "      <th>mean_concavity</th>\n",
       "      <th>mean_concave_points</th>\n",
       "      <th>mean_symmetry</th>\n",
       "      <th>mean_fractal_dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst_texture</th>\n",
       "      <th>worst_perimeter</th>\n",
       "      <th>worst_area</th>\n",
       "      <th>worst_smoothness</th>\n",
       "      <th>worst_compactness</th>\n",
       "      <th>worst_concavity</th>\n",
       "      <th>worst_concave_points</th>\n",
       "      <th>worst_symmetry</th>\n",
       "      <th>worst_fractal_dimension</th>\n",
       "      <th>Diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_radius   mean_texture   mean_perimeter  mean_area  mean_smoothness   \\\n",
       "0         17.99          10.38          122.80     1001.0           0.11840   \n",
       "1         20.57          17.77          132.90     1326.0           0.08474   \n",
       "2         19.69          21.25          130.00     1203.0           0.10960   \n",
       "3         11.42          20.38           77.58      386.1           0.14250   \n",
       "4         20.29          14.34          135.10     1297.0           0.10030   \n",
       "\n",
       "   mean_compactness   mean_concavity   mean_concave_points   mean_symmetry   \\\n",
       "0            0.27760           0.3001               0.14710          0.2419   \n",
       "1            0.07864           0.0869               0.07017          0.1812   \n",
       "2            0.15990           0.1974               0.12790          0.2069   \n",
       "3            0.28390           0.2414               0.10520          0.2597   \n",
       "4            0.13280           0.1980               0.10430          0.1809   \n",
       "\n",
       "   mean_fractal_dimension     ...      worst_texture   worst_perimeter  \\\n",
       "0                 0.07871     ...               17.33           184.60   \n",
       "1                 0.05667     ...               23.41           158.80   \n",
       "2                 0.05999     ...               25.53           152.50   \n",
       "3                 0.09744     ...               26.50            98.87   \n",
       "4                 0.05883     ...               16.67           152.20   \n",
       "\n",
       "   worst_area  worst_smoothness   worst_compactness   worst_concavity   \\\n",
       "0      2019.0             0.1622              0.6656            0.7119   \n",
       "1      1956.0             0.1238              0.1866            0.2416   \n",
       "2      1709.0             0.1444              0.4245            0.4504   \n",
       "3       567.7             0.2098              0.8663            0.6869   \n",
       "4      1575.0             0.1374              0.2050            0.4000   \n",
       "\n",
       "   worst_concave_points   worst_symmetry   worst_fractal_dimension   Diagnosis  \n",
       "0                 0.2654           0.4601                  0.11890           1  \n",
       "1                 0.1860           0.2750                  0.08902           1  \n",
       "2                 0.2430           0.3613                  0.08758           1  \n",
       "3                 0.2575           0.6638                  0.17300           1  \n",
       "4                 0.1625           0.2364                  0.07678           1  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testing the TPOT Model\n",
    "cancer_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_radius</th>\n",
       "      <th>mean_texture</th>\n",
       "      <th>mean_perimeter</th>\n",
       "      <th>mean_area</th>\n",
       "      <th>mean_smoothness</th>\n",
       "      <th>mean_compactness</th>\n",
       "      <th>mean_concavity</th>\n",
       "      <th>mean_concave_points</th>\n",
       "      <th>mean_symmetry</th>\n",
       "      <th>mean_fractal_dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst_texture</th>\n",
       "      <th>worst_perimeter</th>\n",
       "      <th>worst_area</th>\n",
       "      <th>worst_smoothness</th>\n",
       "      <th>worst_compactness</th>\n",
       "      <th>worst_concavity</th>\n",
       "      <th>worst_concave_points</th>\n",
       "      <th>worst_symmetry</th>\n",
       "      <th>worst_fractal_dimension</th>\n",
       "      <th>Diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_radius   mean_texture   mean_perimeter  mean_area  mean_smoothness   \\\n",
       "0         17.99          10.38          122.80     1001.0           0.11840   \n",
       "1         20.57          17.77          132.90     1326.0           0.08474   \n",
       "2         19.69          21.25          130.00     1203.0           0.10960   \n",
       "3         11.42          20.38           77.58      386.1           0.14250   \n",
       "4         20.29          14.34          135.10     1297.0           0.10030   \n",
       "\n",
       "   mean_compactness   mean_concavity   mean_concave_points   mean_symmetry   \\\n",
       "0            0.27760           0.3001               0.14710          0.2419   \n",
       "1            0.07864           0.0869               0.07017          0.1812   \n",
       "2            0.15990           0.1974               0.12790          0.2069   \n",
       "3            0.28390           0.2414               0.10520          0.2597   \n",
       "4            0.13280           0.1980               0.10430          0.1809   \n",
       "\n",
       "   mean_fractal_dimension     ...      worst_texture   worst_perimeter  \\\n",
       "0                 0.07871     ...               17.33           184.60   \n",
       "1                 0.05667     ...               23.41           158.80   \n",
       "2                 0.05999     ...               25.53           152.50   \n",
       "3                 0.09744     ...               26.50            98.87   \n",
       "4                 0.05883     ...               16.67           152.20   \n",
       "\n",
       "   worst_area  worst_smoothness   worst_compactness   worst_concavity   \\\n",
       "0      2019.0             0.1622              0.6656            0.7119   \n",
       "1      1956.0             0.1238              0.1866            0.2416   \n",
       "2      1709.0             0.1444              0.4245            0.4504   \n",
       "3       567.7             0.2098              0.8663            0.6869   \n",
       "4      1575.0             0.1374              0.2050            0.4000   \n",
       "\n",
       "   worst_concave_points   worst_symmetry   worst_fractal_dimension   Diagnosis  \n",
       "0                 0.2654           0.4601                  0.11890           1  \n",
       "1                 0.1860           0.2750                  0.08902           1  \n",
       "2                 0.2430           0.3613                  0.08758           1  \n",
       "3                 0.2575           0.6638                  0.17300           1  \n",
       "4                 0.1625           0.2364                  0.07678           1  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rename Diagnosis to Target\n",
    "cancer_df_tpot = cancer_df\n",
    "cancer_df_tpot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mean_radius ', 'mean_texture ', 'mean_perimeter', 'mean_area',\n",
       "       'mean_smoothness ', 'mean_compactness ', 'mean_concavity ',\n",
       "       'mean_concave_points ', 'mean_symmetry ', 'mean_fractal_dimension',\n",
       "       'Standard_dev_radius ', 'Standard_dev_texture ',\n",
       "       'Standard_dev_perimeter', 'Standard_dev_area',\n",
       "       'Standard_dev_smoothness ', 'Standard_dev_compactness ',\n",
       "       'Standard_dev_concavity ', 'Standard_dev_concave_points ',\n",
       "       'Standard_dev_symmetry ', 'Standard_dev_fractal_dimension',\n",
       "       'worst_radius ', 'worst_texture ', 'worst_perimeter', 'worst_area',\n",
       "       'worst_smoothness ', 'worst_compactness ', 'worst_concavity ',\n",
       "       'worst_concave_points ', 'worst_symmetry ', 'worst_fractal_dimension',\n",
       "       'target'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer_df_tpot.rename(columns={' Diagnosis':'target'}, inplace=True)\n",
    "cancer_df_tpot.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "cancer_df_tpot.to_csv('tpot_cleaned_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analyzing Credit Card Data.ipynb\n",
      "Breast Cancer Predicition - Classification Algorithms.ipynb\n",
      "credit_card_data.csv\n",
      "Customer Gas Geospatial Analysis.ipynb\n",
      "customer_gas_survey.csv\n",
      "Dr nyawate\n",
      "ecommerce_data.csv\n",
      "README.md\n",
      "so_cancer_detection\n",
      "tpot_breast_cancer_pipepline.py\n",
      "tpot_cleaned_data.csv\n",
      "wbc_cancer_data.csv\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating the TPOT Generated Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mean_radius ', 'mean_texture ', 'mean_perimeter', 'mean_area',\n",
       "       'mean_smoothness ', 'mean_compactness ', 'mean_concavity ',\n",
       "       'mean_concave_points ', 'mean_symmetry ', 'mean_fractal_dimension',\n",
       "       'Standard_dev_radius ', 'Standard_dev_texture ',\n",
       "       'Standard_dev_perimeter', 'Standard_dev_area',\n",
       "       'Standard_dev_smoothness ', 'Standard_dev_compactness ',\n",
       "       'Standard_dev_concavity ', 'Standard_dev_concave_points ',\n",
       "       'Standard_dev_symmetry ', 'Standard_dev_fractal_dimension',\n",
       "       'worst_radius ', 'worst_texture ', 'worst_perimeter', 'worst_area',\n",
       "       'worst_smoothness ', 'worst_compactness ', 'worst_concavity ',\n",
       "       'worst_concave_points ', 'worst_symmetry ', 'worst_fractal_dimension',\n",
       "       'target'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot_data = pd.read_csv('tpot_cleaned_data.csv')\n",
    "tpot_data.drop(columns=['Unnamed: 0'], inplace=True)\n",
    "tpot_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPOT Accuracy:  0.9230769230769231\n",
      "Jaccard Similarity :  0.9230769230769231\n",
      "F1 Score :  0.9225582017661227\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import make_pipeline, make_union\n",
    "from tpot.builtins import StackingEstimator\n",
    "\n",
    "# NOTE: Make sure that the class is labeled 'target' in the data file\n",
    "# tpot_data = pd.read_csv('tpot_cleaned_data.csv')\n",
    "features = tpot_data.drop('target', axis=1).values\n",
    "training_features, testing_features, training_target, testing_target = \\\n",
    "            train_test_split(features, tpot_data['target'].values, random_state=None)\n",
    "\n",
    "# Average CV score on the training set was:0.9889616180920529\n",
    "exported_pipeline = make_pipeline(\n",
    "    StackingEstimator(estimator=LogisticRegression(C=0.5, dual=False, penalty=\"l2\")),\n",
    "    StackingEstimator(estimator=GradientBoostingClassifier(learning_rate=0.1, max_depth=8, max_features=0.2, min_samples_leaf=10, min_samples_split=5, n_estimators=100, subsample=0.6000000000000001)),\n",
    "    LogisticRegression(C=0.5, dual=True, penalty=\"l2\")\n",
    ")\n",
    "\n",
    "exported_pipeline.fit(training_features, training_target)\n",
    "results = exported_pipeline.predict(testing_features)\n",
    "results_proba = exported_pipeline.predict_proba(testing_features)\n",
    "\n",
    "metric_report_dict[\"Algorithm\"].append(\"TPOT Classifier\")\n",
    "metric_report_dict[\"Log Loss\"].append(log_loss(testing_target, results_proba))\n",
    "metric_report_dict[\"Jaccard Index\"].append(jaccard_similarity_score(testing_target, results))\n",
    "metric_report_dict[\"F1 Score\"].append(f1_score(testing_target, results, average='weighted'))\n",
    "metric_report_dict[\"Accuracy Score\"].append(metrics.accuracy_score(testing_target, results))\n",
    "\n",
    "print(\"TPOT Accuracy: \", metrics.accuracy_score(testing_target, results))\n",
    "print(\"Jaccard Similarity : \", jaccard_similarity_score(testing_target, results))\n",
    "print(\"F1 Score : \", f1_score(testing_target, results, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Algorithm</th>\n",
       "      <th>Log Loss</th>\n",
       "      <th>Jaccard Index</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Accuracy Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Support Vector Machine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.991228</td>\n",
       "      <td>0.991264</td>\n",
       "      <td>0.991228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>K-Nearest Neighbour</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.991228</td>\n",
       "      <td>0.991264</td>\n",
       "      <td>0.991228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Decision Tree Classifier</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.877193</td>\n",
       "      <td>0.881425</td>\n",
       "      <td>0.877193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Logistics Regression</td>\n",
       "      <td>0.076105</td>\n",
       "      <td>0.982456</td>\n",
       "      <td>0.982596</td>\n",
       "      <td>0.982456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TPOT Classifier</td>\n",
       "      <td>0.211891</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>0.922558</td>\n",
       "      <td>0.923077</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Algorithm  Log Loss  Jaccard Index  F1 Score  Accuracy Score\n",
       "0    Support Vector Machine       NaN       0.991228  0.991264        0.991228\n",
       "1       K-Nearest Neighbour       NaN       0.991228  0.991264        0.991228\n",
       "2  Decision Tree Classifier       NaN       0.877193  0.881425        0.877193\n",
       "3      Logistics Regression  0.076105       0.982456  0.982596        0.982456\n",
       "4           TPOT Classifier  0.211891       0.923077  0.922558        0.923077"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df = pd.DataFrame(metric_report_dict)\n",
    "metrics_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}