notebooks/train_reg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model\n",
    "\n",
    "This notebook attempts to train the model, choose the best one based on a Metric(MAE) and deploy automatically\n",
    "\n",
    "\n",
    " - loads the data from a dat location and extracts some more features for the prediction\n",
    " - trains \n",
    "     - a Linear regression model\n",
    "     - a Random Forest Regression model\n",
    "     \n",
    " - Running the notebookl shoud automaticaly store/deployed the better model(based on MAE) at a location/model registry\n",
    " \n",
    " - the web api is plugged in to the model registry and will automatically picks up the deployed model\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.  Load Setup on Collab Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #uncomment and run if working on collab\n",
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #uncomment and run if working on collab\n",
    "# !rm -rf mlcore\n",
    "# !cp -r /content/drive/MyDrive/data/ data/\n",
    "# !mkdir logs\n",
    "# !mkdir models\n",
    "# !unzip /content/drive/MyDrive/data/mlcore.zip\n",
    "#!cd mlcore && pip install -e . && cd .. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.  Load Env Variables ( Uncomment if not running on Docker)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # run this if running locally not required if you used the docker script\n",
    "# #!pip install python-dotenv\n",
    "# from dotenv import load_dotenv\n",
    "# load_dotenv(dotenv_path = '../.env')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import requires packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from datetime import datetime\n",
    "from mlcore.data_helper import load_data\n",
    "from mlcore.utils import set_logger\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = datetime.now()\n",
    "nb_run_id = 'trng_'+ ts.strftime(\"%m_%d_%Y_%H_%M_%S\")\n",
    "training_logger = set_logger(nb_run_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 load data from given files' (schemas)\n",
    "\n",
    "A data_dict and helper function is used to allow data loaa from multiple files/data sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:41:29,062:Loaded schema AB_NYC_2019 in dataframe with shape (48895, 16)\n"
     ]
    }
   ],
   "source": [
    "data_dict = {\n",
    "    'AB_NYC_2019':None,\n",
    "}\n",
    "\n",
    "\n",
    "for schema_name in data_dict:\n",
    "    data_dict[schema_name] = load_data(schema_name, logger = training_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48895, 16)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "id                                                              2539\n",
       "name                              Clean & quiet apt home by the park\n",
       "host_id                                                         2787\n",
       "host_name                                                       John\n",
       "neighbourhood_group                                         Brooklyn\n",
       "neighbourhood                                             Kensington\n",
       "latitude                                                    40.64749\n",
       "longitude                                                  -73.97237\n",
       "room_type                                               Private room\n",
       "price                                                            149\n",
       "minimum_nights                                                     1\n",
       "number_of_reviews                                                  9\n",
       "last_review                                               2018-10-19\n",
       "reviews_per_month                                               0.21\n",
       "calculated_host_listings_count                                     6\n",
       "availability_365                                                 365\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abnb_data = data_dict['AB_NYC_2019']\n",
    "print(abnb_data.shape)\n",
    "abnb_data.iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1  Set up experiment data/features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_outliers(df, feature_name, lower_qtl=0.01, upper_qtl=0.99):\n",
    "    \n",
    "    cutoff_low = df[feature_name].quantile(lower_qtl)\n",
    "    cutoff_high  = abnb_data[feature_name].quantile(upper_qtl)\n",
    "\n",
    "    df = df[(df[feature_name] < cutoff_high) & (df[feature_name] > cutoff_low)]\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:41:29,126:org_shape:(48895, 16) \n",
      "<ipython-input-9-87d4d6c13543>:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  abnb_data.drop_duplicates(inplace=True)\n",
      "<ipython-input-9-87d4d6c13543>:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  abnb_data.dropna(inplace=True)\n",
      "2022-04-30 06:41:29,243:after outlier removal for feature::price --> shape:(47728, 9) \n",
      "2022-04-30 06:41:29,270:train_shape:(38182, 9) \n",
      "2022-04-30 06:41:29,271:test_shape:(9546, 9)  \n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "cat_features = ['neighbourhood_group', 'room_type','neighbourhood']\n",
    "num_features =['minimum_nights', 'calculated_host_listings_count' ]\n",
    "long_lat = ['latitude', 'longitude']\n",
    "text_features = ['name']\n",
    "review_features = ['number_of_reviews', 'reviews_per_month']\n",
    "target=['price']\n",
    "\n",
    "training_logger.info('org_shape:{} '.format(abnb_data.shape))\n",
    "\n",
    "\n",
    "\n",
    "features = cat_features + num_features + target +long_lat + text_features\n",
    "abnb_data = abnb_data[features]\n",
    "abnb_data.drop_duplicates(inplace=True)\n",
    "abnb_data.dropna(inplace=True)\n",
    "\n",
    "# Train after removing outliers (commenting below 2 lines push up the error)\n",
    "for ftr in ['price']:\n",
    "    abnb_data = remove_outliers(abnb_data, ftr, lower_qtl=0.01, upper_qtl=0.99)\n",
    "    training_logger.info('after outlier removal for feature::{} --> shape:{} '.format(ftr, abnb_data.shape))\n",
    "\n",
    "#split randomly\n",
    "train_data,test_data = train_test_split(abnb_data,test_size=0.2,shuffle=True,random_state=0)\n",
    "\n",
    "train_data.fillna(0)\n",
    "test_data.fillna(0)\n",
    "\n",
    "\n",
    "training_logger.info('train_shape:{} '.format(train_data.shape))\n",
    "training_logger.info('test_shape:{}  '.format(test_data.shape))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2  Feature Transformation/Scaling pipeline set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.pipeline import Pipeline,FeatureUnion\n",
    "\n",
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up pipeline for classical models\n",
    "categorical_preprocessor = OneHotEncoder(handle_unknown=\"ignore\")\n",
    "numerical_preprocessor = StandardScaler()\n",
    "tfidf_preprocesor = TfidfVectorizer()\n",
    "\n",
    "preprocessor = ColumnTransformer([\n",
    "    ('one-hot-encoder', categorical_preprocessor, cat_features),\n",
    "    ('standard-scaler', numerical_preprocessor, num_features),\n",
    "    ('tf-idf', tfidf_preprocesor, text_features[0])\n",
    "    ])\n",
    "\n",
    "preprocessor_notext = ColumnTransformer([\n",
    "    ('one-hot-encoder', categorical_preprocessor, cat_features),\n",
    "    ('standard-scaler', numerical_preprocessor, num_features),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Model Training and Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.1 Load previous models and new models which will be trained and compared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlcore.train_eval_helper_reg import *\n",
    "from mlcore.modelops import load_model, save_model, read_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mldbpath = '../data/mldbreg.sqlite'\n",
    "deployed_model_info=None\n",
    "deployed_model_obj = None\n",
    "load_prev_model = False\n",
    "#from tensorflow import keras\n",
    "\n",
    "if load_prev_model:\n",
    "    try:\n",
    "        deployed_model_info = read_data(mldbpath, 'deployed_model').iloc[0].to_dict()\n",
    "        if deployed_model_info:\n",
    "            deployed_model_name = deployed_model_info['final_model_name']\n",
    "            deployed_model_obj = load_model(deployed_model_name, cur_logger=training_logger) \n",
    "            if deployed_model_obj['type']=='DL':\n",
    "                actual_obj = keras.models.load_model('../models/'+deployed_model_name+'.deep_mdl')\n",
    "            deployed_model_obj['obj'] = actual_obj\n",
    "            training_logger.info('Loaded model {}'.format(deployed_model_info))\n",
    "    except :\n",
    "        training_logger.info('Could not load deployed model it may not exist')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.2 Set up  Models which need to be trained on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set up Models\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "\n",
    "models = {\n",
    "        'LinReg' : {'obj': LinearRegression(),\n",
    "             'param_grid':{},\n",
    "             \"type\": 'classical',\n",
    "             'preprocessor' : preprocessor,\n",
    "             \"features\":cat_features+num_features + text_features\n",
    "                 \n",
    "                },\n",
    "\n",
    "    'SVR' : {'obj': SVR(),\n",
    "             'param_grid':{'svr__C': [0.1,1, 10, 100], \n",
    "                           'svr__gamma': [1,0.1,0.01,0.001],\n",
    "                           'svr__kernel': ['rbf', 'poly', 'sigmoid']\n",
    "                          },\n",
    "             \"type\": \"classical\",\n",
    "             \"preprocessor\" : preprocessor,\n",
    "             \"features\":cat_features+num_features+text_features,\n",
    "             \n",
    "            },\n",
    "    \n",
    "    'RF' : {'obj': RandomForestRegressor(),\n",
    "         'param_grid':{'randomforestregressor__max_depth': [10, 30, 60, 90,100],\n",
    "                       'randomforestregressor__min_samples_leaf': [2, 4, 6,8,20],\n",
    "                       'randomforestregressor__min_samples_split': [5, 10,15],\n",
    "                       'randomforestregressor__n_estimators': [200, 300,600,900,1200],\n",
    "                       'randomforestregressor__bootstrap' : [True, False],\n",
    "                       'randomforestregressor__max_features' : ['auto', 'sqrt']\n",
    "                       \n",
    "                      },\n",
    "         \"type\": \"classical\",\n",
    "         \"preprocessor\" : preprocessor_notext,\n",
    "         \"features\":cat_features+num_features,\n",
    "\n",
    "        }    \n",
    "        }\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>obj</th>\n",
       "      <th>param_grid</th>\n",
       "      <th>type</th>\n",
       "      <th>preprocessor</th>\n",
       "      <th>features</th>\n",
       "      <th>mean_squared_error</th>\n",
       "      <th>mean_absolute_error</th>\n",
       "      <th>r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinReg</td>\n",
       "      <td>LinearRegression()</td>\n",
       "      <td>{}</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVR</td>\n",
       "      <td>SVR()</td>\n",
       "      <td>{'svr__C': [0.1, 1, 10, 100], 'svr__gamma': [1...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RF</td>\n",
       "      <td>RandomForestRegressor()</td>\n",
       "      <td>{'randomforestregressor__max_depth': [10, 30, ...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index                      obj  \\\n",
       "0  LinReg       LinearRegression()   \n",
       "1     SVR                    SVR()   \n",
       "2      RF  RandomForestRegressor()   \n",
       "\n",
       "                                          param_grid       type  \\\n",
       "0                                                 {}  classical   \n",
       "1  {'svr__C': [0.1, 1, 10, 100], 'svr__gamma': [1...  classical   \n",
       "2  {'randomforestregressor__max_depth': [10, 30, ...  classical   \n",
       "\n",
       "                                        preprocessor  \\\n",
       "0  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "1  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "2  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "\n",
       "                                            features  mean_squared_error  \\\n",
       "0  [neighbourhood_group, room_type, neighbourhood...                   0   \n",
       "1  [neighbourhood_group, room_type, neighbourhood...                   0   \n",
       "2  [neighbourhood_group, room_type, neighbourhood...                   0   \n",
       "\n",
       "   mean_absolute_error  r2_score  \n",
       "0                    0         0  \n",
       "1                    0         0  \n",
       "2                    0         0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = {'mean_squared_error':0, 'mean_absolute_error':0, 'r2_score':0}\n",
    "if deployed_model_obj:\n",
    "    models['deployed_model'] = {\n",
    "        'obj':deployed_model_obj['obj'],\n",
    "        'param_grid':deployed_model_obj['param_grid'],\n",
    "        'features':deployed_model_obj['features'],\n",
    "        'preprocessor':deployed_model_obj['preprocessor'],\n",
    "         'type':deployed_model_obj['type']\n",
    "    }\n",
    "    \n",
    "for metric in metrics:\n",
    "    for mdl in models:\n",
    "        models[mdl][metric]=0\n",
    "\n",
    "get_df_from_dict(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.3 Train Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:41:30,991:Training started for LinReg\n",
      "2022-04-30 06:41:32,612:Training ended for LinReg\n",
      "2022-04-30 06:41:32,613:Training started for SVR\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/utils/validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n",
      "2022-04-30 06:43:05,014:Training ended for SVR\n",
      "2022-04-30 06:43:05,015:Training started for RF\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/pipeline.py:394: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  self._final_estimator.fit(Xt, y, **fit_params_last_step)\n",
      "2022-04-30 06:43:13,712:Training ended for RF\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>obj</th>\n",
       "      <th>param_grid</th>\n",
       "      <th>type</th>\n",
       "      <th>preprocessor</th>\n",
       "      <th>features</th>\n",
       "      <th>mean_squared_error</th>\n",
       "      <th>mean_absolute_error</th>\n",
       "      <th>r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinReg</td>\n",
       "      <td>LinearRegression()</td>\n",
       "      <td>{}</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVR</td>\n",
       "      <td>SVR()</td>\n",
       "      <td>{'svr__C': [0.1, 1, 10, 100], 'svr__gamma': [1...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RF</td>\n",
       "      <td>(DecisionTreeRegressor(max_features='auto', ra...</td>\n",
       "      <td>{'randomforestregressor__max_depth': [10, 30, ...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    model                                                obj  \\\n",
       "0  LinReg                                 LinearRegression()   \n",
       "1     SVR                                              SVR()   \n",
       "2      RF  (DecisionTreeRegressor(max_features='auto', ra...   \n",
       "\n",
       "                                          param_grid       type  \\\n",
       "0                                                 {}  classical   \n",
       "1  {'svr__C': [0.1, 1, 10, 100], 'svr__gamma': [1...  classical   \n",
       "2  {'randomforestregressor__max_depth': [10, 30, ...  classical   \n",
       "\n",
       "                                        preprocessor  \\\n",
       "0  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "1  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "2  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "\n",
       "                                            features  mean_squared_error  \\\n",
       "0  [neighbourhood_group, room_type, neighbourhood...                   0   \n",
       "1  [neighbourhood_group, room_type, neighbourhood...                   0   \n",
       "2  [neighbourhood_group, room_type, neighbourhood...                   0   \n",
       "\n",
       "   mean_absolute_error  r2_score  \n",
       "0                    0         0  \n",
       "1                    0         0  \n",
       "2                    0         0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Train Models (trained model is stored back in dict)\n",
    "use_dask = True\n",
    "\n",
    "# daks is still buggy\n",
    "if use_dask:\n",
    "    from dask.distributed import Client, progress\n",
    "    dask_client = Client(processes=False, threads_per_worker=2,\n",
    "                    n_workers=2, memory_limit='3GB')\n",
    "else:\n",
    "    dask_client = None\n",
    "\n",
    "comparison_result_dict = train_models(models, \n",
    "                                      train_data, \n",
    "                                      target, \n",
    "                                      training_logger, \n",
    "                                      dask_client = dask_client, \n",
    "                                      randomized_search=True,\n",
    "                                      scoring_metric = 'r2_score'\n",
    "                                     )\n",
    "\n",
    "#comparison_result_dict = train_models(classifiers, X_train,y_train, training_logger)\n",
    "comparison_result = get_df_from_dict(comparison_result_dict, idxname='model')\n",
    "comparison_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.3 Test/Compare  Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:43:13,760:Testing started for LinReg\n",
      "2022-04-30 06:43:13,889:Testing ended for LinReg on metric mean_squared_error\n",
      "2022-04-30 06:43:13,891:Testing started for SVR\n",
      "2022-04-30 06:43:34,100:Testing ended for SVR on metric mean_squared_error\n",
      "2022-04-30 06:43:34,102:Testing started for RF\n",
      "2022-04-30 06:43:34,182:Testing ended for RF on metric mean_squared_error\n",
      "2022-04-30 06:43:34,184:Testing started for LinReg\n",
      "2022-04-30 06:43:34,264:Testing ended for LinReg on metric mean_absolute_error\n",
      "2022-04-30 06:43:34,267:Testing started for SVR\n",
      "2022-04-30 06:43:54,267:Testing ended for SVR on metric mean_absolute_error\n",
      "2022-04-30 06:43:54,270:Testing started for RF\n",
      "2022-04-30 06:43:54,348:Testing ended for RF on metric mean_absolute_error\n",
      "2022-04-30 06:43:54,350:Testing started for LinReg\n",
      "2022-04-30 06:43:54,433:Testing ended for LinReg on metric r2_score\n",
      "2022-04-30 06:43:54,436:Testing started for SVR\n",
      "2022-04-30 06:44:14,681:Testing ended for SVR on metric r2_score\n",
      "2022-04-30 06:44:14,684:Testing started for RF\n",
      "2022-04-30 06:44:14,764:Testing ended for RF on metric r2_score\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>obj</th>\n",
       "      <th>param_grid</th>\n",
       "      <th>type</th>\n",
       "      <th>preprocessor</th>\n",
       "      <th>features</th>\n",
       "      <th>mean_squared_error</th>\n",
       "      <th>mean_absolute_error</th>\n",
       "      <th>r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinReg</td>\n",
       "      <td>LinearRegression()</td>\n",
       "      <td>{}</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>6312.595211</td>\n",
       "      <td>49.398590</td>\n",
       "      <td>0.413396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVR</td>\n",
       "      <td>SVR()</td>\n",
       "      <td>{'svr__C': [0.1, 1, 10, 100], 'svr__gamma': [1...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>7118.395927</td>\n",
       "      <td>46.375446</td>\n",
       "      <td>0.338516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RF</td>\n",
       "      <td>(DecisionTreeRegressor(max_features='auto', ra...</td>\n",
       "      <td>{'randomforestregressor__max_depth': [10, 30, ...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[neighbourhood_group, room_type, neighbourhood...</td>\n",
       "      <td>6299.533762</td>\n",
       "      <td>47.898508</td>\n",
       "      <td>0.414610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    model                                                obj  \\\n",
       "0  LinReg                                 LinearRegression()   \n",
       "1     SVR                                              SVR()   \n",
       "2      RF  (DecisionTreeRegressor(max_features='auto', ra...   \n",
       "\n",
       "                                          param_grid       type  \\\n",
       "0                                                 {}  classical   \n",
       "1  {'svr__C': [0.1, 1, 10, 100], 'svr__gamma': [1...  classical   \n",
       "2  {'randomforestregressor__max_depth': [10, 30, ...  classical   \n",
       "\n",
       "                                        preprocessor  \\\n",
       "0  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "1  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "2  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "\n",
       "                                            features  mean_squared_error  \\\n",
       "0  [neighbourhood_group, room_type, neighbourhood...         6312.595211   \n",
       "1  [neighbourhood_group, room_type, neighbourhood...         7118.395927   \n",
       "2  [neighbourhood_group, room_type, neighbourhood...         6299.533762   \n",
       "\n",
       "   mean_absolute_error  r2_score  \n",
       "0            49.398590  0.413396  \n",
       "1            46.375446  0.338516  \n",
       "2            47.898508  0.414610  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute errr ON TEST DATA ( RMSE is computed and stored back in dict)\n",
    "for metric in metrics:\n",
    "    comparison_result_dict = test_models(models, test_data, target, training_logger,dask_client, error_metric=metric)\n",
    "    #comparison_result_dict = test_models(classifiers, X_test,y_test, target)\n",
    "\n",
    "comparison_result = get_df_from_dict(comparison_result_dict, idxname='model')\n",
    "comparison_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:44:14,811:Best performing model on basic of metric mean_absolute_error is SVR\n"
     ]
    }
   ],
   "source": [
    "#Get best Model Based on a metric\n",
    "metric = 'mean_absolute_error'\n",
    "best_model_row = get_best_model(comparison_result, metric,reverse=True)\n",
    "best_model = best_model_row['model']\n",
    "best_model_id = models[best_model]['obj']\n",
    "training_logger.info(\"Best performing model on basic of metric {} is {}\".format(metric, best_model))\n",
    "#best_model_row.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model stored at ../models/SVR_model_04_30_2022_06_41_28.mdl\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_model_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVR_model_04_30_2022_06_41_28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                final_model_name\n",
       "0  SVR_model_04_30_2022_06_41_28"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Save/Deploy final_trained_model\n",
    "final_trained_model = best_model_row.to_dict()\n",
    "final_model_name =final_trained_model['model']+'_'+nb_run_id.replace('trng','model')\n",
    "\n",
    "#save model\n",
    "model_type = final_trained_model['type']\n",
    "\n",
    "save_model(final_trained_model, final_model_name, model_type=model_type)\n",
    "\n",
    "deploy_df = pd.DataFrame([[final_model_name]], \n",
    "                         columns =['final_model_name'])\n",
    "deploy_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.3 Update Model registry/Deploy Best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_dict = {\n",
    "    'deployed_model':deploy_df,\n",
    "    'hist_deployed_models':deploy_df,\n",
    "    #'train_report':comparison_result\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model SVR_model_04_30_2022_06_41_28 deployed \n",
      " associated reports saved in respective tables with id:trng_04_30_2022_06_41_28\n"
     ]
    }
   ],
   "source": [
    "from mlcore.dbhelper import store_data, overwrite_data\n",
    "for dkey in schema_dict:\n",
    "    data_to_be_stored = schema_dict[dkey]\n",
    "    if dkey=='deployed_model':\n",
    "         overwrite_data(data_to_be_stored, mldbpath, dkey)\n",
    "    else:\n",
    "        store_data(data_to_be_stored, mldbpath, dkey)\n",
    "\n",
    "dep_model_name = deploy_df['final_model_name'].iloc[0]\n",
    "print('Model {} deployed \\n associated reports saved in respective tables with id:{}'.format(dep_model_name,nb_run_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:44:15,007:Traning job with id trng_04_30_2022_06_41_28 finished\n"
     ]
    }
   ],
   "source": [
    "training_logger.info('Traning job with id {} finished'.format(nb_run_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}