notebooks/train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model\n",
    "\n",
    "This notebook attempts to train the model, choose the best one based on a Metric and deploy automatically\n",
    "\n",
    "\n",
    " - loads the data from a dat location and extracts some more features for the prediction\n",
    " - trains \n",
    "     - a simple classical model\n",
    "     - a node2vec models\n",
    "     - a LSTM model on sequence of items browsed\n",
    " \n",
    " - Running the notebookl shoud automaticaly store/deployed the better model(based on MRR) at a location/model registry\n",
    " \n",
    " - the web api is plugged in to the model registry and will automatically picks up the deployed model\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.  Load Setup on Collab Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #uncomment and run if working on collab\n",
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #uncomment and run if working on collab\n",
    "# !rm -rf mlcore\n",
    "# !cp -r /content/drive/MyDrive/data/ data/\n",
    "# !mkdir logs\n",
    "# !mkdir models\n",
    "# !unzip /content/drive/MyDrive/data/mlcore.zip\n",
    "#!cd mlcore && pip install -e . && cd .. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.  Load Env Variables ( Uncomment if not running on Docker)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # run this if running locally not required if you used the docker script\n",
    "# #!pip install python-dotenv\n",
    "# from dotenv import load_dotenv\n",
    "# load_dotenv(dotenv_path = '../.env')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import requires packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from datetime import datetime\n",
    "from mlcore.data_helper import load_data\n",
    "from mlcore.utils import set_logger\n",
    "from mlcore.feature_extractor import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = datetime.now()\n",
    "nb_run_id = 'trng_'+ ts.strftime(\"%m_%d_%Y_%H_%M_%S\")\n",
    "training_logger = set_logger(nb_run_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 .Load Click Data  and extract features\n",
    "    - join with stores data to get merchant_info\n",
    "    - extract time based features such which day/hour a click was made\n",
    "    - extract user based features such as age of user till click etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:45:51,612:Loaded schema clicks in dataframe with shape (10000, 8)\n",
      "2022-04-30 06:45:51,617:Loaded schema users in dataframe with shape (500, 3)\n",
      "2022-04-30 06:45:51,620:Loaded schema stores in dataframe with shape (100, 2)\n"
     ]
    }
   ],
   "source": [
    "data_dict = {\n",
    "    'clicks':None,\n",
    "    'users':None,\n",
    "    'stores':None\n",
    "}\n",
    "\n",
    "\n",
    "for schema_name in data_dict:\n",
    "    data_dict[schema_name] = load_data(schema_name, logger = training_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_data = data_dict['clicks']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'index': {0: 1000},\n",
       " 'id_x': {0: 7882},\n",
       " 'user_id': {0: 125},\n",
       " 'store_id': {0: 30},\n",
       " 'created_at': {0: Timestamp('2021-09-26 01:17:41')},\n",
       " 'device': {0: 'desktop'},\n",
       " 'channel': {0: 'direct'},\n",
       " 'platform': {0: 'extension'},\n",
       " 'id_y': {0: 30},\n",
       " 'merchant_id': {0: 5},\n",
       " 'hour_of_day': {0: 1},\n",
       " 'day_of_week': {0: 'Sunday'},\n",
       " 'month_of_year': {0: 9},\n",
       " 'date': {0: datetime.date(2021, 9, 26)}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# join with stores and extract time features\n",
    "stores_df = data_dict['stores']\n",
    "transformed_data = pd.merge(transformed_data, stores_df, left_on = 'store_id', right_on = 'id')\n",
    "extract_time_features(transformed_data)\n",
    "transformed_data.iloc[0:1].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'index': 1000,\n",
       " 'id_x': 7882,\n",
       " 'user_id': 125,\n",
       " 'store_id': 30,\n",
       " 'created_at': Timestamp('2021-09-26 01:17:41'),\n",
       " 'device': 'desktop',\n",
       " 'channel': 'direct',\n",
       " 'platform': 'extension',\n",
       " 'id_y': 30,\n",
       " 'merchant_id': 5,\n",
       " 'hour_of_day': 1,\n",
       " 'day_of_week': 'Sunday',\n",
       " 'month_of_year': 9,\n",
       " 'date': datetime.date(2021, 9, 26),\n",
       " 'id': 125,\n",
       " 'signup_datetime': '2021-08-25 07:55:34',\n",
       " 'lifetime_first_purchase_datetime': '2021-09-16 12:06:58',\n",
       " 'click_delta_signup': 45682.11666666667,\n",
       " 'click_delta_first_purchase': 13750.716666666667}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# join with users\n",
    "\n",
    "users_df = data_dict['users']\n",
    "transformed_data = extract_user_features(transformed_data, users_df)\n",
    "transformed_data.iloc[0].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 19)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def create_store_item_click_matrix(data):\n",
    "    \n",
    "#     clicks = data[['user_id','merchant_id']]\n",
    "#     user_item_table = clicks.groupby(['user_id','merchant_id']).size().reset_index(name='merchant_clicked_times')\n",
    "#     user_item_matrix = user_item_table.pivot(index = 'user_id', \n",
    "#                                              columns='merchant_id', \n",
    "#                                              values='merchant_clicked_times').fillna(0)#.unstack(1)\n",
    "\n",
    "#     user_item_matrix_reset = user_item_matrix.rename_axis(None, axis=1).reset_index()\n",
    "#     return user_item_matrix_reset\n",
    "\n",
    "# user_item_preference = create_store_item_click_matrix(transformed_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1  Set up experiment data/features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:45:51,751:train_shape:(8421, 9) \n",
      "2022-04-30 06:45:51,752:test_shape:(1579, 9)  \n"
     ]
    }
   ],
   "source": [
    "cat_features = ['device', 'platform', 'channel','hour_of_day', 'day_of_week']\n",
    "num_features =['click_delta_signup']\n",
    "target=['merchant_id']\n",
    "rnn_features = ['user_id', 'created_at']\n",
    "n2v_features = ['user_id']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# split by date\n",
    "train_data = transformed_data[transformed_data.created_at<='2021-11-07']\n",
    "test_data = transformed_data[transformed_data.created_at>'2021-11-07']\n",
    "\n",
    "\n",
    "train_data = train_data[cat_features+num_features+rnn_features+target]\n",
    "train_data.fillna(0)\n",
    "\n",
    "\n",
    "test_data =  test_data[cat_features +rnn_features+ num_features+target]\n",
    "test_data.fillna(0)\n",
    "\n",
    "\n",
    "training_logger.info('train_shape:{} '.format(train_data.shape))\n",
    "training_logger.info('test_shape:{}  '.format(test_data.shape))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2  Feature Transformation/Scaling pipeline set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up pipeline for classical models\n",
    "categorical_preprocessor = OneHotEncoder(handle_unknown=\"ignore\")\n",
    "numerical_preprocessor = StandardScaler()\n",
    "\n",
    "preprocessor = ColumnTransformer([\n",
    "    ('one-hot-encoder', categorical_preprocessor, cat_features),\n",
    "    ('standard-scaler', numerical_preprocessor, num_features)]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Model Training and Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.1 Load previous models and new models which will be trained and compared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from mlcore.train_eval_helper import *\n",
    "from mlcore.modelops import load_model, save_model, read_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "mldbpath = '../data/mldb.sqlite'\n",
    "deployed_model_info=None\n",
    "deployed_model_obj = None\n",
    "load_prev_model = False\n",
    "from tensorflow import keras\n",
    "\n",
    "if load_prev_model:\n",
    "    try:\n",
    "        deployed_model_info = read_data(mldbpath, 'deployed_model').iloc[0].to_dict()\n",
    "        if deployed_model_info:\n",
    "            deployed_model_name = deployed_model_info['final_model_name']\n",
    "            deployed_model_obj = load_model(deployed_model_name, cur_logger=training_logger) \n",
    "            if deployed_model_obj['type']=='SEQDL':\n",
    "                actual_obj = keras.models.load_model('../models/'+deployed_model_name+'.deep_mdl')\n",
    "            deployed_model_obj['obj'] = actual_obj\n",
    "    except :\n",
    "        training_logger.info('Could not load deployed model it may not exist')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.2 Set up  Classifiers which need to be trained on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set up classfiers\n",
    "\n",
    "classifiers = {\n",
    "        'KNN' : {'obj': KNeighborsClassifier(),\n",
    "             'param_grid':{   \n",
    "                    'leaf_size' : list(range(20,30)),\n",
    "                    'n_neighbors' : list(range(5,30)),\n",
    "                    'p':[1,2]\n",
    "                },\n",
    "             \"type\": 'classical',\n",
    "             'preprocessor' : preprocessor,\n",
    "             \"features\":cat_features+num_features\n",
    "                 \n",
    "                },\n",
    "\n",
    "    'N2V' : {'obj': None,\n",
    "             'param_grid':{   \n",
    "                    'workers':1,\n",
    "                    'dimensions':72,\n",
    "                    'walk_length':18,\n",
    "                    'num_walks':100,\n",
    "                    'window':9,\n",
    "                    'min_count':1,\n",
    "                    'batch_words':5\n",
    "                },\n",
    "             \"type\": \"N2V\",\n",
    "             \"preprocessor\" : None,\n",
    "             \"features\":cat_features+num_features+['user_id'],\n",
    "             \"merchants2vecdict\":{}\n",
    "             \n",
    "            },\n",
    "        'LSTM' : {'obj': None,\n",
    "             'param_grid':{   \n",
    "                    'epochs' :10,\n",
    "                },\n",
    "             \"type\": 'SEQDL',\n",
    "             'preprocessor' : None,\n",
    "             \"features\":rnn_features+target,\n",
    "             \"user_merchant_hist_path\":None,\n",
    "              \n",
    "                 \n",
    "                 \n",
    "                }  \n",
    "    \n",
    "        }\n",
    "\n",
    "# for clf in classifiers:\n",
    "#     classifiers[clf]['preprocessor'] = preprocessor\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#original_data\n",
    "# X_train = train_data[cat_features+num_features]\n",
    "# y_train = train_data[target]\n",
    "# X_test = test_data[cat_features + num_features]\n",
    "# y_test = test_data[target]\n",
    "\n",
    "#X_train = X_train.iloc[0:50000]\n",
    "#y_train = y_train.iloc[0:50000]\n",
    "\n",
    "#X_test_small = X_test.iloc[0:15000]\n",
    "#y_test_small = y_test.iloc[0:15000]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *** Please comment below cell if you want to run on a full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ##shorten_data for demo\n",
    "# train_data = train_data.iloc[0:5000]\n",
    "# test_data = test_data.iloc[0:1000]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.python.client import device_lib\n",
    "# def get_available_devices():\n",
    "#     local_device_protos = device_lib.list_local_devices()\n",
    "#     return [x.name for x in local_device_protos if x.device_type == 'GPU' or x.device_type == 'CPU']\n",
    "# get_available_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>obj</th>\n",
       "      <th>param_grid</th>\n",
       "      <th>type</th>\n",
       "      <th>preprocessor</th>\n",
       "      <th>features</th>\n",
       "      <th>MRR</th>\n",
       "      <th>merchants2vecdict</th>\n",
       "      <th>user_merchant_hist_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN</td>\n",
       "      <td>KNeighborsClassifier()</td>\n",
       "      <td>{'leaf_size': [20, 21, 22, 23, 24, 25, 26, 27,...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[device, platform, channel, hour_of_day, day_o...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N2V</td>\n",
       "      <td>None</td>\n",
       "      <td>{'workers': 1, 'dimensions': 72, 'walk_length'...</td>\n",
       "      <td>N2V</td>\n",
       "      <td>None</td>\n",
       "      <td>[device, platform, channel, hour_of_day, day_o...</td>\n",
       "      <td>0</td>\n",
       "      <td>{}</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>None</td>\n",
       "      <td>{'epochs': 10}</td>\n",
       "      <td>SEQDL</td>\n",
       "      <td>None</td>\n",
       "      <td>[user_id, created_at, merchant_id]</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  index                     obj  \\\n",
       "0   KNN  KNeighborsClassifier()   \n",
       "1   N2V                    None   \n",
       "2  LSTM                    None   \n",
       "\n",
       "                                          param_grid       type  \\\n",
       "0  {'leaf_size': [20, 21, 22, 23, 24, 25, 26, 27,...  classical   \n",
       "1  {'workers': 1, 'dimensions': 72, 'walk_length'...        N2V   \n",
       "2                                     {'epochs': 10}      SEQDL   \n",
       "\n",
       "                                        preprocessor  \\\n",
       "0  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "1                                               None   \n",
       "2                                               None   \n",
       "\n",
       "                                            features  MRR merchants2vecdict  \\\n",
       "0  [device, platform, channel, hour_of_day, day_o...    0               NaN   \n",
       "1  [device, platform, channel, hour_of_day, day_o...    0                {}   \n",
       "2                 [user_id, created_at, merchant_id]    0               NaN   \n",
       "\n",
       "   user_merchant_hist_path  \n",
       "0                      NaN  \n",
       "1                      NaN  \n",
       "2                      NaN  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = {'MRR':0}\n",
    "if deployed_model_obj:\n",
    "    classifiers['deployed_model'] = {\n",
    "        'obj':deployed_model_obj['obj'],\n",
    "        'param_grid':deployed_model_obj['param_grid'],\n",
    "        'features':deployed_model_obj['features'],\n",
    "        'preprocessor':deployed_model_obj['preprocessor'],\n",
    "         'type':deployed_model_obj['type']\n",
    "    }\n",
    "    \n",
    "    if classifiers['deployed_model']['type']=='N2V':\n",
    "        classifiers['deployed_model']['merchants2vecdict'] = deployed_model_obj['merchants2vecdict']\n",
    "        \n",
    "for metric in metrics:\n",
    "    for clf in classifiers:\n",
    "        classifiers[clf][metric]=0\n",
    "\n",
    "get_df_from_dict(classifiers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.3 Train Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:45:53,240:Training started for KNN\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/neighbors/_classification.py:198: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  return self._fit(X, y)\n",
      "2022-04-30 06:45:53,259:Training ended for KNN\n",
      "2022-04-30 06:45:53,260:Training ended for KNN\n",
      "2022-04-30 06:45:53,260:Training started for N2V\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ae2c9e3c07b466b9ec4ae4928152460",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Computing transition probabilities:   0%|          | 0/510 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating walks (CPU: 1): 100%|██████████| 100/100 [00:00<00:00, 433.75it/s]\n",
      "2022-04-30 06:46:11,185:Training ended for N2V\n",
      "2022-04-30 06:46:11,186:Training ended for N2V\n",
      "2022-04-30 06:46:11,187:Training started for LSTM\n",
      "2022-04-30 06:46:11,232:Training ended for LSTM\n",
      "2022-04-30 06:46:11,421:Saved schema user_merchant_hist_20220430__064611 in dataframe with shape (500, 2) at path ../data/user_merchant_hist_20220430__064611.csv\n",
      "2022-04-30 06:46:11,421:loaded user_merchant hist  of size:: (500, 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 26, 200)           2200      \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 26, 100)           120400    \n",
      "                                                                 \n",
      " lstm_1 (LSTM)               (None, 200)               240800    \n",
      "                                                                 \n",
      " dense (Dense)               (None, 11)                2211      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 365,611\n",
      "Trainable params: 365,611\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "47/47 [==============================] - 5s 32ms/step - loss: 2.2807 - accuracy: 0.1457 - val_loss: 2.2854 - val_accuracy: 0.1390\n",
      "Epoch 2/10\n",
      "47/47 [==============================] - 1s 19ms/step - loss: 2.2598 - accuracy: 0.1610 - val_loss: 2.2770 - val_accuracy: 0.1381\n",
      "Epoch 3/10\n",
      "47/47 [==============================] - 1s 19ms/step - loss: 2.2555 - accuracy: 0.1536 - val_loss: 2.2938 - val_accuracy: 0.1333\n",
      "Epoch 4/10\n",
      "47/47 [==============================] - 1s 17ms/step - loss: 2.2491 - accuracy: 0.1696 - val_loss: 2.2700 - val_accuracy: 0.1600\n",
      "Epoch 5/10\n",
      "47/47 [==============================] - 1s 15ms/step - loss: 2.2370 - accuracy: 0.1755 - val_loss: 2.2587 - val_accuracy: 0.1581\n",
      "Epoch 6/10\n",
      "47/47 [==============================] - 1s 12ms/step - loss: 2.2232 - accuracy: 0.1878 - val_loss: 2.2539 - val_accuracy: 0.1762\n",
      "Epoch 7/10\n",
      "47/47 [==============================] - 1s 12ms/step - loss: 2.2158 - accuracy: 0.1878 - val_loss: 2.2504 - val_accuracy: 0.1771\n",
      "Epoch 8/10\n",
      "47/47 [==============================] - 1s 12ms/step - loss: 2.2040 - accuracy: 0.1857 - val_loss: 2.2425 - val_accuracy: 0.1733\n",
      "Epoch 9/10\n",
      "47/47 [==============================] - 1s 12ms/step - loss: 2.1923 - accuracy: 0.1868 - val_loss: 2.2440 - val_accuracy: 0.1705\n",
      "Epoch 10/10\n",
      "47/47 [==============================] - 1s 12ms/step - loss: 2.1800 - accuracy: 0.1876 - val_loss: 2.2300 - val_accuracy: 0.1705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:46:24,513:Training ended for LSTM\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Classifier</th>\n",
       "      <th>obj</th>\n",
       "      <th>param_grid</th>\n",
       "      <th>type</th>\n",
       "      <th>preprocessor</th>\n",
       "      <th>features</th>\n",
       "      <th>MRR</th>\n",
       "      <th>merchants2vecdict</th>\n",
       "      <th>user_merchant_hist_path</th>\n",
       "      <th>train_tokenizer</th>\n",
       "      <th>max_sequence_len</th>\n",
       "      <th>user_merchant_hist_data_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN</td>\n",
       "      <td>KNeighborsClassifier()</td>\n",
       "      <td>{'leaf_size': [20, 21, 22, 23, 24, 25, 26, 27,...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[device, platform, channel, hour_of_day, day_o...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N2V</td>\n",
       "      <td>Word2Vec(vocab=510, vector_size=72, alpha=0.025)</td>\n",
       "      <td>{'workers': 1, 'dimensions': 72, 'walk_length'...</td>\n",
       "      <td>N2V</td>\n",
       "      <td>None</td>\n",
       "      <td>[device, platform, channel, hour_of_day, day_o...</td>\n",
       "      <td>0</td>\n",
       "      <td>{'merchant_1': [-0.2271407, 0.019114045, 0.443...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>&lt;keras.engine.sequential.Sequential object at ...</td>\n",
       "      <td>{'epochs': 10}</td>\n",
       "      <td>SEQDL</td>\n",
       "      <td>None</td>\n",
       "      <td>[user_id, created_at, merchant_id]</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;keras_preprocessing.text.Tokenizer object at ...</td>\n",
       "      <td>27.0</td>\n",
       "      <td>../data/user_merchant_hist_20220430__064611.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Classifier                                                obj  \\\n",
       "0        KNN                             KNeighborsClassifier()   \n",
       "1        N2V   Word2Vec(vocab=510, vector_size=72, alpha=0.025)   \n",
       "2       LSTM  <keras.engine.sequential.Sequential object at ...   \n",
       "\n",
       "                                          param_grid       type  \\\n",
       "0  {'leaf_size': [20, 21, 22, 23, 24, 25, 26, 27,...  classical   \n",
       "1  {'workers': 1, 'dimensions': 72, 'walk_length'...        N2V   \n",
       "2                                     {'epochs': 10}      SEQDL   \n",
       "\n",
       "                                        preprocessor  \\\n",
       "0  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "1                                               None   \n",
       "2                                               None   \n",
       "\n",
       "                                            features  MRR  \\\n",
       "0  [device, platform, channel, hour_of_day, day_o...    0   \n",
       "1  [device, platform, channel, hour_of_day, day_o...    0   \n",
       "2                 [user_id, created_at, merchant_id]    0   \n",
       "\n",
       "                                   merchants2vecdict  user_merchant_hist_path  \\\n",
       "0                                                NaN                      NaN   \n",
       "1  {'merchant_1': [-0.2271407, 0.019114045, 0.443...                      NaN   \n",
       "2                                                NaN                      NaN   \n",
       "\n",
       "                                     train_tokenizer  max_sequence_len  \\\n",
       "0                                                NaN               NaN   \n",
       "1                                                NaN               NaN   \n",
       "2  <keras_preprocessing.text.Tokenizer object at ...              27.0   \n",
       "\n",
       "                      user_merchant_hist_data_path  \n",
       "0                                              NaN  \n",
       "1                                              NaN  \n",
       "2  ../data/user_merchant_hist_20220430__064611.csv  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Train Models (trained model is stored back in dict)\n",
    "use_dask = False\n",
    "\n",
    "if use_dask:\n",
    "    from dask.distributed import Client, progress\n",
    "    dask_client = Client(processes=False, threads_per_worker=2,\n",
    "                    n_workers=1, memory_limit='3GB')\n",
    "else:\n",
    "    dask_client = None\n",
    "\n",
    "#comparison_result_dict = train_models(classifiers, X_train,y_train, training_logger, dask_client)\n",
    "comparison_result_dict = train_models(classifiers, train_data, target, training_logger, dask_client)\n",
    "\n",
    "#comparison_result_dict = train_models(classifiers, X_train,y_train, training_logger)\n",
    "comparison_result = get_df_from_dict(comparison_result_dict, idxname='Classifier')\n",
    "comparison_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.3 Tes/Compare  Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:46:24,535:Testing started for KNN\n",
      "2022-04-30 06:46:25,015:Testing ended for KNN\n",
      "2022-04-30 06:46:25,017:Testing started for N2V\n",
      "/usr/local/lib/python3.8/dist-packages/mlcore/train_eval_helper.py:199: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pred_df[\"order_of_preds\"] = X_test.user_id.apply(\n",
      "2022-04-30 06:46:25,321:Testing ended for N2V\n",
      "2022-04-30 06:46:25,322:Testing started for LSTM\n",
      "2022-04-30 06:46:25,330:Loaded data file ../data/user_merchant_hist_20220430__064611.csv in dataframe with shape (500, 2)\n",
      "/usr/local/lib/python3.8/dist-packages/mlcore/train_eval_helper.py:215: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pred_df[\"y_org\"] = y_test[target]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No history found for 0 users dropping them for MRR compute\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:47:22,975:Testing ended for LSTM\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Classifier</th>\n",
       "      <th>obj</th>\n",
       "      <th>param_grid</th>\n",
       "      <th>type</th>\n",
       "      <th>preprocessor</th>\n",
       "      <th>features</th>\n",
       "      <th>MRR</th>\n",
       "      <th>merchants2vecdict</th>\n",
       "      <th>user_merchant_hist_path</th>\n",
       "      <th>train_tokenizer</th>\n",
       "      <th>max_sequence_len</th>\n",
       "      <th>user_merchant_hist_data_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN</td>\n",
       "      <td>KNeighborsClassifier()</td>\n",
       "      <td>{'leaf_size': [20, 21, 22, 23, 24, 25, 26, 27,...</td>\n",
       "      <td>classical</td>\n",
       "      <td>ColumnTransformer(transformers=[('one-hot-enco...</td>\n",
       "      <td>[device, platform, channel, hour_of_day, day_o...</td>\n",
       "      <td>0.302937</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N2V</td>\n",
       "      <td>Word2Vec(vocab=510, vector_size=72, alpha=0.025)</td>\n",
       "      <td>{'workers': 1, 'dimensions': 72, 'walk_length'...</td>\n",
       "      <td>N2V</td>\n",
       "      <td>None</td>\n",
       "      <td>[device, platform, channel, hour_of_day, day_o...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>{'merchant_1': [-0.2271407, 0.019114045, 0.443...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>&lt;keras.engine.sequential.Sequential object at ...</td>\n",
       "      <td>{'epochs': 10}</td>\n",
       "      <td>SEQDL</td>\n",
       "      <td>None</td>\n",
       "      <td>[user_id, created_at, merchant_id]</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;keras_preprocessing.text.Tokenizer object at ...</td>\n",
       "      <td>27.0</td>\n",
       "      <td>../data/user_merchant_hist_20220430__064611.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Classifier                                                obj  \\\n",
       "0        KNN                             KNeighborsClassifier()   \n",
       "1        N2V   Word2Vec(vocab=510, vector_size=72, alpha=0.025)   \n",
       "2       LSTM  <keras.engine.sequential.Sequential object at ...   \n",
       "\n",
       "                                          param_grid       type  \\\n",
       "0  {'leaf_size': [20, 21, 22, 23, 24, 25, 26, 27,...  classical   \n",
       "1  {'workers': 1, 'dimensions': 72, 'walk_length'...        N2V   \n",
       "2                                     {'epochs': 10}      SEQDL   \n",
       "\n",
       "                                        preprocessor  \\\n",
       "0  ColumnTransformer(transformers=[('one-hot-enco...   \n",
       "1                                               None   \n",
       "2                                               None   \n",
       "\n",
       "                                            features       MRR  \\\n",
       "0  [device, platform, channel, hour_of_day, day_o...  0.302937   \n",
       "1  [device, platform, channel, hour_of_day, day_o...  0.100000   \n",
       "2                 [user_id, created_at, merchant_id]  0.100000   \n",
       "\n",
       "                                   merchants2vecdict  user_merchant_hist_path  \\\n",
       "0                                                NaN                      NaN   \n",
       "1  {'merchant_1': [-0.2271407, 0.019114045, 0.443...                      NaN   \n",
       "2                                                NaN                      NaN   \n",
       "\n",
       "                                     train_tokenizer  max_sequence_len  \\\n",
       "0                                                NaN               NaN   \n",
       "1                                                NaN               NaN   \n",
       "2  <keras_preprocessing.text.Tokenizer object at ...              27.0   \n",
       "\n",
       "                      user_merchant_hist_data_path  \n",
       "0                                              NaN  \n",
       "1                                              NaN  \n",
       "2  ../data/user_merchant_hist_20220430__064611.csv  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute MRR ON TEST DATA ( MRR is computed and stored back in dict)\n",
    "\n",
    "comparison_result_dict = test_models(classifiers, test_data, target, training_logger,dask_client)\n",
    "#comparison_result_dict = test_models(classifiers, X_test,y_test, target)\n",
    "\n",
    "comparison_result = get_df_from_dict(comparison_result_dict, idxname='Classifier')\n",
    "comparison_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:47:22,996:Best performing model on basic of metric MRR is KNN\n"
     ]
    }
   ],
   "source": [
    "#Get best Model Based on a metric\n",
    "metric = 'MRR'\n",
    "best_model_row = get_best_model(comparison_result, metric)\n",
    "best_model = best_model_row['Classifier']\n",
    "best_model_id = classifiers[best_model]['obj']\n",
    "training_logger.info(\"Best performing model on basic of metric {} is {}\".format(metric, best_model))\n",
    "#best_model_row.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # #ovverride best model\n",
    "# model_key_to_deploy = 'LSTM'\n",
    "# best_model_row = comparison_result[comparison_result.Classifier==model_key_to_deploy].iloc[0]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model stored at ../models/KNN_model_04_30_2022_06_45_51.mdl\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_model_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN_model_04_30_2022_06_45_51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                final_model_name\n",
       "0  KNN_model_04_30_2022_06_45_51"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Save/Deploy final_trained_model\n",
    "final_trained_model = best_model_row.to_dict()\n",
    "final_model_name =final_trained_model['Classifier']+'_'+nb_run_id.replace('trng','model')\n",
    "\n",
    "#save model\n",
    "model_type = final_trained_model['type']\n",
    "if model_type=='SEQDL':\n",
    "    final_model_name = 'SEQDL_'+final_model_name\n",
    "save_model(final_trained_model, final_model_name, model_type=model_type)\n",
    "\n",
    "deploy_df = pd.DataFrame([[final_model_name]], \n",
    "                         columns =['final_model_name'])\n",
    "deploy_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_trained_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3.3 Update Model registry/Deploy Best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_dict = {\n",
    "    'deployed_model':deploy_df,\n",
    "    'hist_deployed_models':deploy_df,\n",
    "    #'train_report':comparison_result\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model KNN_model_04_30_2022_06_45_51 deployed \n",
      " associated reports saved in respective tables with id:trng_04_30_2022_06_45_51\n"
     ]
    }
   ],
   "source": [
    "from mlcore.dbhelper import store_data, overwrite_data\n",
    "for dkey in schema_dict:\n",
    "    data_to_be_stored = schema_dict[dkey]\n",
    "    if dkey=='deployed_model':\n",
    "         overwrite_data(data_to_be_stored, mldbpath, dkey)\n",
    "    else:\n",
    "        store_data(data_to_be_stored, mldbpath, dkey)\n",
    "\n",
    "dep_model_name = deploy_df['final_model_name'].iloc[0]\n",
    "print('Model {} deployed \\n associated reports saved in respective tables with id:{}'.format(dep_model_name,nb_run_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-30 06:47:23,157:Traning job with id trng_04_30_2022_06_45_51 finished\n"
     ]
    }
   ],
   "source": [
    "training_logger.info('Traning job with id {} finished'.format(nb_run_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}