501-EVAL-1dCNN.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.10 64-bit ('ml': conda)"
  },
  "interpreter": {
   "hash": "8585eeb1b9bd760897ad2b22a667cc2885f7735477e4a7ab88df4f229e63fedd"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import glob\n",
    "import os, gc\n",
    "import numpy as numpy\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import datatable as dt\n",
    "from collections import defaultdict\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import r2_score\n",
    "from numba import njit\n",
    "from utils import *\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n",
    "# TF\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.callbacks import Callback, ReduceLROnPlateau, ModelCheckpoint, EarlyStopping"
   ],
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {}
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "N_FOLD = 10\n",
    "N_MINS = 5\n",
    "MIN_SIZE = 600 // N_MINS\n",
    "\n",
    "SOL_NAME = '501-1dCNN'\n",
    "DATA_NAME = '501'\n",
    "mkdir(f'./models/{SOL_NAME}/')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "# CONSTANT\n",
    "MEAN = -5.762330803300896\n",
    "STD = 0.6339307835941186\n",
    "EPS = 1e-9"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "# get ids\n",
    "list_stock_id = get_stock_id()\n",
    "list_time_id = get_time_id()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Functions"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "def transform_target(target):\n",
    "    return (np.log(target + EPS) - MEAN) / STD\n",
    "\n",
    "def inverse_target(target):\n",
    "    return np.exp(MEAN + STD * target) - EPS\n",
    "\n",
    "def np_rmspe(y_true, y_pred):\n",
    "    y_true = inverse_target(y_true)\n",
    "    y_pred = inverse_target(y_pred)\n",
    "    return np.sqrt(np.mean(np.square((y_true - y_pred) / y_true)))\n",
    "\n",
    "def mspe_loss(y_true, y_pred):\n",
    "    y_true = K.exp(MEAN + STD * y_true) - EPS\n",
    "    y_pred = K.exp(MEAN + STD * y_pred) - EPS\n",
    "    return K.sqrt(K.mean(K.square((y_true - y_pred) / y_true)))\n",
    "\n",
    "def rmspe_keras(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square((y_true - y_pred) / y_true)))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "def create_1dcnn(num_columns, num_labels, learning_rate):\n",
    "    # input\n",
    "    inp = tf.keras.layers.Input(shape=(num_columns,))\n",
    "    x = tf.keras.layers.BatchNormalization()(inp)\n",
    "    # 1dcnn\n",
    "    x = tf.keras.layers.Dense(256, activation='relu')(x)\n",
    "    x = tf.keras.layers.Reshape((16, 16))(x)\n",
    "    x = tf.keras.layers.Conv1D(filters=12,\n",
    "                      kernel_size=2,\n",
    "                      strides=1,\n",
    "                      activation='swish')(x)\n",
    "    x = tf.keras.layers.MaxPooling1D(pool_size=2)(x)\n",
    "    x = tf.keras.layers.Flatten()(x)\n",
    "    \n",
    "    # ffn\n",
    "    for i in range(3):\n",
    "        x = tf.keras.layers.Dense(64 // (2 ** i), activation='swish')(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.GaussianNoise(0.01)(x)\n",
    "        x = tf.keras.layers.Dropout(0.20)(x)\n",
    "    x = tf.keras.layers.Dense(num_labels)(x)\n",
    "    \n",
    "    model = tf.keras.models.Model(inputs=inp, outputs=x)\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
    "        loss=mspe_loss,\n",
    "    )\n",
    "    return model"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Loading data"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "# train\n",
    "df_train = dt.fread(f'./dataset/train_{DATA_NAME}_NN.csv').to_pandas()\n",
    "fea_cols = [f for f in df_train.columns if f.startswith('B_') or f.startswith('T_') or f.startswith('Z_')]\n",
    "# result\n",
    "df_result = dt.fread('./dataset/train.csv').to_pandas()\n",
    "df_result = gen_row_id(df_result)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "fea_cols_TA = [f for f in fea_cols if 'min_' not in f]\n",
    "df_time_mean = df_train.groupby('time_id')[fea_cols_TA].mean()\n",
    "df_time_mean.columns = [f'{c}_TA_mean' for c in df_time_mean.columns]\n",
    "df_time_mean = df_time_mean.reset_index()\n",
    "df_train = df_train.merge(df_time_mean, on='time_id', how='left')\n",
    "\n",
    "del df_time_mean\n",
    "gc.collect()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "df_train['target'] = transform_target(df_train['target'])\n",
    "df_train = gen_row_id(df_train)\n",
    "df_train = add_time_fold(df_train, N_FOLD)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Evaluation"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "def add_time_stats(df_train):\n",
    "    time_cols = [f for f in df_train.columns if f.endswith('_time')]\n",
    "    df_gp_stock = df_train.groupby('stock_id')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].mean().reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_mean' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].std().reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_std' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].skew().reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_skew' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].min().reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_min' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].max().reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_max' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].quantile(0.25).reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_q1' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].quantile(0.50).reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_q2' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    #\n",
    "    df_stats = df_gp_stock[time_cols].quantile(0.75).reset_index()\n",
    "    df_stats.columns = ['stock_id'] + [f'{f}_q3' for f in time_cols]\n",
    "    df_train = df_train.merge(df_stats, on=['stock_id'], how='left')\n",
    "    return df_train"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "batch_size = 1024\n",
    "learning_rate = 6e-3\n",
    "epochs = 1000\n",
    "\n",
    "list_seeds = [0, 11, 42, 777, 2045]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "list_rmspe = []\n",
    "for i_seed, seed in enumerate(list_seeds):\n",
    "    df_train = add_time_fold(df_train, N_FOLD, seed=seed)\n",
    "    list_rmspe += [[]]\n",
    "    for i_fold in range(N_FOLD):\n",
    "        gc.collect()\n",
    "        df_tr = df_train.loc[df_train.fold!=i_fold]\n",
    "        df_te = df_train.loc[df_train.fold==i_fold]\n",
    "        df_tr = add_time_stats(df_tr)\n",
    "        df_te = add_time_stats(df_te)\n",
    "\n",
    "        fea_cols = [f for f in df_tr if f.startswith('B_') or f.startswith('T_') or f.startswith('Z_')]\n",
    "\n",
    "        X_train = df_tr[fea_cols].values\n",
    "        y_train = df_tr[['target']].values\n",
    "        X_test = df_te[fea_cols].values\n",
    "        y_test = df_te[['target']].values\n",
    "        idx_test = df_train.loc[df_train.fold==i_fold].index\n",
    "        print(f'Fold {i_seed+1}/{len(list_seeds)} | {i_fold+1}/{N_FOLD}', X_train.shape, X_test.shape)\n",
    "\n",
    "\n",
    "        # Callbacks\n",
    "        ckp_path = f'./models/{SOL_NAME}/model_{i_seed}_{i_fold}.hdf5'\n",
    "        rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_delta=1e-5, verbose=2)\n",
    "        es = EarlyStopping(monitor='val_loss', min_delta=1e-5, patience=31, restore_best_weights=True, verbose=2)\n",
    "\n",
    "        model = create_1dcnn(X_train.shape[1], 1, learning_rate)\n",
    "        history = model.fit(X_train, y_train,\n",
    "            epochs=epochs,\n",
    "            validation_data=(X_test, y_test),\n",
    "            validation_batch_size=len(y_test),\n",
    "            batch_size=batch_size,\n",
    "            verbose=2,\n",
    "            callbacks=[rlr, es]\n",
    "        ) \n",
    "        # model = tf.keras.models.load_model(ckp_path, custom_objects={'mspe_loss': mspe_loss})\n",
    "\n",
    "        y_pred = model.predict(X_test, batch_size=len(y_test))\n",
    "        curr_rmspe = np_rmspe(y_test, y_pred)\n",
    "        list_rmspe[-1] += [curr_rmspe]\n",
    "        model.save(ckp_path)\n",
    "        # generate and save preds\n",
    "        df_result.loc[idx_test, f'pred_{i_seed}'] = inverse_target(y_pred)\n",
    "        clear_output()\n",
    "        print(list_rmspe)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[0.21022636450052495, 0.21939386684082246, 0.20975052519321782, 0.2085773537791255, 0.2113484591068515, 0.2119121523184086, 0.21110068228865625, 0.21208264595637888, 0.21416047509182187, 0.22773273338705263], [0.22096742507383604, 0.20646272827577072, 0.2115436938575406, 0.21968584354115303, 0.22984816197720548, 0.21101167408065497, 0.21269305781660341, 0.21244591125991594, 0.20925710000483455, 0.2111199499437056], [0.2131945100539358, 0.2154680645073522, 0.21478817587600602, 0.21147221223707088, 0.20577207180073784, 0.22450232155237212, 0.22015399685845816, 0.2091533701698531, 0.2094901784494245, 0.21914447248675742], [0.21141307303191084, 0.20700794105401787, 0.21007405037706894, 0.2241200798910265, 0.21281092406777036, 0.2131235356880075, 0.21761252555658234, 0.20780915023404992, 0.2187906140839999, 0.22313538070705846], [0.21206174502296368, 0.20933323728227768, 0.21184183928517433, 0.21033346929252794, 0.21737091357149843, 0.21735295651157147, 0.21245263600520106, 0.21799232179000397, 0.21017008566661857, 0.2234358511351555]]\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "df_result.to_csv(f'./results/{SOL_NAME}.csv', index=False)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "source": [
    "for i in range(len(list_seeds)):\n",
    "    print(i, rmspe(df_result['target'], df_result[f'pred_{i}']))\n",
    "print('All: ', rmspe(df_result['target'], df_result[[f'pred_{i}' for i in range(len(list_seeds))]].mean(axis=1)))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0 0.21369911137011632\n",
      "1 0.21460543478949706\n",
      "2 0.21438267580899686\n",
      "3 0.2146666805788994\n",
      "4 0.21427824604107953\n",
      "All:  0.21010537596822407\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ]
}