examples/ieee_fraud_simulated_covariate_shift_card6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dac7dd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import log_loss, roc_auc_score, accuracy_score\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import cinnamon\n",
    "from cinnamon.drift import ModelDriftExplainer, AdversarialDriftExplainer\n",
    "\n",
    "# config\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "\n",
    "seed = 2021"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52e44e05",
   "metadata": {},
   "source": [
    "# IEEE fraud data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89e02e80",
   "metadata": {},
   "source": [
    "Download data with kaggle CLI if it is setup on your computer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd8d0059",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!kaggle competitions download -c ieee-fraud-detection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45b35398",
   "metadata": {},
   "source": [
    "Else you can download the data here: https://www.kaggle.com/c/ieee-fraud-detection/data, and you will have to accept the competition rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b491e5b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train_transaction.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aab1755d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86f34a8b",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f619582",
   "metadata": {},
   "outputs": [],
   "source": [
    "# count missing values per column\n",
    "missing_values = df.isnull().sum(axis=0)\n",
    "missing_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9cf9f5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# keep only columns with less than 10000 values\n",
    "selected_columns = [col for col in df.columns if missing_values[col] < 10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97537677",
   "metadata": {},
   "outputs": [],
   "source": [
    "# in the resulting columns, drop rows with any missing value\n",
    "df = df[selected_columns].dropna(axis=0, how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0a0f9a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for the variable 'card6', keep only rows corresponding to 'debit' and 'credit'modalities\n",
    "df = df.loc[df['card6'].isin(['debit', 'credit']), :].copy()\n",
    "df['card6'] = (df['card6'] == 'credit') * 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de334813",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52eafff6",
   "metadata": {},
   "source": [
    "# Sampling\n",
    "\n",
    "We replicate a typical production situation where we would have:\n",
    "- training data\n",
    "- test data\n",
    "- production data\n",
    "\n",
    "Also, we introduce so data drift on the variable `card6` by using downsampling. This data drift corresponds to covariate shift."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d39c8a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select features by keeping only numerical features\n",
    "features = [col for col in df.columns if col not in ['TransactionID', 'isFraud', 'TransactionDT',\n",
    "                                                     'ProductCD', 'card4']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66601a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we do a time split (shuffle=False) to seperate between df_temp (train-test data) and df_prod (production data)\n",
    "df_temp, df_prod = train_test_split(df.copy(), test_size=0.25, shuffle=False, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8285f980",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp['card6'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bc4cde1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# in df_temp, we downsample the modality '0' to introduce covariate shift\n",
    "# (distribution before and after sampling are given in cell above and below)\n",
    "np.random.seed(seed)\n",
    "df_temp = df_temp.loc[((np.random.randint(low=0, high=9, size=df_temp.shape[0]) == 0) |\n",
    "                       (df_temp['card6'].values == 1)), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a369630d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp['card6'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d97717e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we do a time split (shuffle=False) to seperate between training data and test data\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_temp[features].copy(),\n",
    "                                                      df_temp['isFraud'].values,\n",
    "                                                      test_size=1/3,\n",
    "                                                      shuffle=False,\n",
    "                                                      random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30dd532d",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_prod, y_prod = df_prod[features], df_prod['isFraud'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1691addc",
   "metadata": {},
   "source": [
    "# Build model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79c9db1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = XGBClassifier(n_estimators=1000,\n",
    "                    booster=\"gbtree\",\n",
    "                    objective=\"binary:logistic\",\n",
    "                    learning_rate=0.2,\n",
    "                    max_depth=6,\n",
    "                    use_label_encoder=False,\n",
    "                    seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bffd4f89",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(X=X_train, y=y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=20,\n",
    "        verbose=10, eval_metric=['auc', 'logloss'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c14e9263",
   "metadata": {},
   "source": [
    "# Detection of data drift\n",
    "\n",
    "We do detect a data drift in this case. Our three indicators:\n",
    "- distribution of predictions\n",
    "- distribution of target labels\n",
    "- performance metrics\n",
    "\n",
    "show a data drift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "772cdb4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize and fit a model drift explainer on test and prod data\n",
    "drift_explainer = ModelDriftExplainer(clf)\n",
    "drift_explainer.fit(X1=X_test, X2=X_prod, y1=y_test, y2=y_prod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b68b066c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "cinnamon.plot_prediction_drift(drift_explainer, bins=100)\n",
    "drift_explainer.get_prediction_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f1aacc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_target_drift(drift_explainer)\n",
    "drift_explainer.get_target_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76734d7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'log_loss test: {log_loss(y_test, clf.predict_proba(X_test))}')\n",
    "print(f'log_loss prod: {log_loss(y_prod, clf.predict_proba(X_prod))}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b76a183f",
   "metadata": {},
   "source": [
    "# Explaination of data drift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "713ca0ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot drift importances in order to identify features that have the higher impacts on data drift\n",
    "cinnamon.plot_tree_based_drift_importances(drift_explainer, type='node_size')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f165f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# first drift value feature is 'card6' : the one we voluntarily add drift to.\n",
    "cinnamon.plot_feature_drift(drift_explainer, 'card6', as_discrete=True)\n",
    "drift_explainer.get_feature_drift('card6')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36e340a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature importance of the model\n",
    "# interestingly, 'card6' feature is quite far in the list of impartant feature in the model\n",
    "pd.DataFrame(clf.feature_importances_, X_train.columns).sort_values(0, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f72d8046",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_feature_drift(drift_explainer, 'C5')\n",
    "drift_explainer.get_feature_drift('C5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f720d908",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_feature_drift(drift_explainer, 'TransactionAmt')\n",
    "drift_explainer.get_feature_drift('TransactionAmt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dc33a09",
   "metadata": {},
   "source": [
    "# Correction of data drift"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45b56fc5",
   "metadata": {},
   "source": [
    "## Correction on test dataset\n",
    "\n",
    "We apply our methodology which uses adversarial learning to correct data drift between test and prod data.\n",
    "\n",
    "We then check our three indicators of  data drift in order to see if we get improvement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "692ebce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# weights computed with the adversarial method\n",
    "# feature_subset=['card6']: only the first feature in terms of drift value is selected here\n",
    "sample_weights_test_adversarial = (AdversarialDriftExplainer(feature_subset=['card6'], seed=2021)\n",
    "                                    .fit(X_test, X_prod)\n",
    "                                    .get_adversarial_correction_weights(max_ratio=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a23a22f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# study the drift with the new weights on test data\n",
    "drift_explainer2 = ModelDriftExplainer(clf)\n",
    "drift_explainer2.fit(X1=X_test, X2=X_prod, y1=y_test, y2=y_prod,\n",
    "                     sample_weights1=sample_weights_test_adversarial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a24f94ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# with new weigts on test data, distributions of predictions are much closer\n",
    "cinnamon.plot_prediction_drift(drift_explainer2, bins=100)\n",
    "drift_explainer2.get_prediction_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f04e7fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# distribution of target if also corrected\n",
    "cinnamon.plot_target_drift(drift_explainer2)\n",
    "drift_explainer2.get_target_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f838f9a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we now replicates much better what happens in production (also in terms of log loss)\n",
    "print(f'log_loss test: {log_loss(y_test, clf.predict_proba(X_test), sample_weight=sample_weights_test_adversarial)}')\n",
    "print(f'log_loss prod: {log_loss(y_prod, clf.predict_proba(X_prod))}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a7561bb",
   "metadata": {},
   "source": [
    "## Correction on validation dataset and train dataset (in order to retrain the model)\n",
    "\n",
    "We apply the same adversarial strategy on training data.\n",
    "\n",
    "With the new weights, we observe the model trained on re-weighted does not seem to perform better than the model trained on non-weighted data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d759e11",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# weights computed with the adversarial method on training data\n",
    "sample_weights_train_adversarial = (AdversarialDriftExplainer(feature_subset=['card6'], seed=2021, verbosity=False)\n",
    "                                    .fit(X_train, X_prod)\n",
    "                                    .get_adversarial_correction_weights(max_ratio=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f88e4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = XGBClassifier(n_estimators=1000,\n",
    "                    booster=\"gbtree\",\n",
    "                    objective=\"binary:logistic\",\n",
    "                    learning_rate=0.2,\n",
    "                    max_depth=5,\n",
    "                    use_label_encoder=False,\n",
    "                    seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f51d194",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train a new classifier with the reweighted samples\n",
    "clf2.fit(X=X_train, y=y_train, eval_set=[(X_test, y_test)], sample_weight=sample_weights_train_adversarial,\n",
    "         early_stopping_rounds=20, verbose=10, eval_metric=['auc', 'logloss'],\n",
    "         sample_weight_eval_set=[sample_weights_test_adversarial])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd84f943",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we see no improvement on prod loss when we train with both train and test datasets reweighted\n",
    "print(f'log_loss test: {log_loss(y_test, clf2.predict_proba(X_test), sample_weight=sample_weights_test_adversarial)}')\n",
    "print(f'log_loss prod: {log_loss(y_prod, clf2.predict_proba(X_prod))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12ae1ff1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}