training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Melanoma Classification.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zFudWpcPMpFZ",
        "colab_type": "text"
      },
      "source": [
        "#Dependencies and Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lu_aGWsJKR-M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install efficientnet_pytorch torchtoolbox"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zBxwjCB6KYMd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torch.nn.functional as F\n",
        "import torch.nn as nn\n",
        "import torchtoolbox.transform as transforms\n",
        "from torch.utils.data import Dataset, DataLoader, Subset\n",
        "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
        "from sklearn.metrics import accuracy_score, roc_auc_score\n",
        "from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import gc\n",
        "import os\n",
        "import cv2\n",
        "import time\n",
        "import datetime\n",
        "import warnings\n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from efficientnet_pytorch import EfficientNet\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChqyIBKmK0dU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "warnings.simplefilter('ignore')\n",
        "def seed_everything(seed):\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = True\n",
        "\n",
        "seed_everything(47)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WqLIIGvdK53E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FDybB8ldMyAn",
        "colab_type": "text"
      },
      "source": [
        "#Datatset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8UwxB8MTK7o9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MelanomaDataset(Dataset):\n",
        "    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):\n",
        "        \"\"\"\n",
        "        Class initialization\n",
        "        Args:\n",
        "            df (pd.DataFrame): DataFrame with data description\n",
        "            imfolder (str): folder with images\n",
        "            train (bool): flag of whether a training dataset is being initialized or testing one\n",
        "            transforms: image transformation method to be applied\n",
        "            meta_features (list): list of features with meta information, such as sex and age\n",
        "            \n",
        "        \"\"\"\n",
        "        self.df = df\n",
        "        self.imfolder = imfolder\n",
        "        self.transforms = transforms\n",
        "        self.train = train\n",
        "        self.meta_features = meta_features\n",
        "        \n",
        "    def __getitem__(self, index):\n",
        "        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n",
        "        x = cv2.imread(im_path)\n",
        "        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n",
        "\n",
        "        if self.transforms:\n",
        "            x = self.transforms(x)\n",
        "            \n",
        "        if self.train:\n",
        "            y = self.df.iloc[index]['target']\n",
        "            return (x, meta), y\n",
        "        else:\n",
        "            return (x, meta)\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.df)\n",
        "    \n",
        "    \n",
        "class Net(nn.Module):\n",
        "    def __init__(self, arch, n_meta_features: int):\n",
        "        super(Net, self).__init__()\n",
        "        self.arch = arch\n",
        "        if 'ResNet' in str(arch.__class__):\n",
        "            self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n",
        "        if 'EfficientNet' in str(arch.__class__):\n",
        "            self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)\n",
        "        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n",
        "                                  nn.BatchNorm1d(500),\n",
        "                                  nn.ReLU(),\n",
        "                                  nn.Dropout(p=0.2),\n",
        "                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n",
        "                                  nn.BatchNorm1d(250),\n",
        "                                  nn.ReLU(),\n",
        "                                  nn.Dropout(p=0.2))\n",
        "        self.ouput = nn.Linear(500 + 250, 1)\n",
        "        \n",
        "    def forward(self, inputs):\n",
        "        \"\"\"\n",
        "        No sigmoid in forward because we are going to use BCEWithLogitsLoss\n",
        "        Which applies sigmoid for us when calculating a loss\n",
        "        \"\"\"\n",
        "        x, meta = inputs\n",
        "        cnn_features = self.arch(x)\n",
        "        meta_features = self.meta(meta)\n",
        "        features = torch.cat((cnn_features, meta_features), dim=1)\n",
        "        output = self.ouput(features)\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SphY1nlEM1eW",
        "colab_type": "text"
      },
      "source": [
        "#Augmentations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orKBu23YK_M9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class AdvancedHairAugmentation:\n",
        "    \"\"\"\n",
        "    Impose an image of a hair to the target image\n",
        "\n",
        "    Args:\n",
        "        hairs (int): maximum number of hairs to impose\n",
        "        hairs_folder (str): path to the folder with hairs images\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n",
        "        self.hairs = hairs\n",
        "        self.hairs_folder = hairs_folder\n",
        "\n",
        "    def __call__(self, img):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            img (PIL Image): Image to draw hairs on.\n",
        "\n",
        "        Returns:\n",
        "            PIL Image: Image with drawn hairs.\n",
        "        \"\"\"\n",
        "        n_hairs = random.randint(0, self.hairs)\n",
        "        \n",
        "        if not n_hairs:\n",
        "            return img\n",
        "        \n",
        "        height, width, _ = img.shape  # target image width and height\n",
        "        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n",
        "        \n",
        "        for _ in range(n_hairs):\n",
        "            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n",
        "            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n",
        "            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n",
        "\n",
        "            h_height, h_width, _ = hair.shape  # hair image width and height\n",
        "            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n",
        "            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n",
        "            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n",
        "\n",
        "            # Creating a mask and inverse mask\n",
        "            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n",
        "            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n",
        "            mask_inv = cv2.bitwise_not(mask)\n",
        "\n",
        "            # Now black-out the area of hair in ROI\n",
        "            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n",
        "\n",
        "            # Take only region of hair from hair image.\n",
        "            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n",
        "\n",
        "            # Put hair in ROI and modify the target image\n",
        "            dst = cv2.add(img_bg, hair_fg)\n",
        "\n",
        "            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n",
        "                \n",
        "        return img\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06xojr__LEi-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DrawHair:\n",
        "    \"\"\"\n",
        "    Draw a random number of pseudo hairs\n",
        "\n",
        "    Args:\n",
        "        hairs (int): maximum number of hairs to draw\n",
        "        width (tuple): possible width of the hair in pixels\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, hairs:int = 4, width:tuple = (1, 2)):\n",
        "        self.hairs = hairs\n",
        "        self.width = width\n",
        "\n",
        "    def __call__(self, img):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            img (PIL Image): Image to draw hairs on.\n",
        "\n",
        "        Returns:\n",
        "            PIL Image: Image with drawn hairs.\n",
        "        \"\"\"\n",
        "        if not self.hairs:\n",
        "            return img\n",
        "        \n",
        "        width, height, _ = img.shape\n",
        "        \n",
        "        for _ in range(random.randint(0, self.hairs)):\n",
        "            # The origin point of the line will always be at the top half of the image\n",
        "            origin = (random.randint(0, width), random.randint(0, height // 2))\n",
        "            # The end of the line \n",
        "            end = (random.randint(0, width), random.randint(0, height))\n",
        "            color = (0, 0, 0)  # color of the hair. Black.\n",
        "            cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))\n",
        "        \n",
        "        return img\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WFRnVXcKLHRt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Microscope:\n",
        "    \"\"\"\n",
        "    Cutting out the edges around the center circle of the image\n",
        "    Imitating a picture, taken through the microscope\n",
        "\n",
        "    Args:\n",
        "        p (float): probability of applying an augmentation\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, p: float = 0.5):\n",
        "        self.p = p\n",
        "\n",
        "    def __call__(self, img):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            img (PIL Image): Image to apply transformation to.\n",
        "\n",
        "        Returns:\n",
        "            PIL Image: Image with transformation.\n",
        "        \"\"\"\n",
        "        if random.random() < self.p:\n",
        "            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n",
        "                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n",
        "                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n",
        "                        (0, 0, 0), # color\n",
        "                        -1)\n",
        "\n",
        "            mask = circle - 255\n",
        "            img = np.multiply(img, mask)\n",
        "        \n",
        "        return img\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}(p={self.p})'\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVssFkrxLJcO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_transform = transforms.Compose([\n",
        "    AdvancedHairAugmentation(hairs_folder='./hair'),\n",
        "    transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.RandomVerticalFlip(),\n",
        "    Microscope(p=0.5),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n",
        "])\n",
        "test_transform = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "llHolCmgNJwu",
        "colab_type": "text"
      },
      "source": [
        "#MetaData Features\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pu3EHT-xLOR0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_df = pd.read_csv('./train.csv')\n",
        "test_df = pd.read_csv('./test.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MrzTKN3zLRM-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Using triple stratified KFolds\n",
        "tmp = pd.read_csv('./train.csv')\n",
        "train_df['fold'] = tmp['tfrecord']\n",
        "del tmp\n",
        "gc.collect();"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ORqVM10BLTFU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# One-hot encoding of anatom_site_general_challenge feature\n",
        "concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\n",
        "dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\n",
        "train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\n",
        "test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n",
        "\n",
        "# Sex features\n",
        "train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\n",
        "test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\n",
        "train_df['sex'] = train_df['sex'].fillna(-1)\n",
        "test_df['sex'] = test_df['sex'].fillna(-1)\n",
        "\n",
        "# Age features\n",
        "train_df['age_approx'] /= train_df['age_approx'].max()\n",
        "test_df['age_approx'] /= test_df['age_approx'].max()\n",
        "train_df['age_approx'] = train_df['age_approx'].fillna(0)\n",
        "test_df['age_approx'] = test_df['age_approx'].fillna(0)\n",
        "\n",
        "train_df['patient_id'] = train_df['patient_id'].fillna(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CUBpPy1TLVXN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\n",
        "meta_features.remove('anatom_site_general_challenge')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WLXVPMHvM5MG",
        "colab_type": "text"
      },
      "source": [
        "#Model Architechture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ykCYY4wLLdc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "arch = EfficientNet.from_pretrained('efficientnet-b1')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fPN8rmMvLW5l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test = MelanomaDataset(df=test_df,\n",
        "                       imfolder='./jpeg-melanoma-256x256/test/', \n",
        "                       train=False,\n",
        "                       transforms=train_transform,  # For TTA\n",
        "                       meta_features=meta_features)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nFOXW60zNWAQ",
        "colab_type": "text"
      },
      "source": [
        "#Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JRUzm6kYLYhH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "epochs = 15  # Number of epochs to run\n",
        "es_patience = 3  # Early Stopping patience - for how many epochs with no improvements to wait\n",
        "TTA = 3 # Test Time Augmentation rounds\n",
        "\n",
        "oof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\n",
        "preds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)  # Predictions for test test\n",
        "\n",
        "skf = KFold(n_splits=5, shuffle=True, random_state=47)\n",
        "for fold,(idxT, idxV) in enumerate(skf.split(np.arange(15)), 1):\n",
        "    print('=' * 20, 'Fold', fold, '=' * 20)\n",
        "    \n",
        "    train_idx = train_df.loc[train_df['fold'].isin(idxT)].index\n",
        "    val_idx = train_df.loc[train_df['fold'].isin(idxV)].index\n",
        "    \n",
        "    \n",
        "    model_path = f'model_{fold}.pth'  # Path and filename to save model to\n",
        "    best_val = 0  # Best validation score within this fold\n",
        "    patience = es_patience  # Current patience counter\n",
        "    arch = EfficientNet.from_pretrained('efficientnet-b1')\n",
        "    model = Net(arch=arch, n_meta_features=len(meta_features))  # New model for each fold\n",
        "    model = model.to(device)\n",
        "    \n",
        "    \n",
        "    optim = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "    scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\n",
        "    criterion = nn.BCEWithLogitsLoss()\n",
        "    \n",
        "    train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n",
        "                            imfolder='./train/', \n",
        "                            train=True, \n",
        "                            transforms=train_transform,\n",
        "                            meta_features=meta_features)\n",
        "    val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n",
        "                            imfolder='./train/', \n",
        "                            train=True, \n",
        "                            transforms=test_transform,\n",
        "                            meta_features=meta_features)\n",
        "    \n",
        "    train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)\n",
        "    val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)\n",
        "    test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)\n",
        "    \n",
        "    for epoch in range(epochs):\n",
        "        start_time = time.time()\n",
        "        correct = 0\n",
        "        epoch_loss = 0\n",
        "        model.train()\n",
        "        \n",
        "        for x, y in train_loader:\n",
        "            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n",
        "            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n",
        "            y = torch.tensor(y, device=device, dtype=torch.float32)\n",
        "            optim.zero_grad()\n",
        "            z = model(x)\n",
        "            loss = criterion(z, y.unsqueeze(1))\n",
        "            loss.backward()\n",
        "            optim.step()\n",
        "            pred = torch.round(torch.sigmoid(z))  # round off sigmoid to obtain predictions\n",
        "            correct += (pred.cpu() == y.cpu().unsqueeze(1)).sum().item()  # tracking number of correctly predicted samples\n",
        "            epoch_loss += loss.item()\n",
        "        train_acc = correct / len(train_idx)\n",
        "        \n",
        "        model.eval()  # switch model to the evaluation mode\n",
        "        val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n",
        "        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n",
        "            # Predicting on validation set\n",
        "            for j, (x_val, y_val) in enumerate(val_loader):\n",
        "                x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n",
        "                x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n",
        "                y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n",
        "                z_val = model(x_val)\n",
        "                val_pred = torch.sigmoid(z_val)\n",
        "                val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n",
        "            val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))\n",
        "            val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())\n",
        "            \n",
        "            print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(\n",
        "            epoch + 1, \n",
        "            epoch_loss, \n",
        "            train_acc, \n",
        "            val_acc, \n",
        "            val_roc, \n",
        "            str(datetime.timedelta(seconds=time.time() - start_time))[:7]))\n",
        "            \n",
        "            scheduler.step(val_roc)\n",
        "                \n",
        "            if val_roc >= best_val:\n",
        "                best_val = val_roc\n",
        "                patience = es_patience  # Resetting patience since we have new best validation accuracy\n",
        "                torch.save(model, model_path)  # Saving current best model\n",
        "            else:\n",
        "                patience -= 1\n",
        "                if patience == 0:\n",
        "                    print('Early stopping. Best Val roc_auc: {:.3f}'.format(best_val))\n",
        "                    break\n",
        "                \n",
        "    model = torch.load(model_path)  # Loading best model of this fold\n",
        "    model.eval()  # switch model to the evaluation mode\n",
        "    val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n",
        "    with torch.no_grad():\n",
        "        # Predicting on validation set once again to obtain data for OOF\n",
        "        for j, (x_val, y_val) in enumerate(val_loader):\n",
        "            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n",
        "            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n",
        "            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n",
        "            z_val = model(x_val)\n",
        "            val_pred = torch.sigmoid(z_val)\n",
        "            val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n",
        "        oof[val_idx] = val_preds.cpu().numpy()\n",
        "        \n",
        "        # Predicting on test set\n",
        "        for _ in range(TTA):\n",
        "            for i, x_test in enumerate(test_loader):\n",
        "                x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)\n",
        "                x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)\n",
        "                z_test = model(x_test)\n",
        "                z_test = torch.sigmoid(z_test)\n",
        "                preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test\n",
        "        preds /= TTA\n",
        "        \n",
        "    del train, val, train_loader, val_loader, x, y, x_val, y_val\n",
        "    gc.collect()\n",
        "    \n",
        "preds /= skf.n_splits"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "olUFt_jmNaU1",
        "colab_type": "text"
      },
      "source": [
        "#Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KLiCIwbbLbbu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "my1IWAc4LeaN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sns.kdeplot(pd.Series(preds.cpu().numpy().reshape(-1,)));"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjCMgIPdLggG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Saving OOF predictions so stacking would be easier\n",
        "pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4qqW8jcLiH-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sub = pd.read_csv('./sample_submission.csv')\n",
        "sub['target'] = preds.cpu().numpy().reshape(-1,)\n",
        "sub.to_csv('submission.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}