Notebooks/Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EPEBQBTzcZiS"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import PIL\n",
    "import sys\n",
    "import sls\n",
    "import cv2\n",
    "import json\n",
    "import time\n",
    "import glob\n",
    "import math\n",
    "import timm\n",
    "import copy\n",
    "import torch\n",
    "import pickle\n",
    "import geffnet\n",
    "import logging\n",
    "import fnmatch\n",
    "import argparse\n",
    "import itertools\n",
    "import torchvision\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import torch.nn as nn\n",
    "from PIL import Image\n",
    "from glob import glob\n",
    "from pathlib import Path\n",
    "from copy import deepcopy\n",
    "from sklearn import metrics\n",
    "import torch.optim as optim\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as data\n",
    "from torchvision import transforms\n",
    "from torch.autograd import Variable\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from torch.optim import lr_scheduler\n",
    "import torch.utils.model_zoo as model_zoo\n",
    "from timm.models.layers.activations import *\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "from efficientnet_pytorch import EfficientNet\n",
    "from collections import OrderedDict, defaultdict\n",
    "from torchvision import transforms, models, datasets\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "from sklearn.metrics import classification_report, confusion_matrix,accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "CA9wZQTds_Av",
    "outputId": "e34e2986-0c5f-4723-fe0a-60e01320e951"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['COVID-19', 'normal', 'pneumonia']\n",
      "{'data': 15000}\n",
      "cuda:0\n",
      "{0: 'COVID-19', 1: 'normal', 2: 'pneumonia'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 3, 224, 224])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = '/home/linh/Downloads/Covid-19'\n",
    "\n",
    "valid_dir = data_dir + \"/data\"\n",
    "\n",
    "# Define your transforms for the training and testing sets\n",
    "data_transforms = {  \n",
    "    'data':  transforms.Compose([\n",
    "             #transforms.RandomRotation(30),\n",
    "             transforms.Resize(256),\n",
    "             #transforms.RandomResizedCrop(224),        \n",
    "             transforms.CenterCrop(224),\n",
    "             #transforms.RandomHorizontalFlip(),\n",
    "             transforms.ToTensor(),\n",
    "             transforms.Normalize([0.485, 0.456, 0.406], \n",
    "                             [0.229, 0.224, 0.225])\n",
    "                                ])\n",
    "\n",
    "                    }\n",
    "\n",
    "# Load the datasets with ImageFolder\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n",
    "                                          data_transforms[x])\n",
    "                  for x in ['data']}\n",
    "\n",
    "batch_size = 4\n",
    "data_loader = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size,\n",
    "                                            shuffle=True, num_workers=4)\n",
    "              for x in ['data']}\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['data']}\n",
    "\n",
    "class_names = image_datasets['data'].classes\n",
    "print(class_names)\n",
    "print(dataset_sizes)\n",
    "print(device)\n",
    "\n",
    "\n",
    "### we get the class_to_index in the data_Set but what we really need is the cat_to_names  so we will create\n",
    "_ = image_datasets['data'].class_to_idx\n",
    "cat_to_name = {_[i]: i for i in list(_.keys())}\n",
    "print(cat_to_name)\n",
    "    \n",
    "# Run this to test the data loader\n",
    "images, labels = next(iter(data_loader['data']))\n",
    "images.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EfficientNet_B0 with image size=224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction time for complete in 8226 milisecond\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0   0.000000  0.000000  0.000000        10\n",
      "           1   0.942794  0.968362  0.955407       885\n",
      "           2   0.939655  0.917508  0.928450       594\n",
      "\n",
      "    accuracy                       0.941572      1489\n",
      "   macro avg   0.627483  0.628623  0.627952      1489\n",
      "weighted avg   0.935210  0.941572  0.938237      1489\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/linh/.conda/envs/CV/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAHwCAYAAABHf7LhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de7h253wn8O9PIlToG422jm1KE1U9CkW0QVRKqxqkU72GpjrqTBGu6TjTanXEOYZWVRw6EyaKcQoqSYNoVYrQqiCCoK1DEnIQTfY9fzxrs7Oz95tn7/2sd9/vXp+P67lWnnW8H1788s1v3Xe11gIAAFNyje0eAAAA7GmKYAAAJkcRDADA5CiCAQCYHEUwAACTowgGAGByFMEAAEyOIhgAgMlRBAMAMDmKYAAAJkcRDADA5CiCAQCYHEUwAACTs+92D2Ar9t3vJm27xwDAzDWqtnsIsBDfuey8Lv4w/+fXzll4nXPNG9y8i9/WA0kwAACTs1cnwQAAO9bSFds9gh1NEgwAwORIggEAetSWtnsEO5oiGACgR0uK4DFphwAAYHIkwQAAHWraIUYlCQYAYHIkwQAAPdITPCpFMABAj7RDjEo7BAAAkyMJBgDokRXjRiUJBgBgciTBAAA90hM8KkkwAACTIwkGAOiRKdJGpQgGAOiQFePGpR0CAIDJkQQDAPRIO8SoJMEAAEyOJBgAoEd6gkelCAYA6JEV40alHQIAgMmRBAMA9Eg7xKgkwQAATI4kGACgR6ZIG5UiGACgR9ohRqUdAgCAyZEEAwD0SDvEqCTBAABMjiQYAKBDrVksY0yKYACAHnkxblTaIQAAmBxJMABAj7wYNypJMAAAkyMJBgDokZ7gUUmCAQCYHEkwAECPlkyRNiZFMABAj7RDjEo7BAAAkyMJBgDokSnSRiUJBgBgciTBAAA90hM8KkUwAECPtEOMSjsEAACTIwkGAOiRJHhUkmAAACZHEQwA0KHWrlj4Z6uq6nerql3N54oV5x90NeeeuJtnHVNVH6qqi6rqwqo6raruteUfMdAOAQDQoz7bIT6a5JnrHPulJEckeecaxz6W5M1r7P/EWjeqquOSHJvkvCSvSLJfkvsneWtVPbq1dvwGx30VimAAAObSWvtoZoXwVVTVB4e//Is1Dn+0tfaMeZ5RVYdlVgB/NsntWmvnD/ufm+TMJMdV1dtaa+dubPRXph0CAKBHbWnxn5FU1U8nuUOSLyV5+xZv97Bh++zlAjhJhqL3pUmuleRBW3yGIhgAgC17yLB9ZVu7+fjGVfXQqnrSsP2Z3dzriGF78hrH3rnqnE3TDgEA0KMReoKr6sz1jrXWDt3kPb8vyQOSXJHkL9c57e7DZ+V1pyU5prX2hRX79k9ykyQXtda+ssZ9Pj1sD9nMWFeSBAMAsBX/JckBSU5urX1x1bFLkvxRkkOTXH/43DnJqUnukuS9Q+G7bNewvXCdZy3vP2Crg5YEAwD0aIQe3s2mvVdjuRXiz9d43n8kedqq3adX1ZFJ3p/k9kkenORFI4xrtyTBAAA9Wlpa/GfBqurWSQ7LbCqzd8x7XWvt8nyvdeLwFYeWk95dWdvy/gs2MMw1KYIBANisq3shbne+Omy/2w7RWrs4sxkmrltVN1rjmoOH7dkbfNZVKIIBAHrU+RRpVXXtJA/M7IW4V27iFncYtues2n/KsL3HGtfcc9U5m6YIBgBgM34zsxfd3rnGC3FJkqq6TVVdpd6sqrsledzw9XWrDr982D65qq6/4pqDkjwyyWVJXrWlkceLcQAAfepz2eSVllsh1lohbtnzkxxcVWdk1jecJD+T783z+9TW2hkrL2itnVFVz0/y+CRnVdVJmS2b/FtJfiDJo7e6WlyiCAYA6FPHRXBV3SrJL+bqX4h7bZL7JLldZq0M10zy70nekOT41tr71rqotXZsVX08s+T3IUmWkvxTkue21t62iN+gCAYAYENaa59MUnOc98psrl84rbUTkpywmWvnoQgGAOjRCPME8z1ejAMAYHIkwQAAPeq4J3gnUAQDAPRIO8SotEMAADA5C02Cq+r7Mlv945AkBwy7L8hsabu/b61dusjnAQDsWNohRrWQInhYzePZmS2dd511Trukql6T5CmttfMX8VwAANiMLRfBVXVAkg8k+YkkFyd5T5JPJ7lwOGVXkoOT3CnJw5Pctaru2Fq7cI3bAQCQ6Ake2SKS4KdnVgC/IMnTW2sXrXVSVV03ybOSPDbJ05IcO8/Nq+rM9Y7tc80bb3iwAACwiBfjjkpySmvt2PUK4CRprV3UWnt8ktOS3HcBzwUA2LmWlhb/4bsWkQTfKMn/2cD5f5/ksHlPbq0dut6xffe7SdvAcwEA9h6K1lEtIgn+epJbbuD8Ww3XAADAtlhEEfyuJEdV1SOu7sSqelSSeyc5eQHPBQDYuVpb/IfvWkQ7xFOT/FqSl1TVsUnendm8wCtnhzgkyZFJDkryH5m9GAcAANtiy0Vwa+1LVXXHJC9LcvckD02y+m81ati+O8kjWmtf2upzAQB2ND3Bo1rIYhmttXOS/EpV3TzJXTPrEd41HL4wyaeSnDqcBwDA1VEEj2qhyyYPRa5CFwCAri20CAYAYEGsGDeqRcwOAQAAexVJMABAj/QEj0oRDADQI/P6jko7BAAAkyMJBgDokXaIUUmCAQCYHEkwAECPJMGjkgQDADA5kmAAgB5ZLGNUimAAgA61JVOkjUk7BAAAkyMJBgDokRfjRiUJBgBgciTBAAA98mLcqBTBAAA98mLcqLRDAAAwOZJgAIAeeTFuVJJgAAAmRxIMANAjSfCoFMEAAD1qXowbk3YIAAAmRxIMANAj7RCjkgQDADA5kmAAgB5ZLGNUkmAAACZHEgwA0KOmJ3hMimAAgB5phxiVdggAACZHEgwA0KFmirRRSYIBAJgcSTAAQI/0BI9KEQwA0COzQ4xKOwQAAJMjCQYA6JF2iFFJggEAmBxJMABAj0yRNipFMABAj7RDjEo7BAAAkyMJBgDokSnSRiUJBgBgciTBAAA90hM8KkkwAACTIwkGAOhQM0XaqCTBAAA9WmqL/yxQVd2tqt5UVf9WVZdV1Zer6l1V9atrnHtYVb2jqr5RVZdW1VlV9diq2mc3979XVZ1WVRdW1UVV9Q9Vdcyixq8IBgBgQ6rqfyb52yS3TfL/kjwvyduT/GCSu6w69zeSnJ7k8CRvSnJ8kv2SvCDJievc/1FJ3prkp5K8Lskrktw4yQlVddxCfkNre2/T9b773WTvHTzADnONqu0eAizEdy47r4s/zBc98T4Lr3Ou+9w3bfm3VdXvJ/mLJK9O8pDW2ndWHb9ma+0/h7/+/iSfSbIryZ1aax8e9l87ySlJ7pjkt1trJ664/qAk/5rk4iSHttbOHfZfP8k/JrlFksNaax/cyu+QBAMAMJequlaSZyf5QtYogJNkuQAeHJ1ZOnzicgE8nPPtJE8Zvj581S1+L8m1khy/XAAP15yf5E+Grw/b2i/xYhwAQJ/6XCzj7pkVtS9MslRVv5ZZy8K3k3xojXT2iGF78hr3Oj3JJUkOq6prtdYum+Oad646Z9MUwQAAPRphnuCqOnO9Y621Q+e4xe2G7beTfCSzAnjl/U9PcnRr7avDrlsO27PXeN7lVfW5JLdOcvMkn5zjmq9U1cVJblpV12mtXTLHmNekHQIAgHn90LB9YpKW5JeSXC/JzyR5d2Yvv/3fFefvGrYXrnO/5f0HbOKaXescn4skGACgQ22EJHjOtHd3lgPUy5Pce0XP7ser6j5JPpXkzlV1x62+uDY2STAAAPO6YNh+ZOVLa0kytCa8a/j6C8P26lLb5f0XrNg37zXrJcVzUQQDAPSoz8UyPjVsL1jn+PnD9vtWnX/I6hOrat8kP5ZZqnzOGs9Y65obJdk/yXlb6QdOFMEAAH1aWlr8Z+vem1kv8E9W1Vp15PKLcp8btqcM23usce7hSa6T5IwVM0Nc3TX3XHXOpimCAQCYS2vt85mt5PYjSf5g5bGqOjLJr2SWEi9Pb3ZSkq8luX9V3XbFuddO8sfD15etesyrklyW5FHDwhnL11w/yZOGry/f6m/xYhwAQI9GeDFuQR6Z5OeTPH+YJ/gjmbU1HJXkiiQPbq1dmCSttW8OK8ydlOS0qjoxyTeS3DuzqdBOSvL6lTdvrX2uqp6Y5MVJPlxVr0/yncwW3rhpkuct4qU7RTAAAHNrrZ1XVYcmeVpmxezhSb6ZWUL8p621D606/81VdeckT05yvyTXzmwp5ccneXFr7SrVfmvtJVV1bpInJPmdzLoX/iXJU1prr17E76g1nrvX2He/m+y9gwfYYa5Rtd1DgIX4zmXndfGH+VsPu8fC65zrvfzkLn5bD/QEAwAwOdohAAA6tDf/0/q9gSIYAKBH/b4YtyNohwAAYHIkwQAAPZIEj0oRDNvs0i+/b7uHAAtx4I/+8nYPAWBuimAAgA41SfCoFMEAAD1SBI/Ki3EAAEyOJBgAoEdL2z2AnU0SDADA5EiCAQA65MW4cSmCAQB6pAgelXYIAAAmRxIMANAjL8aNShIMAMDkSIIBADrkxbhxSYIBAJgcSTAAQI/0BI9KEQwA0CHtEOPSDgEAwORIggEAeqQdYlSSYAAAJkcSDADQoSYJHpUiGACgR4rgUWmHAABgciTBAAAd0g4xLkkwAACTIwkGAOiRJHhUimAAgA5phxiXdggAACZHEgwA0CFJ8LgkwQAATI4kGACgQ5LgcSmCAQB61Gq7R7CjaYcAAGByJMEAAB3SDjEuSTAAAJMjCQYA6FBb0hM8JkkwAACTIwkGAOiQnuBxKYIBADrUTJE2Ku0QAABMjiQYAKBD2iHGJQkGAGByJMEAAB0yRdq4FMEAAB1qbbtHsLNphwAAYHIkwQAAHdIOMS5JMAAAkyMJBgDokCR4XIpgAIAOeTFuXNohAACYHEkwAECHtEOMSxIMAMDkSIIBADrUmiR4TJJgAAAmRxIMANChtrTdI9jZFMEAAB1a0g4xKu0QAABMjiQYAKBDXowblyQYAIBNq6oHVFUbPg9edewuK46t9XnOOvfcp6oeV1VnVdWlVfWNqnpHVR22qHFLggEAOrQ3LJZRVTdLcnySi5Jcdzen/l2S09bY//417llJTkxydJJPDff/gSS/leT0qrpfa+0tWxu5IhgAoEutbfcIdm8oVl+V5OtJ/ibJE3Zz+mmttWfMeev7Z1YAn5Hkbq21bw/Pe3lmRfMrquqU1tq3Njv2RDsEAACb85gkRyR5UJKLF3jfhw/bpywXwEnSWvvHJK9P8oOZFclboggGAOhQW6qFfxalqm6V5DlJXtRaO32OS368qh5VVU+qqt+rqoPXue+1kxyW5JIk71vjlHcO2yM2M+6VtEMAAExEVZ253rHW2qFz3mPfJK9N8oUkT5rz0f91+Ky8zxuT/H5r7fwVu2+RZJ8k57TWLl/jPp8etofM+dx1KYIBADrU8WIZT0vy80l+sbV26dWc+9Ukf5jk7UnOTXLtJLdN8idJ7pfkhlV1eGvfXR9v17C9cJ37Le8/YHND/x5FMABAh8aYJ3jetHc9VXX7zNLf57XWPjjH8/45yT+v2HVRkpOr6owkH01ypyS/nmTLsz1slJ5gAACu1tAG8ZokZyd56lbu1Vr7ZpL/PXw9fMWh5aR3V9a2vP+CrTw/UQQDAHSptcV/tui6mfXi3irJt1cuepHk6cM5rxj2vXCO+3112O6/Yt9nk1yR5OZD0b3a8gt1Z298+FemHQIAgHlcluSV6xy7TWZ9wu/PbIGLq22VSHKHYXvO8o7W2reHVolfGj6nrrrmnsP2lDnHvC5FMABAh3p7MW54Ce7Bax2rqmdkVgS/urX2lyv237a19uE1zn9AZivAfSfJG1YdfllmBfAfV9XKxTJuN1zz1SRv3OrvUQQDADCWk6rq8iQfTnJeZrND3C7JLyS5PMlDW2vnrrrmxCT3zWxBjI9U1VuTHJhZAbxPZtOqfXOrA1MEAwB0aIzZIbbBy5L8cmazQNwgSSX5UpITkrywtfax1Re01lpV/XZmyyb/XpJHJ/l2ktOT/HFr7YxFDKxa7wtT78a++91k7x08DC798loL4sDe58Af/eXtHgIsxDcvPqeL6vOfbvYbC69zbvPFt3Tx23qwLbNDVNVzq+qz2/FsAADYrnaIGyQ5aJ4Td7e83z7XvPGixgMA0JXeXozbacwTDADA5CwkCa6q12zwksPmPXF3y/vpCQYAdqod8mJctxbVDvGAJC2zN/7mpYAFAFiHdohxLaoI/lZmc789Ys7z/zDJkQt6NgAAbMiiiuCPJfnZ1trfzXNyVf3ugp4LALAj+Ufm41rUi3EfTXLdqrrFgu4HAACjWVQS/HeZrfF80yTzzP/75iTnLujZAAA7jp7gcS2kCG6tvTHJGzdw/luSvGURzwYA2InMDjEu8wQDADA527ViHAAAu7G03QPY4STBAABMjiQYAKBDbUNrkLFRkmAAACZHEgwA0KElq2WMShEMANChJe0Qo9IOAQDA5EiCAQA65MW4cUmCAQCYHEkwAECHLJYxLkUwAECHtEOMSzsEAACTIwkGAOiQdohxSYIBAJgcSTAAQIckweNSBAMAdMiLcePSDgEAwORIggEAOrQkCB6VJBgAgMmRBAMAdGhJT/CoJMEAAEyOJBgAoENtuwewwymCAQA6ZJ7gcWmHAABgciTBAAAdWiovxo1JEgwAwORIggEAOuTFuHEpggEAOuTFuHFphwAAYHIkwQAAHVryXtyoJMEAAEyOJBgAoENLEQWPSREMANAhs0OMSzsEAACTIwkGAOiQF+PGJQkGAGByJMEAAB2yWMa4JMEAAEyOJBgAoENmhxiXIhgAoENejBuXdggAACZHEgwA0CEvxo1LEgwAwORIggEAOiQJHpciGACgQ82LcaPSDgEAwORIggEAOqQdYlySYAAAJkcSDADQIUnwuBTBAAAdsmzyuLRDAAAwOYpgAIAOLdXiP4tQVX9WVe+tqi9W1aVV9Y2q+khVPb2qDlznmsOq6h3DuZdW1VlV9diq2mc3z7lXVZ1WVRdW1UVV9Q9VdcxifoUiGACAjXlckv2TvCfJi5L8dZLLkzwjyVlVdbOVJ1fVbyQ5PcnhSd6U5Pgk+yV5QZIT13pAVT0qyVuT/FSS1yV5RZIbJzmhqo5bxI/QEwwA0KGOX4z7/tbat1fvrKpnJ3lSkv+R5BHDvu/PrIC9IsldWmsfHvY/NckpSY6uqvu31k5ccZ+DkhyX5BtJbttaO3fY/6wk/5jk2Kp6Y2vtg1v5EZJgAADmtlYBPHjDsD14xb6jk/xgkhOXC+AV93jK8PXhq+7ze0muleT45QJ4uOb8JH8yfH3Ypga/giQYAKBDYyTBVXXmesdaa4du8fa/PmzPWrHviGF78hrnn57kkiSHVdW1WmuXzXHNO1eds2mKYACADvU+RVpVPSHJdZPsSnLbJL+YWQH8nBWn3XLYnr36+tba5VX1uSS3TnLzJJ+c45qvVNXFSW5aVddprV2y2fErggEAJmIBae9KT0jywyu+n5zkd1trX12xb9ewvXCdeyzvP2CD1+w/nLfpIlhPMABAh3qdIm1Za+2GrbVKcsMk980szf1IVd1msU8ahyIYAIBNa639e2vtTUmOTHJgktesOLyc5u66yoVX3n/BJq5ZLymeiyIYAKBDSyN8xtRa+3ySf0ly66q6wbD7U8P2kNXnV9W+SX4sszmGz1lxaHfX3CizVojzttIPnCiCAQC61Eb47AE3HrZXDNtThu091jj38CTXSXLGipkhru6ae646Z9MUwQAAzKWqDqmqq7QpVNU1hsUyfiizovb84dBJSb6W5P5VddsV5187yR8PX1+26navSnJZkkcNC2csX3P9zBbjSJKXb/W3mB0CttlNb/Gr2z0EWIhzj7zZ1Z8EzG2pz0nSfjXJn1bV+5N8LsnXM5sh4s6ZvRj3b0l+f/nk1to3q+r3MyuGT6uqEzNbCe7emU2FdlKS1698QGvtc1X1xCQvTvLhqnp9ku9ktvDGTZM8b6urxSWKYAAA5ve3SX48szmBfz6zqc0uzmxO39cmeXFr7RsrL2itvbmq7pzkyUnul+TaST6T5PHD+Vep9ltrL6mqczObhu13Mute+JckT2mtvXoRP0QRDADQobFfZNuM1tonkjxqE9d9ILMUeSPXvDXJWzf6rHkpggEAOtRlM8QO4sU4AAAmRxIMANChHtshdhJJMAAAkyMJBgDo0FJt9wh2NkUwAECHOp0neMfQDgEAwORIggEAOiQHHpckGACAyZEEAwB0yBRp45IEAwAwOZJgAIAOmR1iXIpgAIAOKYHHpR0CAIDJkQQDAHTIi3HjkgQDADA5kmAAgA55MW5cimAAgA4pgcelHQIAgMmRBAMAdMiLceOSBAMAMDmSYACADjVdwaNSBAMAdEg7xLi0QwAAMDmSYACADpkneFySYAAAJkcSDADQITnwuCTBAABMjiQYAKBDeoLHpQgGAOiQKdLGpR0CAIDJkQQDAHTIinHjkgQDADA5kmAAgA7pCR6XIhgAoEPaIcalHQIAgMmRBAMAdEg7xLgkwQAATI4kGACgQ0tNT/CYFMEAAB1SAo9LOwQAAJMjCQYA6NCSLHhUkmAAACZHEgwA0CGLZYxLEgwAwORIggEAOmSxjHEpggEAOuTFuHFphwAAYHIkwQAAHfJi3LgkwQAATI4kGACgQ16MG5ciGACgQ61phxiTdggAACZHEgwA0CFTpI1LEgwAwORIggEAOuTFuHEpggEAOmSe4HFphwAAYHIkwQAAHfJi3LgkwQAATI4kGACgQxbLGJckGACAuVTV0VX1kqp6X1V9s6paVb1unXMPGo6v9zlxN885pqo+VFUXVdWFVXVaVd1rkb9FEgwA0KFOp0h7SpKfTXJRkvOS/MQc13wsyZvX2P+JtU6uquOSHDvc/xVJ9kty/yRvrapHt9aO38S4r0IRDADQoU6nSHtcZsXpZ5LcOcmpc1zz0dbaM+a5eVUdllkB/Nkkt2utnT/sf26SM5McV1Vva62du/GhX5l2CAAA5tJaO7W19uk2XsPyw4bts5cL4OG55yZ5aZJrJXnQIh4kCQYA6NAYU6RV1ZnrHWutHbrwB87cuKoemuTAJF9P8sHW2lnrnHvEsD15jWPvTPLU4Zynb3VQimAAAMZ09+HzXVV1WpJjWmtfWLFv/yQ3SXJRa+0ra9zn08P2kEUMamFFcFXdJ8ldklye5OTW2nvWOe+YzH70EWsdBwBgnCnSRkx713JJkj/K7KW4c4Z9P5PkGUnumuS9VfVzrbWLh2O7hu2F69xvef8Bixjclovgqqokr09yvyQ17H5sVb09ye+01i5YdclBmTVSAwCwjr19xbjW2n8kedqq3adX1ZFJ3p/k9kkenORFe3psyWKS4AclOTrJF5O8PMl/Jjkmyb2SvL+qjhj+TdiU3fWu7HPNG2/2tgAAbIPW2uVV9ZeZFcGH53tF8HLSu2vNC7+3f3XAuimLmB3iQZkN5nattT9trR2X5OeSPD/JTyb526q6wQKeAwAwGW2Ef3Xkq8N2/+UdQ1vEl5Jct6putMY1Bw/bsxcxgEUkwT+d5KSVaW9r7YokT6iqLyR5YWaF8F1XTnUxr931ruy73026+k8TAIC53GHYnrNq/ylJHpjkHkleterYPVecs2WLSIL3S/Lvax1orb04yWMya4J+T1UtpJEZAGCnW2pt4Z89qapuU1VXqTWr6m6ZLbqRJKuXXH75sH1yVV1/xTUHJXlkksty1eJ4UxaRBH8pyY+sd7C1dnxV7ZtZe8S7knxgAc8EANjRevzH3VV1VJKjhq83HLZ3rKoThr/+WmvtCcNfPz/JwVV1RmarzCWzYHR5hrCnttbOWHn/1toZVfX8JI9PclZVnZRZ4PpbSX4gyaMXsVpcspgi+OOZTXOxrtbaC6vqWkn+NMnPL+CZAADseT+X2QQIK918+CTJ55MsF8GvTXKfJLfLrJXhmpl1D7whyfGttfet9YDW2rFV9fHMkt+HJFlK8k9Jnttae9uifsgiiuB3JDmqqn6ttfb29U5qrf1ZVe2X5Jnp829uAAC60eMUaa21Z2Q2z+88574yySs3+ZwTkpywmWvntYgi+G+S7JPk4qs7sbX2R8PLcgct4LkAALApWy6CW2vfSPLnGzj/1Vt9JgDATtdjEryTLGJ2CAAA2Kssoh0CAIAFa3t4SrOpUQQDAHRIO8S4tEMAADA5kmAAgA41SfCoJMEAAEyOJBgAoENejBuXIhgAoENejBuXdggAACZHEgwA0CHtEOOSBAMAMDmSYACADukJHpciGACgQ+YJHpd2CAAAJkcSDADQoSUvxo1KEgwAwORIggEAOqQneFySYAAAJkcSDADQIT3B41IEAwB0SDvEuLRDAAAwOZJgAIAOaYcYlyQYAIDJkQQDAHRIT/C4FMEAAB3SDjEu7RAAAEyOJBgAoEPaIcYlCQYAYHIkwQAAHWptabuHsKMpggEAOrSkHWJU2iEAAJgcSTAAQIeaKdJGJQkGAGByJMEAAB3SEzwuSTAAAJMjCQYA6JCe4HEpggEAOrSkCB6VdggAACZHEgwA0KHmxbhRSYIBAJgcSTAAQIe8GDcuRTAAQIfMEzwu7RAAAEyOJBgAoEPaIcYlCQYAYHIkwQAAHbJYxrgUwQAAHdIOMS7tEAAATI4kGACgQ6ZIG5ckGACAyZEEAwB0SE/wuCTBAABMjiQYAKBDpkgblyIYAKBDzYtxo9IOAQDA5EiCAQA6pB1iXJJgAAAmRxIMANAhU6SNSxEMANAhL8aNSzsEAAAbUlU3raq/qqovV9VlVXVuVb2wqq6/3WOblyQYAKBDvbZDVNUtkpyR5IeSvCXJvyb5hSR/kOQeVXWn1trXt3GIc5EEAwCwEf8rswL4Ma21o1prf9haOyLJC5LcMsmzt3V0c1IEAwB0qLW28M9WDSnwkUnOTfLSVYefnuTiJA+sqv23/LCRKYIBADrURvgswF2H7btba0tXGm9r30rygSTXSXKHxTxuPHqCAQAmoqrOXO9Ya+3QOW5xy2F79jrHP51ZUnxIkvdubHR71q72M1MAAARVSURBVF5dBF/+nS/Vdo9hp1v+L8uc/8WALvlzzE7hz/K0jFHn7K4IntOuYXvhOseX9x+wxeeMbq8uggEAmJ+/gfoePcEAAMxrOendtc7x5f0X7IGxbIkiGACAeX1q2B6yzvGDh+16PcPdUAQDADCvU4ftkVV1pTqyqq6X5E5JLkny93t6YBulCAYAYC6ttc8meXeSg5I8ctXhZybZP8lrW2sX7+GhbZgX4wAA2IhHZLZs8our6m5JPpnk9pnNIXx2kidv49jmVr2uSw0AQJ+q6mZJnpXkHkkOTPKVJG9K8szW2vnbObZ5KYIBAJgcPcEAAEyOIhgAgMlRBAMAMDmKYAAAJkcRDADA5CiCAQCYHEUwa6qqm1bVX1XVl6vqsqo6t6peWFXX3+6xwTyq6uiqeklVva+qvllVrapet93jgo2oqgOr6sFV9aaq+kxVXVpVF1bV+6vqv61ethaYn3mCuYqqukVmK8H8UJK3JPnXJL+Q2Uown0pyp9ba17dvhHD1quqjSX42yUVJzkvyE0n+urX2gG0dGGxAVT0sycsyW4jg1CRfSPLDSe6bZFeSNyb5zeb/zGHDFMFcRVW9K8mRSR7TWnvJiv3PT/K4JH/eWnvYdo0P5lFVd82s+P1MkjtnVkAogtmrVNURSfZP8vbW2tKK/TdM8qEkN0tydGvtjds0RNhr+ccoXMmQAh+Z5NwkL111+OlJLk7ywKrafw8PDTaktXZqa+3TEjL2Zq21U1prb11ZAA/7/y3Jy4evd9njA4MdQBHMancdtu9e4390v5XkA0muk+QOe3pgAFzJfw7by7d1FLCXUgSz2i2H7dnrHP/0sD1kD4wFgDVU1b5Jfmf4evJ2jgX2VopgVts1bC9c5/jy/gP2wFgAWNtzkvxUkne01t613YOBvZEiGAD2IlX1mCTHZjZzzwO3eTiw11IEs9py0rtrnePL+y/YA2MBYIWqelSSFyX5lyR3ba19Y5uHBHstRTCrfWrYrtfze/CwXa9nGIARVNVjk7wkyScyK4D/bZuHBHs1RTCrnTpsj1y9ElFVXS/JnZJckuTv9/TAAKaqqv57khck+WhmBfB/bPOQYK+nCOZKWmufTfLuJAcleeSqw8/MbNL217bWLt7DQwOYpKp6amYvwp2Z5G6tta9t85BgR7BiHFexxrLJn0xy+8zmED47yWGWTaZ3VXVUkqOGrzdM8itJzknyvmHf11prT9iOscG8quqYJCckuSKzVoi1Zu45t7V2wh4cFuwIimDWVFU3S/KsJPdIcmBm69a/KckzW2vnb+fYYB5V9YzMVjlcz+dbawftmdHA5szx5zhJ/q61dpfxRwM7iyIYAIDJ0RMMAMDkKIIBAJgcRTAAAJOjCAYAYHIUwQAATI4iGACAyVEEAwAwOYpgAAAmRxEMAMDkKIIBAJgcRTAAAJOjCAYAYHIUwQAATI4iGACAyVEEAwAwOYpgAAAmRxEMAMDk/H+k42WEIU9AfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 248,
       "width": 352
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = timm.create_model('efficientnet_b0', num_classes=4, checkpoint_path='/home/linh/Downloads/Covid-19/weights/EfficientNet_B0.pth')\n",
    "model.to(device)\n",
    "since = round(time.monotonic() * 1000)\n",
    "model.eval()\n",
    "y_test = []\n",
    "y_pred = []\n",
    "\n",
    "for i, (images, labels) in enumerate(data_loader['val']):\n",
    "    images = images.to(device)\n",
    "    labels = labels.to(device)\n",
    "    outputs = model(images)\n",
    "    _, predictions = outputs.max(1)\n",
    "    \n",
    "    y_test.append(labels.data.cpu().numpy())\n",
    "    y_pred.append(predictions.data.cpu().numpy())\n",
    "\n",
    "#set the end time to predict\n",
    "time_elapsed = round(time.monotonic() * 1000) - since   \n",
    "print('prediction time for complete in {:.0f} milisecond'.format(time_elapsed))\n",
    "#print('Prediction time to complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "#concat predicted results to be dataframe\n",
    "y_test = np.concatenate(y_test)\n",
    "y_pred = np.concatenate(y_pred)\n",
    "\n",
    "pd.DataFrame({'true_label':y_test,'predicted_label':y_pred}).to_csv('/home/linh/Downloads/Covid-19/results/EfficientNet_B0.csv',index=False)\n",
    "\n",
    "sns.heatmap(confusion_matrix(y_test, y_pred))\n",
    "# set accuracy score to control processes\n",
    "accuracy_score(y_test, y_pred)\n",
    "\n",
    "# Generate a classification report\n",
    "\n",
    "report = classification_report(y_test, y_pred, digits=6)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modified EfficientNet_B0 with image size = 224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint loaded\n",
      "Training complete in 1m 49s\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.73      0.84       108\n",
      "           1       0.94      0.99      0.97      8851\n",
      "           2       0.98      0.92      0.95      6041\n",
      "\n",
      "    accuracy                           0.96     15000\n",
      "   macro avg       0.97      0.88      0.92     15000\n",
      "weighted avg       0.96      0.96      0.96     15000\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs4AAAHwCAYAAAC2dOlsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de9iu13wv+u9vJZJItBOhRaJXKiuhBz0gRdI6xJLSaqsOG9fWpizUeSG6dy+HogvtWtI4xaZLW1HW3okdxQ4SVKRB1CGl0YWERIrQOkRm5KjJO/Yf9/3q48nzznnP933mfMec8/NxPdfIc99j3Pd4XIn85s9vjFGttQAAANv2HzZ7AgAAsDsQOAMAwAQCZwAAmEDgDAAAEwicAQBgAoEzAABMIHAGAIAJBM4AADCBwBkAACYQOAMAwAQCZwAAmEDgDAAAEwicAQBggn03ewIbsd/+h7bNngNs1ErztzFAT274/mW12XNIkn/79iVL/xfEzW5zpy5+2+5KxhkAACbYrTPOAAB7rJUbN3sGzJFxBgCACWScAQB61FY2ewbMETgDAPRoReDcG6UaAAAwgYwzAECHmlKN7sg4AwDABDLOAAA9UuPcHYEzAECPlGp0R6kGAABMIOMMANAjJwd2R8YZAAAmkHEGAOiRGufuyDgDAMAEMs4AAD2yHV13BM4AAB1ycmB/lGoAAMAEMs4AAD1SqtEdGWcAAJhAxhkAoEdqnLsjcAYA6JGTA7ujVAMAACaQcQYA6JFSje7IOAMAwAQyzgAAPbIdXXcEzgAAPVKq0R2lGgAAMIGMMwBAj5RqdEfGGQAAJpBxBgDoUGsOQOmNwBkAoEcWB3ZHqQYAAEwg4wwA0COLA7sj4wwAABPIOAMA9EiNc3dknAEAYAIZZwCAHq3Yjq43AmcAgB4p1eiOUg0AAJhAxhkAoEe2o+uOjDMAAEwg4wwA0CM1zt0ROAMA9EipRneUagAAwAQyzgAAPZJx7o6MMwAATCDjDADQodacHNgbgTMAQI+UanRHqQYAAEwg4wwA0CP7OHdHxhkAACaQcQYA6JEa5+7IOAMAwAQyzgAAPVLj3B0ZZwCAHq2sLP+zQVX1e1XVtvO5cab/Ydvpe+o23nV8VX2iqq6qqq1VdU5VPWQb/fepqmdX1QVVdW1VXV5V762qozf8w0cyzgAATPWZJC9Z496vJDk2yZkL7v1jkncuuP5Pix5UVScmOSHJ15K8Mcl+SR6d5IyqekZr7eS5/pXk1CSPSHJhkpOT3DrJo5KcW1UPb629a9s/bfsEzgAAPeqwVKO19pkMwfNNVNXHxr/8Hwtuf6a19uIp7xgzxCckuTjJUa21747XX5Hk/CQnVtW7W2uXzgx7dIag+bwkD2itXTeOeUOSjyR5Y1Wd3Vr73pQ5rEWpBgAAG1JVd01yrySXJXnPBh/35LF92WrQnCRjoPy6JPsnedzcmKeM7QtWg+ZxzCeTnJbkthkC6w0ROAMA9KjDGudteNLY/mVr7cYF9+9QVb9fVc8b25/bxrOOHduzFtw7c65PquqAJEcnuSbJh6eMWS+lGgAAPdoJgW5Vnb/Wvdba3df5zJsneWySG5P8xRrdHjh+Zsedk+T41tpXZq4dlOSQJFe11r6x4DlfHNsjZ64dnmSfJJe01m6YOGZdZJwBANiI/y3JLZOc1Vr76ty9a5L81yR3T3Kr8XPfJB9Kcr8kHxyD5VVbxnbrGu9avX7LDY5ZFxlnAIAe7YTFgevNKm/HapnGny943zeT/NHc5XOr6rgMi/bumeQJSV69E+a1dDLOAACsS1X9TIb64q8lee/UcWNJxWpZx31mbq1mh7dksdXrV2xwzLrIOAMA9GjnLuZblu0tCtyWb43tD0o1WmtXV9VlSQ6pqtsvqHM+Ymwvmrl2cYb66jtV1b4L6pwXjVkXGWcAgB61leV/lmjczeJ3MgStf7mOR9xrbC+Zu3722D5owZgHz/XJuP3ceUkOzHAIy3bHrJfAGQCA9XhkhsV+Zy5YFJgkqaq7VdVN4s2qekCSZ49f3zp3+w1j+/yqutXMmMOSPC3J9UneNDfm9WP70jGgXx1zVIbTA7+V5O3b/0nbttRSjXE7kntl2O5jdeXiFRlS43/fWrt2me8DANhj9V+qsVqmseikwFUnJTmiqs7LUAedJD+Xf99T+YWttfNmB7TWzquqk5I8J8kFVXV6hiO3H5XhGO1nzJ0amAzHbT8swyEnn66qM5IcPI7ZJ8kTW2tX7vhP/GFLCZzHPw28LEO6/sA1ul1TVX+d4USX767RBwCAzlXVTyX55Wx/UeBbkvx2kqMylEzcLMm/JnlbkpNba4sOLElr7YSq+myGDPOTkqwk+Yckr2itvXtB/1ZVj8lQsvH4JM9Icl2Sc5O8dD44X69qrW3sAVW3zDDJuyS5OslHM2w0PbvC8Ygkx2Qo/v5Cknu31tbaa2+y/fY/dGOThw6sbPCfQQCW64bvX1abPYckufZvXr70f0Hc/GHP6+K37a6WkXF+UYag+ZVJXtRau2pRp6q6RZI/TvKsDPv5nTDl4ds64eZm+x2yw5MFAID1WMbiwIcmObu1dsJaQXOStNauaq09J8k5GWpQAABYy8rK8j9syDIyzrdP8v/sQP+/z7BR9iTbOuFGqQYAsMcS6HZnGRnn7yS58w70/6lxDAAA7DaWETi/L8lDq+qp2+tYVU9P8ptJzlrCewEA9lytLf/DhiyjVOOFSX49yWur6oQk78+wb/PsrhpHJjkuyWFJvplhcSAAAOw2Nhw4t9Yuq6p7Zzix5YFJfj/J/B9pVrc+eX+Sp7bWLtvoewEA9mhqnLuzlANQWmuXJPnVqrpTkvtnqHneMt7emuTCJB8a+wEAsD0C5+4s9cjtMTAWHAMAsMdZauAMAMCSNBnn3ixjVw0AANjjyTgDAPRIjXN3BM4AAD2y73J3lGoAAMAEMs4AAD1SqtEdGWcAAJhAxhkAoEcyzt2RcQYAgAlknAEAeuQAlO4InAEAOtRWbEfXG6UaAAAwgYwzAECPLA7sjowzAABMIOMMANAjiwO7I3AGAOiRxYHdUaoBAAATyDgDAPTI4sDuyDgDAMAEMs4AAD2Sce6OwBkAoEfN4sDeKNUAAIAJZJwBAHqkVKM7Ms4AADCBjDMAQI8cgNIdGWcAAJhAxhkAoEdNjXNvBM4AAD1SqtEdpRoAADCBjDMAQIea7ei6I+MMAAATyDgDAPRIjXN3BM4AAD2yq0Z3lGoAAMAEMs4AAD1SqtEdGWcAAJhAxhkAoEe2o+uOwBkAoEdKNbqjVAMAACaQcQYA6JHt6Loj4wwAABPIOAMA9EiNc3dknAEAYAIZZwCADjXb0XVH4AwA0COlGt1RqgEAABPIOAMA9EjGuTsyzgAAMIGMMwBAjxyA0h2BMwBAj5RqdEepBgAATCBwBgDoUFtpS/8sU1U9oKreUVX/UlXXV9XXq+p9VfVrC/oeXVXvrarLq+raqrqgqp5VVfts4/kPqapzqmprVV1VVR+vquO3M6fjq+oTY/+t4/iHLOP3JgJnAAB2UFX99yR/m+QeSf6/JH+W5D1JbpvkfnN9fyvJuUnuk+QdSU5Osl+SVyY5dY3nPz3JGUl+Nslbk7wxyR2SnFJVJ64x5sQkpyS5/dj/rUnumuSM8XkbVq3tvvUz++1/6O47eRit7Mb/DALsiW74/mW12XNIku898yFL/xfEj7zm3Rv+bVX1xCT/I8mbkzyptfb9ufs3a6392/jXP5rkS0m2JDmmtfap8foBSc5Ocu8kj2mtnToz/rAkX0hydZK7t9YuHa/fKsknkxye5OjW2sdmxhyd5KNJLk5yVGvtuzPPOj/JQUnusvqs9ZJxBgDo0crK8j8bVFX7J3lZkq9kQdCcJKtB8+gRGbLQp64GzWOf65K8YPz6lLlHPD7J/klOng10x2D45ePXJ8+NWf3+stWgeRxzaZLXjc973PZ/4bbZVQMAYC9RVeevda+1dvcJj3hghkD4VUlWqurXM5RTXJfkE7NZ4NGxY3vWgmedm+SaJEdX1f6ttesnjDlzrs+U95yZ5IVjnxctuD+ZwBkAoEd9bkd31Nhel+TTGYLmH6iqc5M8orX2rfHSncf2ovkHtdZuqKovJ/mZJHdK8vkJY75RVVcnObSqDmytXVNVByU5JMlVrbVvLJjzF8f2yCk/cFsEzgAAe4mJWeVt+bGx/YMkn0vyK0k+k+Qnk5yY5Lgk/2/+fYHglrHdusbzVq/fcubalDEHjf2uWec71kXgDADQoz4zzqvr425I8pszNcifrarfTnJhkvtW1b0XlG3s9iwOBABgqivG9tPzO1S01q5J8r7x6y+N7Wq2d0sWW71+xcy1qWO2zrU78o51ETgDAHSotbb0zxJcOLZrBaGrO1rcfK7/TeqLq2rfDCUeNyS5ZME7Fo25fYYyja+NgXpaa1cnuSzJLcb7844Y25vUTO8ogTMAQI9W2vI/G/fBJC3JT1fVojhydbHgl8f27LF90IK+90lyYJLzZnbU2N6YB8/12ciYHSZwBgBgktbaP2c40e8nkvyX2XtVdVySX82QjV7dFu70JN9O8uiqusdM3wOSvHT8+vq517wpyfVJnj4eYLI65lZJnjd+fcPcmNXvzx/7rY45LMnTxue9adKP3AaLAwEAetTn4sBkCER/MclJ4z7On85QcvHQJDcmeUJrbWuStNauHE8aPD3JOVV1apLLk/xmhm3nTk9y2uzDW2tfrqo/SPKaJJ+qqtOSfD/DYSqHJvmz+YWHrbXzquqkJM9JckFVnZ7hWO9HJbl1kmds9NTAZDc/cnvf/Q7ZfScPo2u//uHNngIsxS0Ove9mTwGW4vrrvtrFkdtX/ucHLj3O+dG//MBSfltV3TbJH2UIgG+f5MokH07yJ621Tyzof0yS52c4YvuADMdw/1WS17TWblzjHb+R5LlJ7pahSuJzGU4TfPM25vV7GQL7n06ykuQfkryitfbudf3Q+ecLnGFzCZzZUwic2VP0Ejhvfdx/Wnqcs+VNf9vFb9tdKdUAAOhRv6Uaey2LAwEAYAIZZwCAHq1s9gSYJ+MMAAATyDgDAHSoqXHujsAZAKBHAufuKNUAAIAJZJwBAHpkcWB3ZJwBAGACGWcAgA5ZHNgfGWcAAJhAxhkAoEdqnLsjcAYA6JBSjf4o1QAAgAlknAEAeqRUozsyzgAAMIGMMwBAh5qMc3cEzgAAPRI4d0epBgAATCDjDADQIaUa/ZFxBgCACWScAQB6JOPcHYEzAECHlGr0R6kGAABMIOMMANAhGef+yDgDAMAEMs4AAB2Sce6PwBkAoEetNnsGzFGqAQAAE8g4AwB0SKlGf2ScAQBgAhlnAIAOtRU1zr2RcQYAgAlknAEAOqTGuT8CZwCADjXb0XVHqQYAAEwg4wwA0CGlGv2RcQYAgAlknAEAOmQ7uv4InAEAOtTaZs+AeUo1AABgAhlnAIAOKdXoj4wzAABMIOMMANAhGef+CJwBADpkcWB/lGoAAMAEMs4AAB1SqtEfGWcAAJhAxhkAoEOtyTj3RsYZAAAmkHEGAOhQW9nsGTBP4AwA0KEVpRrdUaoBAAATyDgDAHTI4sD+yDgDAMAEMs4AAB1yAEp/BM4AAB1qbbNnwDylGgAAMIGMMwBAh5Rq9EfGGQCAdauqx1ZVGz9PmLt3v5l7iz5/usYz96mqZ1fVBVV1bVVdXlXvraqjtzGPm1fVS6rqwqq6rqq+WVVvq6qfWtZvlXEGAOjQ7nAASlXdMcnJSa5KcottdP27JOcsuP6RBc+sJKcmeUSSC8fn3zrJo5KcW1UPb629a27M/kk+kOSYJJ9K8uokd0zyyCS/XlXHttY+vkM/bgGBMwBAh3rfx3kMcN+U5DtJ/ibJc7fR/ZzW2osnPvrRGYLm85I8oLV23fi+N2QItN9YVWe31r43M+Y5GYLm05M8qrXhwPKqOi3JO5P8VVXddfX6einVAABgPZ6Z5Ngkj0ty9RKf+5SxfcFq0JwkrbVPJjktyW0zBNZJfhDAP3n8+n/MBsdjZvrDSX46yX03OjGBMwBAh1pb/mdZxrrhP03y6tbauROG/MeqenpVPa+qHl9VR6zx3AOSHJ3kmgwB77wzx/bYmWuHJ/mJJBe11r48ccy6KNUAANhLVNX5a91rrd194jP2TfKWJF9J8ryJr/7fx8/sc96e5Imtte/OXD48yT5JLmmt3bDgOV8c2yNnrt15bC9a492LxqyLwBkAoEMdLw78oyS/mOSXW2vXbqfvt5L8YZL3JLk0yQFJ7pHk5UkenuR2VXWfmfKKLWO7dY3nrV6/5cy19YxZF4EzAMBeYmpWeS1Vdc8MWeY/a619bML7/leS/zVz6aokZ1XVeUk+k2FB328kedeC4d1R4wwA0KHWaumfjRhLNP46Q0nECzf229qVSf7v8et9Zm6tZoe3ZLHV61dscMy6CJwBADrU4eLAW2SoE/6pJNfNHmSS5EVjnzeO11414XnfGtuDZq5dnOTGJHcaA/V5q4sKZ+uZLxzbtWqYF41Zl00p1aiqVyR5WGvt8M14PwAAO+z6JH+5xr27Zah7/kiGQHa7ZRxJ7jW2l6xeaK1dN5Zx/Mr4+dDcmAeP7dkz1y7OsFDxyKr6yQU7aywasy6bVeN8mySHTem4rdWf+9zsDsuaDwBAV3pbHDguBHzContV9eIMgfObW2t/MXP9Hq21Ty3o/9gMJwF+P8nb5m6/PkPQ/NKqmj0A5ahxzLeSvH1mXm08HOXlSf57Vc0egPJb47M+l+H0wg2xOBAAgJ3l9Kq6IcMx2F/LsKvGUUl+KckNSX6/tXbp3JhTkzwswyEnn66qM5IcnCFo3ifDFnZXzo05KclDxjEfr6oPZtjb+ZEZ9oR+/EZPDUyWFDhX1V/v4JCjp3bc1urPffc7ZIlbeQMA9KP3I7cnen2S/5Rh94zbJKkklyU5JcmrWmv/OD9gzCA/JsOR249P8owk1yU5N8lLW2vnLRhzfVU9MMPWd49J8uwkV2Y4bvtFrbXPLePHVFtCpXhVrSRpGf7LmKq11vbZyHsFzuwJrv36ooORYPdzi0M3fJotdOH6677aRcT68Ts8bOlxzj2//jdd/Lbd1bJKNb6XIf3+1In9/zDJcUt6NwAA7HTLCpz/McnPt9YmFV1X1e8t6b0AAHsk/7d6f5a1j/NnktyiqmwvBwDAHmlZGee/y7DVx6EZ9tLbnndmOK8cAIAFetuOjiUFzq21t2dmP70J/d+V3eRMcgCAzbCH7KqxR3HkNgAATOAAFACADm34tA6WTsYZAAAmkHEGAOhQ26Fz5dgVZJwBAGACGWcAgA6tOAGlOwJnAIAOrSjV6I5SDQAAmEDGGQCgQxYH9kfGGQAAJpBxBgDokANQ+iNwBgDokFKN/ijVAACACWScAQA6pFSjPzLOAAAwgYwzAECHZJz7I3AGAOiQxYH9UaoBAAATyDgDAHRoRcK5OzLOAAAwgYwzAECHVtQ4d0fGGQAAJpBxBgDoUNvsCXATAmcAgA7Zx7k/SjUAAGACGWcAgA6tlMWBvZFxBgCACWScAQA6ZHFgfwTOAAAdsjiwP0o1AABgAhlnAIAOrVgb2B0ZZwAAmEDGGQCgQyuRcu6NwBkAoEN21eiPUg0AAJhAxhkAoEMWB/ZHxhkAACaQcQYA6JADUPoj4wwAABPIOAMAdMiuGv0ROAMAdMjiwP4o1QAAgAlknAEAOmRxYH9knAEAYAIZZwCADsk490fgDADQoWZxYHeUagAAwAQyzgAAHVKq0R8ZZwAAmEDGGQCgQzLO/RE4AwB0yJHb/VGqAQAAE8g4AwB0aMV2dN2RcQYAgAlknAEAOmRxYH9knAEAmKyq/ltVfbCqvlpV11bV5VX16ap6UVUdvMaYo6vqvWPfa6vqgqp6VlXts433PKSqzqmqrVV1VVV9vKqO387cjq+qT4z9t47jH7LR37xK4AwA0KGVnfBZkmcnOSjJB5K8Osn/THJDkhcnuaCq7jjbuap+K8m5Se6T5B1JTk6yX5JXJjl10Quq6ulJzkjys0nemuSNSe6Q5JSqOnGNMScmOSXJ7cf+b01y1yRnjM/bsGpt993sZN/9Dtl9Jw+ja7/+4c2eAizFLQ6972ZPAZbi+uu+2sWyvBN/4rFLj3Oe+5W3bvi3VdUBrbXrFlx/WZLnJXl9a+2p47UfTfKlJFuSHNNa+9TqM5KcneTeSR7TWjt15jmHJflCkquT3L21dul4/VZJPpnk8CRHt9Y+NjPm6CQfTXJxkqNaa9+dedb5GQL9u6w+a71knAEAmGxR0Dx629geMXPtEUlum+TU1aB55hkvGL8+Ze45j0+yf5KTZwPdMRh++fj1yXNjVr+/bDVoHsdcmuR14/Met+aPmsjiQACADu2M7eiq6vy17rXW7r7Bx//G2F4wc+3YsT1rQf9zk1yT5Oiq2r+1dv2EMWfO9ZnynjOTvHDs86LFU59G4AwAwA6rqucmuUWGMox7JPnlDEHzn850u/PYXjQ/vrV2Q1V9OcnPJLlTks9PGPONqro6yaFVdWBr7ZqqOijJIUmuaq19Y8FUvzi2R+7I71tE4AwA0KGdsR3dErLKs56b5Mdnvp+V5Pdaa9+aubZlbLeu8YzV67fcwTEHjf2uWec71kWNMwBAh9pO+Cx1fq3drrVWSW6X5GEZssafrqq7LflV3RA4AwCwbq21f22tvSPJcUkOTvLXM7dXs71bbjLwh69fsY4xW+faHXnHuijVgE126OG/ttlTgKX4xoPvtNlTgD3KytJzxDtXa+2fq+pzSX6hqm7TWvt2kgsz1D8fmWFbuB+oqn2T/GSGPaAvmbl1YZLbjGM+Njfm9hnKNL7WWrtmfO/VVXVZkkOq6vYL6pxXd/m4Sc30jpJxBgBgWe4wtjeO7dlj+6AFfe+T5MAk583sqLG9MQ+e67ORMTtM4AwA0KEeTw6sqiOr6iYlEVX1H8YDUH4sQyC8upfy6Um+neTRVXWPmf4HJHnp+PX1c497U5Lrkzx9PMBkdcytMhywkiRvmBuz+v35Y7/VMYcledr4vDdN+pHboFQDAKBDnRZq/FqSP6mqjyT5cpLvZNhZ474ZFgf+S5InrnZurV1ZVU/MEECfU1WnJrk8yW9m2Hbu9CSnzb6gtfblqvqDJK9J8qmqOi3J9zMcpnJokj+bPTVwHHNeVZ2U5DkZjv0+PcOx3o9Kcuskz9joqYGJwBkAgOn+Nsl/zLBn8y9m2OLt6gz1w29J8prW2uWzA1pr76yq+yZ5fpKHJzkgwzHczxn73+TPCK2111bVpRm2vPvdDFUSn0vygtbamxdNrLV2QlV9NkOG+UkZkuz/kOQVrbV3b/B3JxE4AwB0aWfs47xRrbV/SvL0dYz7aIZs9Y6MOSPJGTs45pQkp+zImB2hxhkAACaQcQYA6NBKbfYMmCdwBgDo0O62j/PeQKkGAABMIOMMANAh+eb+yDgDAMAEMs4AAB3qcTu6vZ2MMwAATCDjDADQIbtq9EfgDADQIWFzf5RqAADABDLOAAAdsjiwPzLOAAAwgYwzAECHLA7sj8AZAKBDwub+KNUAAIAJZJwBADpkcWB/ZJwBAGACGWcAgA41Vc7dETgDAHRIqUZ/lGoAAMAEMs4AAB2yj3N/ZJwBAGACGWcAgA7JN/dHxhkAACaQcQYA6JAa5/4InAEAOmQ7uv4o1QAAgAlknAEAOuTkwP7IOAMAwAQyzgAAHVLj3B+BMwBAh5Rq9EepBgAATCDjDADQIaUa/ZFxBgCACWScAQA6tNLUOPdG4AwA0CFhc3+UagAAwAQyzgAAHVqRc+6OjDMAAEwg4wwA0CEHoPRHxhkAACaQcQYA6JADUPojcAYA6JDFgf1RqgEAABPIOAMAdMjiwP7IOAMAwAQyzgAAHbI4sD8CZwCADrWmVKM3SjUAAGACGWcAgA7Zjq4/Ms4AADCBjDMAQIcsDuyPwBkAoEP2ce6PUg0AAJhAxhkAoEMWB/ZHxhkAACaQcQYA6JADUPoj4wwAABMInAEAOrSyEz4bVVWPqKrXVtWHq+rKqmpV9dY1+h423l/rc+o23nN8VX2iqq6qqq1VdU5VPWQb/fepqmdX1QVVdW1VXV5V762qo5fws39AqQYAQIc63Y7uBUl+PslVSb6W5C4TxvxjkncuuP5PizpX1YlJThif/8Yk+yV5dJIzquoZrbWT5/pXklOTPCLJhUlOTnLrJI9Kcm5VPby19q4J89wugTMAAFM9O0NA+6Uk903yoQljPtNae/GUh48Z4hOSXJzkqNbad8frr0hyfpITq+rdrbVLZ4Y9OkPQfF6SB7TWrhvHvCHJR5K8sarObq19b8octkWpBgBAh1bSlv7ZqNbah1prX2w7b+Xik8f2ZatB8/jeS5O8Lsn+SR43N+YpY/uC1aB5HPPJJKcluW2GwHrDBM4AAOxMd6iq36+q543tz22j77Fje9aCe2fO9UlVHZDk6CTXJPnwlDEbsbRSjar67ST3S3JDkrNaax9Yo9/xSY5vrS3lBwAA7Il2RlK3qs7fxvvuvvQXDh44fmbncU6GePArM9cOSnJIkqtaa99Y8Jwvju2RM9cOT7JPkktaazdMHLNuGw6cx4Ls05I8PEmNl59VVe9J8ruttSvmhhyWoSYGAIA17AEnB16T5L9mWBh4yXjt55K8OMn9k3ywqn6htXb1eG/L2G5d43mr1285c209Y9ZtGRnnx2WoG/lqkjck+bckxyd5SJKPVNWxrbVvrvfh2/qT0T43u8N6HwsAsNfZiVnlRe/6ZpI/mrt8blUdl2HR3j2TPCHJq3fVnDZqGTXOj0tyRYaVj3/SWjsxyS8kOSnJTyf526q6zRLeAwCw12g74T89GEsq/mL8ep+ZW6vZ4S1ZbPX6bDXDesas2zIyzndNcvpsVrm1dmOS51bVV5K8KkPwfP/Z1ZFTbetPRvvud0gffwcAALAjvjW2B61eaK1dXVWXJTmkqm6/oM75iLG9aObaxUluTHKnqtp3QZ3zojHrtoyM835J/nXRjdbaa5I8M0M9yweqain1JQAAe7qV1pb+6ci9xvaSuetnj+2DFox58FyfjNvPnZfkwCS/MmXMRiwjcJ9cHWMAAAipSURBVL4syU+sdXM83eU5Se6W5H1ZO5UOAMCo7YTPrlRVd6uqm8SaVfWADAepJMn8cd1vGNvnV9WtZsYcluRpSa5P8qa5Ma8f25eO29Otjjkqw+mB30ry9vX9ih+2jFKNz2ZYGbmm1tqrqmr/JH+S5BeX8E4AAHaxqnpokoeOX283tveuqlPGv/52a+2541+flOSIqjovw2mDyVCFsLol8Qtba+fNPr+1dl5VnZQh6XpBVZ2eobrhURmO0X7G3KmByXDc9sMybFbx6ao6I8nB45h9kjyxtXbl+n/1v1tG4PzeJA+tql9vrb1nrU6ttf9WVfsleUl2/R96AAB2K51uR/cLGXZPm3Wn8ZMk/5xkNXB+S5LfTnJUhpKJm2Uo731bkpNba4sOLElr7YSq+myGDPOTkqwk+Yckr2itvXtB/1ZVj8lQsvH4JM9Icl2Sc5O8dD4434ja6ObaVXXrJI9McmFr7ZwJ/Y9Pclhr7SUbenEsDmTPcPDNf2SzpwBL8fljf3yzpwBLcet3/F1tv9fOd8whxy49zvnoZWd38dt2VxvOOLfWLk/y5zvQ/80bfScAwJ6u04zzXm0ZiwMBAGCPt4waZwAAlmyj5bQsn8AZAKBDSjX6o1QDAAAmkHEGAOhQk3HujowzAABMIOMMANAhiwP7I3AGAOiQxYH9UaoBAAATyDgDAHRIqUZ/ZJwBAGACGWcAgA6pce6PwBkAoEP2ce6PUg0AAJhAxhkAoEMrFgd2R8YZAAAmkHEGAOiQGuf+yDgDAMAEMs4AAB1S49wfgTMAQIeUavRHqQYAAEwg4wwA0CGlGv2RcQYAgAlknAEAOqTGuT8CZwCADinV6I9SDQAAmEDGGQCgQ0o1+iPjDAAAE8g4AwB0qLWVzZ4CcwTOAAAdWlGq0R2lGgAAMIGMMwBAh5rt6Loj4wwAABPIOAMAdEiNc39knAEAYAIZZwCADqlx7o/AGQCgQysC5+4o1QAAgAlknAEAOtQsDuyOjDMAAEwg4wwA0CGLA/sjcAYA6JB9nPujVAMAACaQcQYA6JBSjf7IOAMAwAQyzgAAHXIASn8EzgAAHVKq0R+lGgAAMIGMMwBAh2xH1x8ZZwAAmEDGGQCgQ2qc+yPjDAAAE8g4AwB0yHZ0/RE4AwB0qFkc2B2lGgAAMIGMMwBAh5Rq9EfGGQAAJpBxBgDokO3o+iNwBgDokMWB/VGqAQDADqmqQ6vqr6rq61V1fVVdWlWvqqpbbfbcdiYZZwCADvVaqlFVhyc5L8mPJXlXki8k+aUk/yXJg6rqmNbadzZxijuNjDMAADvi/8oQND+ztfbQ1tofttaOTfLKJHdO8rJNnd1OJHAGAOhQa23pn40as83HJbk0yevmbr8oydVJfqeqDtrwyzokcAYA6FDbCZ8luP/Yvr+1tvJD823te0k+muTAJPdazuv6osYZAGAvUVXnr3WvtXb3CY+489hetMb9L2bISB+Z5IM7Nrv+7daB8w3fv6w2ew57utV/wCb+wwRd8vcxewp/L+9ddkacs63AeaItY7t1jfur12+5wfd0abcOnAEAmM4fujZGjTMAAFOtZpS3rHF/9foVu2Auu5zAGQCAqS4c2yPXuH/E2K5VA71bEzgDADDVh8b2uKr6oTiyqn4kyTFJrkny97t6YruCwBkAgElaaxcneX+Sw5I8be72S5IclOQtrbWrd/HUdgmLAwEA2BFPzXDk9muq6gFJPp/knhn2eL4oyfM3cW47VfV6DjoAAH2qqjsm+eMkD0pycJJvJHlHkpe01r67mXPbmQTOAAAwgRpnAACYQOAMAAATCJwBAGACgTMAAEwgcAYAgAkEzgAAMIHAmYWq6tCq+quq+npVXV9Vl1bVq6rqVps9N5iiqh5RVa+tqg9X1ZVV1arqrZs9L9gRVXVwVT2hqt5RVV+qqmuramtVfaSq/vP8kcfAzmUfZ26iqg7PcCLQjyV5V5IvJPmlDCcCXZjkmNbadzZvhrB9VfWZJD+f5KokX0tylyT/s7X22E2dGOyAqnpyktdnOFziQ0m+kuTHkzwsyZYkb0/yyOZf5rBLCJy5iap6X5LjkjyztfbamesnJXl2kj9vrT15s+YHU1TV/TMEzF9Kct8MQYfAmd1KVR2b5KAk72mtrcxcv12STyS5Y5JHtNbevklThL2K/4uHHzJmm49LcmmS183dflGSq5P8TlUdtIunBjuktfah1toXZeLYnbXWzm6tnTEbNI/X/yXJG8av99vlE4O9lMCZefcf2/cv+B/q7yX5aJIDk9xrV08MgB/yb2N7w6bOAvYiAmfm3XlsL1rj/hfH9shdMBcAFqiqfZP87vj1rM2cC+xNBM7M2zK2W9e4v3r9lrtgLgAs9qdJfjbJe1tr79vsycDeQuAMALuRqnpmkhMy7Hj0O5s8HdirCJyZt5pR3rLG/dXrV+yCuQAwo6qenuTVST6X5P6ttcs3eUqwVxE4M+/CsV2rhvmIsV2rBhqAnaCqnpXktUn+KUPQ/C+bPCXY6wicmfehsT1u/kSqqvqRJMckuSbJ3+/qiQHsrarq/0zyyiSfyRA0f3OTpwR7JYEzP6S1dnGS9yc5LMnT5m6/JMNG/G9prV29i6cGsFeqqhdmWAx4fpIHtNa+vclTgr2WkwO5iQVHbn8+yT0z7PF8UZKjHblN76rqoUkeOn69XZJfTXJJkg+P177dWnvuZswNpqqq45OckuTGDGUai3Y8urS1dsounBbstQTOLFRVd0zyx0kelOTgJN9I8o4kL2mtfXcz5wZTVNWLM5x2uZZ/bq0dtmtmA+sz4e/jJPm71tr9dv5sAIEzAABMoMYZAAAmEDgDAMAEAmcAAJhA4AwAABMInAEAYAKBMwAATCBwBgCACQTOAAAwgcAZAAAmEDgDAMAEAmcAAJhA4AwAABMInAEAYAKBMwAATCBwBgCACQTOAAAwgcAZAAAm+P8BLXuWX91NyYIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 248,
       "width": 359
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#model = models.resnet50(pretrained=True)\n",
    "#model = timm.create_model('resnet50', pretrained=True)\n",
    "model = timm.create_model('efficientnet_b0', pretrained=True)\n",
    "#model.fc #show fully connected layer for ResNet family\n",
    "model.classifier #show the classifier layer (fully connected layer) for EfficientNets\n",
    "\n",
    "# Create classifier\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = True\n",
    "# define `classifier` for ResNet\n",
    "# Otherwise, define `fc` for EfficientNet family \n",
    "#because the definition of the full connection/classifier of 2 CNN families is differnt\n",
    "fc = nn.Sequential(OrderedDict([('fc1', nn.Linear(2048, 1000, bias=True)),\n",
    "\t\t\t\t\t\t\t     ('BN1', nn.BatchNorm2d(1000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)),\n",
    "\t\t\t\t\t\t\t\t ('dropout1', nn.Dropout(0.7)),\n",
    "                                 ('fc2', nn.Linear(1000, 512)),\n",
    "\t\t\t\t\t\t\t\t ('BN2', nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)),\n",
    "\t\t\t\t\t\t\t\t ('swish1', Swish()),\n",
    "\t\t\t\t\t\t\t\t ('dropout2', nn.Dropout(0.5)),\n",
    "\t\t\t\t\t\t\t\t ('fc3', nn.Linear(512, 128)),\n",
    "\t\t\t\t\t\t\t\t ('BN3', nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)),\n",
    "\t\t\t\t\t\t\t     ('swish2', Swish()),\n",
    "\t\t\t\t\t\t\t\t ('fc4', nn.Linear(128, 3)),\n",
    "\t\t\t\t\t\t\t\t ('output', nn.Softmax(dim=1))\n",
    "\t\t\t\t\t\t\t ]))\n",
    "# connect base model (EfficientNet_B0) with modified classifier layer\n",
    "model.fc = fc\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "#optimizer = Nadam(model.parameters(), lr=0.001)\n",
    "#optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=0.0001)\n",
    "optimizer = optim.SGD(model.parameters(), \n",
    "                      lr=0.01,momentum=0.9,\n",
    "                      nesterov=True,\n",
    "                      weight_decay=0.0001)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)\n",
    "#show our model architechture and send to GPU\n",
    "model.to(device)\n",
    "CHECK_POINT_PATH = '/home/linh/Downloads/Covid-19/weights/EfficientNet_B0_Covid-19.pth'\n",
    "try:\n",
    "    checkpoint = torch.load(CHECK_POINT_PATH)\n",
    "    print(\"checkpoint loaded\")\n",
    "except:\n",
    "    checkpoint = None\n",
    "    print(\"checkpoint not found\")\n",
    "\n",
    "def load_model(path):                                \n",
    "  model.load_state_dict(checkpoint['model_state_dict'])\n",
    "  best_model_wts = copy.deepcopy(model.state_dict())\n",
    "  optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "  scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n",
    "  best_loss = checkpoint['best_val_loss']\n",
    "  best_acc = checkpoint['best_val_accuracy']\n",
    "load_model(CHECK_POINT_PATH)  \n",
    "\n",
    "since = time.time()\n",
    "model.eval()\n",
    "y_test = []\n",
    "y_pred = []\n",
    "for images, labels in data_loader['data']:\n",
    "    images = Variable(images.cuda())\n",
    "    labels = Variable(labels.cuda())\n",
    "    outputs = model(images)\n",
    "    _, predictions = outputs.max(1)\n",
    "    \n",
    "    y_test.append(labels.data.cpu().numpy())\n",
    "    y_pred.append(predictions.data.cpu().numpy())\n",
    "    \n",
    "y_test = np.concatenate(y_test)\n",
    "y_pred = np.concatenate(y_pred)\n",
    "pd.DataFrame({'true_label':y_test,'predicted_label':y_pred}).to_csv('/home/linh/Downloads/Covid-19/results/Modified_EfficienNet_B0_Covid-19_Whole_Data.csv',index=False)\n",
    "\n",
    "time_elapsed = time.time() - since\n",
    "\n",
    "print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "sns.heatmap(confusion_matrix(y_test, y_pred))\n",
    "accuracy_score(y_test, y_pred)\n",
    "\n",
    "report = classification_report(y_test, y_pred)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EfficientNet_B4 with image size=380"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction time for complete in 17995 milisecond\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0   0.000000  0.000000  0.000000        10\n",
      "           1   0.908894  0.946893  0.927504       885\n",
      "           2   0.930657  0.858586  0.893170       594\n",
      "           3   0.000000  0.000000  0.000000         0\n",
      "\n",
      "    accuracy                       0.905306      1489\n",
      "   macro avg   0.459888  0.451370  0.455169      1489\n",
      "weighted avg   0.911472  0.905306  0.907578      1489\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/linh/.conda/envs/CV/lib/python3.6/site-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAHwCAYAAABHf7LhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de7iudV0n/vdHEEwsMMwUcAY10EbHSjAVz5ikZY4lMznXWFQ/M/M0nrpqPGtaNp6VftpYo6nzG3Q0dfBcIqKimXigxkZUxAN5wAMooCCsz++P51myXKy1WXuv597Pvfb9enE91826j9/nuva192e/9+f+fqu7AwAAU3KtZQ8AAAD2NkUwAACTowgGAGByFMEAAEyOIhgAgMlRBAMAMDmKYAAAJkcRDADA5CiCAQCYHEUwAACTowgGAGByFMEAAEyOIhgAgMnZf9kD2I79Dzi8lz0GxudaVcseAiO00n67ALbmisvPH8UfJN//+rkL/43r2je42Si+2xhIggEAmJwdnQQDAOyzVq5c9gj2aZJgAAAmRxIMADBGvbLsEezTFMEAAGO0oggeknYIAAAmRxIMADBCrR1iUJJgAAAmRxIMADBGeoIHpQgGABgj7RCD0g4BAMDkSIIBAMbIinGDkgQDADA5kmAAgDHSEzwoSTAAAJMjCQYAGCNTpA1KEQwAMEJWjBuWdggAACZHEgwAMEbaIQYlCQYAYHIkwQAAY6QneFCKYACAMbJi3KC0QwAAMDmSYACAMdIOMShJMAAAkyMJBgAYI1OkDUoRDAAwRtohBqUdAgCAyZEEAwCMkXaIQUmCAQCYHEkwAMAIdVssY0iKYACAMfJi3KC0QwAAMDmSYACAMfJi3KAkwQAATI4kGABgjPQED0oSDADA5EiCAQDGaMUUaUNSBAMAjJF2iEFphwAAYEuq6reqqq/hc+Wa84+8hnNP2cWzTqqqD1fVxVV1UVWdXlX3XdR3kQQDAIzROKdI+3iSp29y7C5Jjk/y9g2OfSLJmzbY/08b3aiqnpvkcUm+lOTlSQ5I8sAkp1bVI7v75N0c99UoggEA2JLu/nhmhfDVVNUH5//73zY4/PHuftpWnlFVx2VWAH82ye26+1vz/c9JclaS51bVW7r7vN0b/Q/TDgEAMEa9svjPQKrq3ya5Q5Lzk7x1m7d76Hz7rNUCOEnmRe+fJzkwyW9v8xmSYACAURqgHaKqztrsWHcfs41bP2S+/avu3mhai8Oq6veSHJrkG0k+2N1nb3Kv4+fbd2xw7O1Jnjw/56nbGK8iGACAPVdVP5LkQUmuTPKXm5x2r/ln7XWnJzmpu7+wZt9BSQ5PcnF3f3mD+3x6vj16m8NWBAMAjNIASfA2097N/IckhyR5a3d/cd2xS5P8cWYvxZ0733ebJE9Lco8k766qn+3uS+bHDp5vL9rkWav7D9nuoPUEAwCwHautEH+x/kB3f627n9LdH+3uC+efM5KckOTvk/xUkgfvxbH+gCIYAGCEuq9c+GfRqupWSY7LbCqzt239u/UVuap14q5rDq0mvQdnY6v7L9yNYW5IOwQAwBiNc57g9a7phbhduWC+PWh1R3dfUlXnJzm8qm68QV/wUfPtObs/1B+20CJ43hh9h8yalVd7NS7MbKAf6u7vLvJ5AAAsR1VdJ8lvZPZC3F/twS3uMN+eu27/afP73jvJK9Ydu8+ac7ZlIUVwVV0/ybMyG/B1Nznt0qp6VZInrZ3zDQCADQw4r++C/Psk10/ylg1eiEuSVNVtM1soY2Xd/nsmecz8x9esu+xlmdWUT6yqN61ZLOPIJA9PclmuXhzvtm0XwVV1SJIPJLllkkuS/G1m01es7ek4Ksmdkvx+kntU1R27e7O3/gAAGL/VVoiNVohb9fwkR1XVmZn1DSez2SFW5wJ+cnefufaC7j6zqp6f5LFJzq6q12e2bPKvJ/nxJI/c7mpxyWKS4KdmVgC/IMlTu/vijU6qqusleUaSRyd5SmbL4V2jXU3qvN+1D9vtwQIA7Agj7gmuqp9Ocudc8wtxr07yq0lul1krw7WTfDXJ65Kc3N3v2+ii7n5cVf1jZsnvQ5KsJPlokud091sW8h26e3s3qPpcks929y9s8fzTkty0u2+6xfN3VQTfdmujZEquVbXsITBCK9v8vQ6YjisuP38Uf5B8993/beG/cf3IPR8yiu82BotIgm+c5H/uxvkfymwqjS3Z1aTO+x9wuD/VAIB90/h7gne0RRTB30hyi904/6fn1wAAsJkRt0PsCxaxWMY7k9y/qh52TSdW1SOS3C/JOxbwXAAA2COLSIKfnOSXk7ykqh6X5F2ZzQu8dnaIozNbHu/IJF/L7MU4AAA2ox1iUNsugrv7/Kq6Y5KXJrlXkt9Lsr5Xd7UJ+11JHtbd52/3uQAAsKcWslhGd5+b5Ber6mZJ7pFZj/Dq2s4XJflUkvfMzwMA4JroCR7UQpdNnhe5Cl0AgO1SBA9qES/GAQDAjrLQJBgAgAXxYtygJMEAAEyOJBgAYIz0BA9KEQwAMEbaIQalHQIAgMmRBAMAjJF2iEFJggEAmBxJMADAGOkJHpQkGACAyZEEAwCMkZ7gQSmCAQDGSBE8KO0QAABMjiQYAGCMupc9gn2aJBgAgMmRBAMAjJGe4EEpggEAxkgRPCjtEAAATI4kGABgjKwYNyhJMAAAkyMJBgAYIz3Bg1IEAwCMkXmCB6UdAgCAyZEEAwCMkXaIQUmCAQCYHEkwAMAYSYIHJQkGAGByJMEAAGNksYxBKYIBAEaoV0yRNiTtEAAATI4kGABgjLwYNyhJMAAAkyMJBgAYIy/GDUoRDAAwRl6MG5R2CAAAJkcSDAAwRl6MG5QkGACAyZEEAwCMkSR4UIpgAIAxai/GDUk7BAAAkyMJBgAYI+0Qg5IEAwAwOZJgAIAxsljGoCTBAABMjiQYAGCMWk/wkBTBAABjpB1iUDu6CL5W1bKHwAhdcv4Zyx4CI3TDI09Y9hAYoe9c/t1lDwFYkh1dBAMA7KvaFGmD8mIcAACTIwkGABgjPcGDUgQDAIyR2SEGpR0CAIDJkQQDAIyRdohBSYIBAJgcSTAAwBiZIm1QkmAAgDFa6cV/Fqiq7llVb6yqr1TVZVX1L1X1zqr6pQ3OPa6q3lZV36yq71bV2VX16Krabxf3v29VnV5VF1XVxVX191V10qLGrwgGAGC3VNV/TfJ3SY5N8r+TPC/JW5P8RJK7rzv33yU5I8ldk7wxyclJDkjygiSnbHL/RyQ5Ncmtk7wmycuTHJbklVX13EV8B+0QAABjNNIp0qrqd5P8QZK/TvKQ7r583fFrr/n/H8usgL0yyd27+yPz/U9OclqSE6vqgd19ypprjkzy3CTfTHJsd5833/+MJP+Q5HFV9Ybu/uB2vockGACALamqA5M8K8kXskEBnCTd/f01P56YWTp8ymoBPD/ne0meNP/x99fd4neSHJjk5NUCeH7Nt5L8yfzHh27vm0iCAQDGaZxTpN0rs6L2hUlWquqXM2tZ+F6SD2+Qzh4/375jg3udkeTSJMdV1YHdfdkWrnn7unP2mCIYAGAiquqszY519zFbuMXt5tvvJflYZgXw2vufkeTE7r5gvusW8+05Gzzviqr6XJJbJblZkn/ewjVfrqpLkhxRVdft7ku3MOYNaYcAABihXllZ+GcBbjjf/kGSTnKXJD+a5DZJ3pXZy2//a835B8+3F21yv9X9h+zBNQdvcnxLJMEAAGM0QDvEFtPeXVkNUK9Icr81Pbv/WFW/muRTSe5WVXfc7otrQ5MEAwCwVRfOtx9b+9JaksxbE945//Hn59trSm1X91+4Zt9Wr9ksKd4SRTAAwBiNc7GMT823F25y/Fvz7Y+sO//o9SdW1f5JbppZqnzuBs/Y6JobJzkoyZe20w+cKIIBANi6d2fWC/xvqmqjOnL1RbnPzbenzbf33uDcuya5bpIz18wMcU3X3GfdOXtMEQwAMEa9svjPdofU/fnMVnL7V0n+89pjVXVCkl/MLCVend7s9Um+nuSBVXXsmnOvk+SZ8x9fuu4xr0hyWZJHzBfOWL3m+kmeMP/xZdv9Ll6MAwAYo3HOE5wkD0/yc0meP58n+GOZtTXcP7OV4R7c3RclSXd/e77C3OuTnF5Vp2S2Etz9MpsK7fVJXrv25t39uar6gyQvTvKRqnptksszW3jjiCTPW8RLd4pgAAC2rLu/VFXHJHlKZsXsXZN8O7OE+E+7+8Przn9TVd0tyROTPCDJdZJ8Jsljk7y4u69W7Xf3S6rqvCSPT/KbmXUvfDLJk7r7rxfxPRTBAAAj1ONNgjNfDOOR889Wzv9Akl/azWecmllhPQg9wQAATI4kGABgjEacBO8LFMEAAGO0mGWO2YR2CAAAJkcSDAAwRtohBiUJBgBgciTBAABjJAkelCQYAIDJkQQDAIzQBgupsUCKYACAMdIOMSjtEAAATI4kGABgjCTBg5IEAwAwOZJgAIARaknwoBTBAABjpAgelHYIAAAmRxIMADBGK8sewL5NEgwAwORIggEARsiLccNSBAMAjJEieFDaIQAAmBxJMADAGHkxblCSYAAAJkcSDAAwQl6MG5YkGACAyZEEAwCMkZ7gQSmCAQBGSDvEsLRDAAAwOZJgAIAx0g4xqKUkwVX1nKr67DKeDQAAy0qCb5DkyK2cWFVnbXbs2gccvqjxAACMSkuCB6UdAgBgjBTBg1pIEVxVr9rNS47b6ondfcxmxw448AivTQIAsNsWlQQ/KEknqd24RgELALAJ7RDDWlQR/J0kX0rysC2e/0dJTljQswEAYLcsqgj+RJKf6e73buXkqvqtBT0XAGDfJAke1KKmSPt4kutV1c0XdD8AgEnrlcV/uMqikuD3JrlLkiOSbGX+3zclOW9BzwYAgN2ykCK4u9+Q5A27cf6bk7x5Ec8GANgXSW6HtZQV4wAAYJkslgEAMEKS4GEpggEAxqh3Z/kFdpd2CAAAJkcSDAAwQtohhiUJBgBgciTBAAAj1Ct6gockCQYAYHIkwQAAI6QneFiKYACAEWpTpA1KOwQAAJMjCQYAGCHtEMOSBAMAMDmSYACAETJF2rAUwQAAI9S97BHs27RDAAAwOZJgAIAR0g4xLEkwAACTIwkGABghSfCwFMEAACPkxbhhaYcAAGByJMEAACOkHWJYkmAAACZHEgwAMELdkuAhSYIBANhjVfWgqur558Hrjt19zbGNPs/e5J77VdVjqursqvpuVX2zqt5WVcctatySYACAEeqVZY/gmlXVTZKcnOTiJNfbxanvTXL6Bvvfv8E9K8kpSU5M8qn5/X88ya8nOaOqHtDdb97eyBXBAACjtDLydoh5sfqKJN9I8jdJHr+L00/v7qdt8dYPzKwAPjPJPbv7e/PnvSyzovnlVXVad39nT8eeaIcAAGDPPCrJ8Ul+O8klC7zv78+3T1otgJOku/8hyWuT/ERmRfK2KIIBAEaouxb+WZSq+ukkz07you4+YwuX/FRVPaKqnlBVv1NVR21y3+skOS7JpUnet8Epb59vj9+Tca+lHQIAYCKq6qzNjnX3MVu8x/5JXp3kC0mesMVH/6f5Z+193pDkd7v7W2t23zzJfknO7e4rNrjPp+fbo7f43E0pggEARmjEi2U8JcnPJblzd3/3Gs69IMkfJXlrkvOSXCfJsUn+JMkDktyoqu7a/YPXAA+eby/a5H6r+w/Zs6FfRREMADBC3UPcc2tp72aq6vaZpb/P6+4PbuF5/yfJ/1mz6+Ik76iqM5N8PMmdkvxKkm3P9rC79AQDAHCN5m0Qr0pyTpInb+de3f3tJP/f/Me7rjm0mvQenI2t7r9wO89PJMEAAKM0wnaI6+WqXtzvzWZIu5qXV9XLM3th7tHXcL8L5tuD1uz7bJIrk9ysqvbfoC949YW6c7Y+7I0pggEA2IrLkvzVJsdum1mf8PszW+DiGlslktxhvj13dUd3f2/eKnGX+ec96665z3x72hbHvClFMADACI1tsYz5S3AP3uhYVT0tsyL4r7v7L9fsP7a7P7LB+Q/KbAW4y5O8bt3hl2ZWAD+zqtYulnG7+TUXJHnDdr+PIhgAYIQWOa/vEr2+qq5I8pEkX8psdojbJfn5JFck+b3uPm/dNack+bXMFsT4WFWdmuTQzArg/TKbVu3b2x2YIhgAgKG8NMkvZDYLxA2SVJLzk7wyyQu7+xPrL+jurqr/mNmyyb+T5JFJvpfkjCTP7O4zFzGw6iHm39hLDjjwiJ07eAZzyflbWbiGqbnhkScsewiM0Hcuv6YpTpmiKy4/fxQR7NlH/srC65zbnHfqKL7bGJgiDQCAydEOAQAwQmN7MW5fIwkGAGByJMEAACO0j8wOMVqKYACAEdrBcxfsCNohAACYHEkwAMAIeTFuWDu6CF7x7wRs4Ja3PHHZQ2CEPnX7w5c9BEbo8Pd9ZtlDAJZkRxfBAAD7Ki/GDUsRDAAwQtohhuXFOAAAJkcSDAAwQt58GpYkGACAyZEEAwCMkJ7gYSmCAQBGyOwQw9IOAQDA5EiCAQBGaGXZA9jHSYIBAJgcSTAAwAh19AQPSRIMAMDkSIIBAEZoxWoZg1IEAwCM0Ip2iEFphwAAYHIkwQAAI+TFuGFJggEAmBxJMADACFksY1iKYACAEdIOMSztEAAATI4kGABghLRDDEsSDADA5EiCAQBGSBI8LEUwAMAIeTFuWNohAACYHEkwAMAIrQiCByUJBgBgciTBAAAjtKIneFCSYAAAJkcSDAAwQr3sAezjFMEAACNknuBhaYcAAGByJMEAACO0Ul6MG5IkGACAyZEEAwCMkBfjhqUIBgAYIS/GDUs7BAAAkyMJBgAYoRXvxQ1KEgwAwORIggEARmglouAhKYIBAEbI7BDD0g4BAMDkSIIBAEbIi3HDkgQDADA5kmAAgBGyWMawJMEAAEyOJBgAYITMDjEsRTAAwAh5MW5Y2iEAAJgcSTAAwAh5MW5YC0uCq+pXq+pFVfW8qrrXLs47qapOW9RzAQBgd207Ca6qSvLaJA9IfrDI9aOr6q1JfrO7L1x3yZFJ7rbd5wIA7MskwcNaRDvEbyc5MckXk7wsyfeTnJTkvkneX1XHd/fXFvAcAIDJaC/GDWpRRfCFSW63WuxW1QuS/FmSxyb5u3kh/PU9uXlVnbXZsf2ufdie3BIAgIlbRE/wv03yN2vT3u6+srsfn+TRSW6dWSF8/QU8CwBgElYG+HCVRRTBByT56kYHuvvFSR6V5DZJ/raqDtndm3f3MZt9tjdsAAB2V1X9WVW9u6q+WFXfrapvVtXHquqpVXXoJtccV1Vvm5/73ao6u6oeXVX77eI5962q06vqoqq6uKr+vqpOWtT3WEQRfH6Sf7XZwe4+ObO2iNsmeWeSgxfwTACAfdqIk+DHJDkoyd8meVGS/5HkiiRPS3J2Vd1k7clV9e+SnJHkrknemOTkzELUFyQ5ZaMHVNUjkpyaWUfBa5K8PMlhSV5ZVc9dxJdYRE/wPya5x65O6O4XVtWBSf40yc8t4JkAAPu0ES+b/GPd/b31O6vqWUmekOS/JHnYfN+PZVbAXpnk7t39kfn+Jyc5LcmJVfXA7j5lzX2OTPLcJN9Mcmx3nzff/4wk/5DkcVX1hu7+4Ha+xCKS4LclOayqfnlXJ3X3nyV5aizQAQCwY21UAM+9br49as2+E5P8RJJTVgvgNfd40vzH3193n99JcmCSk1cL4Pk130ryJ/MfH7pHg19jEQXp3yTZL8kl13Rid/9xVX0hs7mCAQDYxMrOmyLtV+bbs9fsO36+fccG55+R5NIkx1XVgd192Rauefu6c/bYtovg7v5mkr/YjfP/ervPBABg9+1q6tndnXSgqh6f5HqZve91bJI7Z1YAP3vNabeYb8/Z4HlXVNXnktwqyc2S/PMWrvlyVV2S5Iiqum53X7o7Y15LawIAwAjtgCnNHp/kJ9f8/I4kv9XdF6zZtzohwkWb3GN1/9oZxLZyzUHz8xTBAADs2iKnmO3uGyVJVf1kkuMyS4A/VlX37e6PLuo5Q1nEi3EAACzYiKdI+yHd/dXufmOSE5IcmuRVaw6vprmbTZG7uv/CPbhms6R4SxTBAAAj1AN8Bh1v9+eTfDLJrarqBvPdn5pvj15/flXtn+Smmc0xfO6aQ7u65saZtUJ8aTv9wIkiGACAxTlsvr1yvj1tvr33BufeNcl1k5y5ZmaIa7rmPuvO2WOKYACAEVqpxX+2q6qOrqqrtSlU1bXmi2XcMLOi9lvzQ69P8vUkD6yqY9ecf50kz5z/+NJ1t3tFksuSPGK+cMbqNdfPbDGOJHnZdr+LF+MAANiqX0ryp1X1/iSfS/KNzGaIuFtm05x9Jcnvrp7c3d+uqt/NrBg+vapOyWwluPtlNhXa65O8du0DuvtzVfUHSV6c5CNV9dokl2e28MYRSZ633dXiEkUwAMAojXSKtL9L8lOZzQn8c5lNbXZJZnP6vjrJi+drSPxAd7+pqu6W5IlJHpDkOkk+k+Sx8/Ov1q7c3S+pqvMym4btNzPrXvhkkictas0JRTAAwAgN/SLbnujuf0ryiD247gOZpci7c82pSU7d3WdtlZ5gAAAmRxIMADBCK6PMgvcdkmAAACZHEgwAMEIjfTFun6EIBgAYIc0Qw9IOAQDA5EiCAQBGSDvEsCTBAABMjiQYAGCEVmrZI9i3KYIBAEbIPMHD0g4BAMDkSIIBAEZIDjwsSTAAAJMjCQYAGCFTpA1LEgwAwORIggEARsjsEMNSBAMAjJASeFjaIQAAmBxJMADACHkxbliSYAAAJkcSDAAwQl6MG5YiGABghJTAw9IOAQDA5EiCAQBGyItxw5IEAwAwOZJgAIARal3Bg1IEAwCMkHaIYWmHAABgciTBAAAjZJ7gYUmCAQCYHEkwAMAIyYGHJQkGAGByJMEAACOkJ3hYimAAgBEyRdqwtEMAADA5kmAAgBGyYtywJMEAAEyOJJh9zue//dVlD4EROux9fl0AO4ue4GEpggEARkg7xLC0QwAAMDmSYACAEdIOMSxJMAAAkyMJBgAYoZXWEzwkRTAAwAgpgYelHQIAgMmRBAMAjNCKLHhQkmAAACZHEgwAMEIWyxiWJBgAgMmRBAMAjJDFMoalCAYAGCEvxg1LOwQAAJMjCQYAGCEvxg1LEgwAwORIggEARsiLccNSBAMAjFC3doghaYcAAGByJMEAACNkirRhSYIBAJgcSTAAwAh5MW5YimAAgBEyT/CwtEMAADA5kmAAgBHyYtywJMEAAGxJVZ1YVS+pqvdV1berqqvqNZuce+T8+GafU3bxnJOq6sNVdXFVXVRVp1fVfRf5XSTBAAAjNNLFMp6U5GeSXJzkS0luuYVrPpHkTRvs/6eNTq6q5yZ53Pz+L09yQJIHJjm1qh7Z3SfvwbivRhEMAMBWPSaz4vQzSe6W5D1buObj3f20rdy8qo7LrAD+bJLbdfe35vufk+SsJM+tqrd093m7P/Qfph0CAGCEVgb4bFd3v6e7P93DxdQPnW+ftVoAz597XpI/T3Jgkt9exIMUwQAAI9QD/Lckh1XV71XVE+bb2+zi3OPn23dscOzt687ZFu0QAAATUVVnbXasu48Z6LH3mn/WjuP0JCd19xfW7DsoyeFJLu7uL29wn0/Pt0cvYlCSYACAEVpJL/yzl12a5I+THJPk+vPPah/x3ZO8e174rjp4vr1ok/ut7j9kEYOTBAMATMSAae9Gz/pakqes231GVZ2Q5P1Jbp/kwUletLfGtJYkGABghLp74Z8x6O4rkvzl/Me7rjm0mvQenI2t7r9wEeOQBAMAjNA+vmLcBfPtD9ohuvuSqjo/yeFVdeMN+oKPmm/PWcQAJMEAAOxtd5hvz123/7T59t4bXHOfdedsiyIYAGCEdvoUaVV126q6Wq1ZVffMbNGNJFm/5PLL5tsnVtX111xzZJKHJ7ksySsWMT7tEAAAbElV3T/J/ec/3mi+vWNVvXL+/1/v7sfP///5SY6qqjMzW2UuSW6Tq+b5fXJ3n7n2/t19ZlU9P8ljk5xdVa/PbNnkX0/y40keuYjV4hJFMADAKK2M5EW2dX42yUnr9t1s/kmSzydZLYJfneRXk9wus1aGayf5apLXJTm5u9+30QO6+3FV9Y+ZJb8PyWyxu48meU53v2VRX6TG8qbgntj/gMN37uABgFG64vLza9ljSJK7HH7Phdc57zv/3aP4bmOgJxgAgMnRDgEAMEL7+BRpSycJBgBgciTBAAAjJAkeliQYAIDJkQQDAIzQTp7BaydQBAMAjJB2iGFphwAAYHIkwQAAI9SS4EFJggEAmJy9kgRX1Y8nubK7L9obzwMA2Om8GDeshSTBVXV4VZ1cVe+sqv9aVYfO9/9sVZ2d5IIk36yqM6rqlot4JgDAvmwlvfAPV9l2EjxPeT+U5PD5rnsl+YWq+sUkb01yaJJPJDksyZ2T/F1V3bq7L9zuswEAYE8soh3iEZkVwH+S5HVJ7pfkGUlemeQ7Se7Y3V9Ikqp6VpL/kuSRSf54KzevqrM2O7bftQ/bzrgBAEZLO8SwFtEOcf8k/9DdT+rus7v7mUnen+TeSf5wtQCee1KSc5PcdwHPBQCAPbKIJPhfJ3nNun0fSXKnJGeu3dndXVXvTfJrW715dx+z2bH9DzjcX5EAgH2SHt5hLaII/pEkl6zbd1GSdPcFG5z/1SQHLeC5AAD7LPMED2sR7RBfT3LDdfsuSfK1Tc4/NImX4gAAWJpFFMHnJPk3a3d093O7+8abnH/TJF9awHMBAPZZK90L/3CVRRTBZyU5pqoOuKYTq+rHMpsm7f0LeC4AAOyRbRfB3f2H3X1gd1++hdN/MrMp0v5iu88FANiX9QD/cZW9smzyqu7+dJIX7c1nAgDAenu1CAYAYGv08A5LEQwAMELaF4a1iBfjAABgR5EEAwCMkHaIYUmCAQCYHEkwAMAI6QkeliIYAGCEtEMMSzsEAACTIwkGABgh7RDDkgQDADA5kmAAgBHqXln2EPZpimAAgBFa0Q4xKO0QAABMjiQYAGCE2hRpg5IEAwAwOZJgAIAR0hM8LEkwAACTIwkGABghPcHDUgQDAIzQiiJ4UNohAMW88zAAAAbwSURBVACYHEkwAMAItRfjBiUJBgBgciTBAAAj5MW4YSmCAQBGyDzBw9IOAQDA5EiCAQBGSDvEsCTBAABMjiQYAGCELJYxLEUwAMAIaYcYlnYIAAAmRxIMADBCpkgbliQYAIDJkQQDAIyQnuBhSYIBAJgcSTAAwAiZIm1YimAAgBFqL8YNSjsEAACTIwkGABgh7RDDkgQDADA5kmAAgBEyRdqwFMEAACPkxbhhaYcAAGC3VNURVfXfq+pfquqyqjqvql5YVddf9ti2ShIMADBCY22HqKqbJzkzyQ2TvDnJ/03y80n+c5J7V9WduvsbSxzilkiCAQDYHf9vZgXwo7r7/t39R919fJIXJLlFkmctdXRbVGP9W8ZW7H/A4Tt38ADAKF1x+fm17DEkybUHqHO+v83vNk+BP5PkvCQ37+6VNcd+NMmXk1SSG3b3Jdt51tAkwQAAI9QDfBbgHvPtu9YWwEnS3d9J8oEk101yh8U8bjh6ggEAJqKqztrsWHcfs4Vb3GK+PWeT459OckKSo5O8e/dGt3ft6CJ4LP9csWyrv6C3+IuXifDrgo34dcFG/LoYpyHqnF0VwVt08Hx70SbHV/cfss3nDG5HF8EAAGydv+hcRU8wAABbtZr0HrzJ8dX9F+6FsWyLIhgAgK361Hx79CbHj5pvN+sZHg1FMAAAW/We+faEqvqhOnI+Rdqdklya5EN7e2C7SxEMAMCWdPdnk7wryZFJHr7u8NOTHJTk1WOfIzjxYhwAALvnYZktm/ziqrpnkn9OcvvM5hA+J8kTlzi2LdvRK8YBALD3VdVNkjwjyb2THJrZSnFvTPL07v7WMse2VYpgAAAmR08wAACTowgGAGByFMEAAEyOIhgAgMlRBAMAMDmKYAAAJkcRvINV1RFV9d+r6l+q6rKqOq+qXlhV11/22FiOqjqxql5SVe+rqm9XVVfVa5Y9Lpanqg6tqgdX1Rur6jNV9d2quqiq3l9V/8/6ZU+Zjqr6s6p6d1V9cf7r4ptV9bGqempVHbrs8cHQzBO8Q1XVzTNbreWGSd6c5P8m+fnMVmv5VJI7dfc3ljdClqGqPp7kZ5JcnORLSW6Z5H9094OWOjCWpqoemuSlmU1k/54kX0jyk0l+LcnBSd6Q5N+3Pwwmp6ouT/LRJJ9M8rXMlru9Q5Jjk/xLkjt09xeXN0IYliJ4h6qqdyY5Icmjuvsla/Y/P8ljkvxFdz90WeNjOarqHpkVv59JcrfMih5F8IRV1fGZFTdv7e6VNftvlOTDSW6S5MTufsOShsiSVNV1uvt7G+x/VpInJHlpdz9s748M9g7/DLYDzVPgE5Kcl+TP1x1+apJLkvxGVR20l4fGknX3e7r701I9VnX3ad196toCeL7/K0leNv/x7nt9YCzdRgXw3Ovm26P21lhgGRTBO9M95tt3bfAH23eSfCDJdTP7Zy2AzXx/vr1iqaNgbH5lvj17qaOAge2/7AGwR24x356zyfFPZ5YUH53k3XtlRMCOUlX7J/nN+Y/vWOZYWK6qenyS62XWI35skjtnVgA/e5njgqEpgnemg+fbizY5vrr/kL0wFmBnenaSWyd5W3e/c9mDYaken9nLkqvekeS3uvuCJY0H9grtEAATU1WPSvK4zGaV+Y0lD4cl6+4bdXcluVFms4bcLMnHquq2yx0ZDEsRvDOtJr0Hb3J8df+Fe2EswA5SVY9I8qLMpsW6R3d/c8lDYiS6+6vd/cbM2ukOTfKqJQ8JBqUI3pk+Nd8evcnx1Td6N+sZBiaoqh6d5CVJ/imzAvgrSx4SI9Tdn8/sL0m3qqobLHs8MBRF8M70nvn2hPWrPVXVjya5U5JLk3xobw8MGKeq+sMkL0jy8cwK4K8teUiM22Hz7ZVLHQUMSBG8A3X3Z5O8K8mRSR6+7vDTM5sY/9XdfcleHhowQlX15MxehDsryT27++tLHhJLVlVHV9XVWuqq6lrzxTJumOTM7v7W3h8d7B1WjNuhNlg2+Z+T3D6zOYTPSXKcZZOnp6run+T+8x9vlOQXk5yb5H3zfV/v7scvY2wsR1WdlOSVmSV6L8nGs8qc192v3IvDYsnmrTF/muT9ST6X5BuZzRBxt8xejPtKZn9h+uTSBgkDUwTvYFV1kyTPSHLvzF5i+HKSNyZ5ur+9T1NVPS2zVQM38/nuPnLvjIYx2MKviSR5b3ffffjRMBZVdeskD81sTuAjMptS85LMQpS3JnmxlybZ1ymCAQCYHD3BAABMjiIYAIDJUQQDADA5imAAACZHEQwAwOQoggEAmBxFMAAAk6MIBgBgchTBAABMjiIYAIDJUQQDADA5imAAACZHEQwAwOQoggEAmBxFMAAAk6MIBgBgchTBAABMzv8PZ6rAa6qR19wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 248,
       "width": 352
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = timm.create_model('efficientnet_b4', num_classes=4, checkpoint_path='/home/linh/Downloads/Covid-19/weights/EfficientNet_B4.pth')\n",
    "model.to(device)\n",
    "\n",
    "since = round(time.monotonic() * 1000)\n",
    "model.eval()\n",
    "y_test = []\n",
    "y_pred = []\n",
    "\n",
    "for i, (images, labels) in enumerate(data_loader['val']):\n",
    "    images = images.to(device)\n",
    "    labels = labels.to(device)\n",
    "    outputs = model(images)\n",
    "    _, predictions = outputs.max(1)\n",
    "    \n",
    "    y_test.append(labels.data.cpu().numpy())\n",
    "    y_pred.append(predictions.data.cpu().numpy())\n",
    "\n",
    "#set the end time to predict\n",
    "time_elapsed = round(time.monotonic() * 1000) - since   \n",
    "print('prediction time for complete in {:.0f} milisecond'.format(time_elapsed))\n",
    "#print('Prediction time to complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "#concat predicted results to be dataframe\n",
    "y_test = np.concatenate(y_test)\n",
    "y_pred = np.concatenate(y_pred)\n",
    "\n",
    "pd.DataFrame({'true_label':y_test,'predicted_label':y_pred}).to_csv('/home/linh/Downloads/Covid-19/results/EfficientNet_B4.csv',index=False)\n",
    "\n",
    "sns.heatmap(confusion_matrix(y_test, y_pred))\n",
    "# set accuracy score to control processes\n",
    "accuracy_score(y_test, y_pred)\n",
    "\n",
    "# Generate a classification report\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "report = classification_report(y_test, y_pred, digits=6)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRseNet_XL with image size=448"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction time for complete in 33727 milisecond\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0   0.000000  0.000000  0.000000        10\n",
      "           1   0.590811  0.929944  0.722564       885\n",
      "           2   1.000000  0.013468  0.026578       594\n",
      "           3   0.000000  0.000000  0.000000         0\n",
      "\n",
      "    accuracy                       0.558093      1489\n",
      "   macro avg   0.397703  0.235853  0.187285      1489\n",
      "weighted avg   0.750079  0.558093  0.440065      1489\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAHwCAYAAABHf7LhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3debRtZXkv6N8riA0mqBgbMLeIXjGJxjRgVOzxSjQxXhJJxRplYkwZY+yuCqnk2sQmmphrr6Q0ZRKNWlVo6VXLqGgiIiJ6jdhgrrmiIirYoTQKCAj7rT/W2rLd7A17n73mWfOc+TyMNSZrtt8a44zBe3688/uquwMAAFNyvWUPAAAAdjdFMAAAk6MIBgBgchTBAABMjiIYAIDJUQQDADA5imAAACZHEQwAwOQoggEAmBxFMAAAk6MIBgBgchTBAABMjiIYAIDJ2XfZA9iJffc7uJc9BsZnn+v5ux3XdNXKyrKHAOwhrrzi3Fr2GJLkB98+a+F1zvVvcbtR/LYxUC0AADA5e3QSDACw11q5atkj2KtJggEAmBxJMADAGLV3GYakCAYAGCMv9A5KOwQAAJMjCQYAGKHWDjEoSTAAAJMjCQYAGCM9wYNSBAMAjJF2iEFphwAAYHIkwQAAY2TFuEFJggEAmBxJMADAGOkJHpQkGACAyZEEAwCMkSnSBqUIBgAYISvGDUs7BAAAkyMJBgAYI+0Qg5IEAwAwOZJgAIAx0hM8KEUwAMAYWTFuUNohAACYHEkwAMAYaYcYlCQYAIDJkQQDAIyRKdIGpQgGABgj7RCD0g4BAMDkSIIBAMZIO8SgJMEAAEyOJBgAYIS6LZYxJEkwAMAY9criPztUVb9XVX0dn6vWnH/IdZx7wrU865FV9bGquriqLqqqk6vqITv+EXOSYAAAtupTSZ6zybF7JzkyyXs2OPbpJG/fYP+/bnSjqnpRkmOTnJPkNUn2S/LwJO+sqid29/HbHPc1KIIBAMZohC/GdfenMiuEr6GqPjL/1/9zg8Of6u5nb+UZVXVEZgXwF5PctbsvmO9/YZLTk7yoqv6xu8/e3uh/lHYIAAB2pKp+Lsndk5yb5F07vN1j59vnrxbASTIvev86yQ2SPGqHz1AEAwCM0gh7gq/FY+bbv+uN3+g7qKr+sKqeNt/e5VrudeR8e+IGx96z7pxdph0CAGAiqur0zY5192G7eM8bJXlEkquS/O0mpz1w/ll73clJHtndX1mzb/8kBye5uLu/vsF9Pj/fHrorY11LEgwAMEYrVy3+M4z/OclNk5zY3V9dd+zSJH+e5LAkN5t/7pvkA0nul+T988J31QHz7UWbPGt1/013OmhJMADAGA3QvrCrae91WG2F+JsNnvetJH+2bvcpVXVUklOT3C3Jo5O8fIBxXStJMAAAu6Sq7pTkiMymMnv3Vq/r7itzdevEfdYcWk16D8jGVvdfuI1hbkgSDAAwRiOcIm0D1/VC3LU5b779YTtEd19SVecmObiqbrNBX/Ad5tsztz/UHyUJBgBg26rqhkl+J7MX4v5uF25x9/n2rHX7T5pvH7TBNQ9ed84uUwQDAIzR+KdI+63MXnR7zwYvxCVJquqXquoa9WZVPSDJU+Zf37ju8Kvn26dX1c3WXHNIkscnuTzJa3c08miHAAAYp/G3Q6y2Qmy0QtyqlyS5Q1WdllnfcJLcJVfP8/vM7j5t7QXdfVpVvSTJU5OcUVVvyWzZ5N9OcvMkT9zpanGJIhgAgG2qqp9Jcq9c9wtxb0jyG0numlkrw/WTfDPJm5Mc390f2uii7j62qj6TWfL7mCQrST6R5IXd/Y8L+Q3dvYj7LMW++x285w6ewexzPV0+XNNV409UgJG48opza9ljSJLLPvSGhdc5N7z374zit42BagEAgMnRDgEAMELbn3GM7VAEAwCMkTauQS20CK6qG2U259uhuXpN5wszm9D4o939/UU+DwAAdsVCiuD5HG7Pz2zC5BtvctqlVfX6JM/o7gsW8VwAgL3W4uf1ZY0dF8FVddMkH07y00kuSfJPST6fH137+Q5J7pnkj5Lcv6ru0d0XbXA7AAAY3CKS4GdlVgC/NMmzuvvijU6qqpskeW6SJyf5syTHbuXmVXX6Zsf2uf5B2x4sAMAeQU/woBYxRdrRSU7q7mM3K4CTpLsv7u6nJjk5yW8u4LkAALBLFpEE3ybJ/7ON8z+a5Iitntzdh212zGIZAMBeS0/woBZRBH8nyR23cf7PzK8BAGAz2iEGtYh2iPcmObqqHnddJ1bVE5I8NMmJC3guAADskkUkwc9M8mtJXllVxyZ5X2bzAq+dHeLQJEclOSTJtzJ7MQ4AgM1ohxjUjovg7j63qu6R5FVJHpjkD5Os79Wt+fZ9SR7X3efu9LkAALCrFrJYRnefleRXqup2Se6fWY/wAfPDFyX5XJIPzM8DAOC66Ake1EKXTZ4XuQpdAICdUgQPahEvxgEAwB5loUkwAAAL4sW4QUmCAQCYHEkwAMAY6QkelCIYAGCMtEMMSjsEAACTIwkGABgj7RCDkgQDADA5kmAAgDHSEzwoSTAAAJMjCQYAGCM9wYNSBAMAjJEieFDaIQAAmBxJMADAGHUvewR7NUkwAACTIwkGABgjPcGDUgQDAIyRInhQ2iEAAJgcSTAAwBhZMW5QkmAAACZHEgwAMEZ6ggelCAYAGCPzBA9KOwQAAJMjCQYAGCPtEIOSBAMAMDmSYACAMZIED0oSDADA5EiCAQDGyGIZg1IEAwCMUK+YIm1I2iEAAJgcSTAAwBh5MW5QkmAAACZHEgwAMEZejBuUIhgAYIy8GDco7RAAAEyOJBgAYIy8GDcoSTAAAJMjCQYAGCNJ8KAUwQAAY9RejBuSdggAACZHEgwAMEbaIQYlCQYAYHIkwQAAY2SxjEFJggEAmBxJMADAGLWe4CEpggEAxkg7xKAUwex1Lj7ng8seAiN0o4PuvewhMEIH3eTmyx4CsCSKYACAEWpTpA3Ki3EAAEyOJBgAYIz0BA9KEgwAMEa9svjPAlXVA6rqbVX1jaq6vKq+VlXvrapf3eDcI6rq3VV1flV9v6rOqKonV9U+13L/h1TVyVV1UVVdXFX/raoeuajxK4IBANiWqvovSf45yeFJ/r8kL07yriQ/keR+6879j0lOSXKfJG9LcnyS/ZK8NMkJm9z/CUnemeTOSd6Y5DVJDkryuqp60SJ+g3YIAIAxGmk7RFX9QZI/TvIPSR7T3VesO379Nf/+45kVsFcluV93f3y+/5lJTkpyTFU9vLtPWHPNIUlelOT8JId399nz/c9N8i9Jjq2qt3b3R3byOyTBAABsSVXdIMnzk3wlGxTASdLdP1jz9ZjM0uETVgvg+TmXJXnG/OsfrbvF7ye5QZLjVwvg+TUXJPmL+dfH7uyXSIIBAMZpnFOkPTCzovZlSVaq6tcya1m4LMnHNkhnj5xvT9zgXqckuTTJEVV1g+6+fAvXvGfdObtMEQwAMEYDtENU1embHevuw7Zwi7vOt5cl+WRmBfDa+5+S5JjuPm++647z7ZkbPO/KqvpSkjsluV2Sf9vCNV+vqkuS3Laqbtzdl25hzBvSDgEAwFbdcr794ySd5N5JfizJXZK8L7OX3/7fNecfMN9etMn9VvffdBeuOWCT41siCQYAGKMFT2mWbDntvTarAeqVSR66pmf3M1X1G0k+l+S+VXWPnb64NjRJMAAAW3XhfPvJtS+tJcm8NeG986+/PN9eV2q7uv/CNfu2es1mSfGWKIIBAMZopRf/2bnPzbcXbnL8gvn2RuvOP3T9iVW1b5KfyixVPmuDZ2x0zW2S7J/knJ30AyeKYAAAtu79mfUC/2xVbVRHrr4o96X59qT59kEbnHufJDdOctqamSGu65oHrztnlymCAQBGqFdWFv7Z8Zi6v5zZSm7/Lsl/Wnusqo5K8iuZpcSr05u9Jcm3kzy8qg5fc+4Nkzxv/vVV6x7z2iSXJ3nCfOGM1WtuluRp86+v3ulv8WIcAMAYjXTFuCSPT/KLSV4ynyf4k5m1NRyd2cpwj+7ui5Kku787X2HuLUlOrqoTMlsJ7qGZTYX2liRvWnvz7v5SVf1xklck+XhVvSnJFZktvHHbJC9exEt3imAAALasu8+pqsOS/Flmxex9knw3s4T4L7v7Y+vOf3tV3TfJ05M8LMkNk3whyVOTvKK7r1Htd/crq+rsJMcl+d3Muhc+m+QZ3f0Pi/gdimAAgDEabxKc+WIYT5x/tnL+h5P86jaf8c7MCutB6AkGAGByJMEAAGM0wGIZXE0RDAAwRiNuh9gbaIcAAGByJMEAACPUkuBBSYIBAJgcSTAAwBhJggelCAYAGKMFLHPM5rRDAAAwOZJgAIAx0g4xKEkwAACTIwkGABgjSfCgJMEAAEyOJBgAYIS6JcFDUgQDAIyRdohBaYcAAGByJMEAAGMkCR6UJBgAgMmRBAMAjFBLggelCAYAGCNF8KC0QwAAMDmSYACAMVpZ9gD2bpJgAAAmRxIMADBCXowbliIYAGCMFMGD0g4BAMDkSIIBAMbIi3GDkgQDADA5kmAAgBHyYtywJMEAAEyOJBgAYIz0BA9KEQwAMELaIYalHQIAgMmRBAMAjJF2iEEtJQmuqhdW1ReX8WwAAFhWEnyLJIds5cSqOn2zY/tc/6BFjQcAYFRaEjwo7RAAAGOkCB7UQorgqnr9Ni85Yqsndvdhmx3bd7+DvTYJAMC2LSoJfkSSTlLbuEYBCwCwCe0Qw1pUEfy9JOckedwWz//TJEct6NkAALAtiyqCP53k57v7g1s5uap+b0HPBQDYO0mCB7WoKdI+leQmVXX7Bd0PAGDSemXxH662qCT4g0nuneS2SbYy/+/bk5y9oGcDAMC2LKQI7u63JnnrNs5/R5J3LOLZAAB7I8ntsJayYhwAACyTxTIAAEZIEjwsRTAAwBj1dpZfYLu0QwAAMDmSYACAEdIOMSxJMAAAkyMJBgAYoV7REzwkSTAAAJMjCQYAGCE9wcNSBAMAjFCbIm1Q2iEAAJgcSTAAwAhphxiWJBgAgMmRBAMAjJAp0oalCAYAGKHuZY9g76YdAgCAyZEEAwCMkHaIYUmCAQCYHEkwAMAISYKHpQgGABghL8YNSzsEAAC7rKoeUVU9/zx63bH7rTm20ecFm9xzn6p6SlWdUVXfr6rzq+rdVXXEosYtCQYAGKE9oR2iqn4yyfFJLk5yk2s59YNJTt5g/6kb3LOSnJDkmCSfm9//5kl+O8kpVfWw7n7HzkauCAYAYBfMi9XXJvlOkv+a5LhrOf3k7n72Fm/98MwK4NOSPKC7L5s/79WZFc2vqaqTuvt7uzr2RDsEAMAoddfCPwv2pCRHJnlUkksWeN8/mm+fsVoAJ0l3/0uSNyX5icyK5B1RBAMAsC1V9TNJXpDk5d19yhYu+fdV9YSqelpV/X5V3WGT+94wyRFJLk3yoQ1Oec98e+SujHst7RAAACPUK4u/Z1Wdvunzug/b4j32TfKGJF9J8rQtPvp/nX/W3uetSf6guy9Ys/v2SfZJclZ3X7nBfT4/3x66xeduShEMADBCK4tvX1iUP0vyi0nu1d3fv45zz0vyp0neleTsJDdMcniSv0jysCS3rqr7dP+w5D9gvr1ok/ut7r/prg39aopgAICJ2Grau5mqultm6e+Lu/sjW3jef0/y39fsujjJiVV1WpJPJblnkl9PsuPZHrZLTzAAwAiN7cW4eRvE65OcmeSZO/tt/d0k//f8633WHFpNeg/Ixlb3X7iT5yeKYAAAtuYmmfXi/kySy9YuepHkWfNzXjPf97It3O+8+Xb/Nfu+mOSqJLebF93rrb5Qd+b2h/+jtEMAAIzQCBfLuDzJ321y7Jcy6xM+NbMFLq6zVSLJ3efbs1Z3dPdl81aJe88/H1h3zYPn25O2OOZNKYIBAEaoe9kj+FHzl+AevdGxqnp2ZkXwP3T3367Zf3h3f3yD8x+R2QpwVyR587rDr8qsAH5eVa1dLOOu82vOS/LWnf4eRTAAAEN5S1VdmeTjSc7JbHaIuyb55SRXJvnD7j573TUnJPnNzBbE+GRVvTPJgZkVwPtkNq3ad3c6MEUwAMAIjbAdYle8Ksl/yGwWiFskqSTnJnldkpd196fXX9DdXVX/S2bLJv9+kicmuSzJKUme192nLWJg1WPL2rdh3/0O3nMHz2C+/7WNFphh6m500L2XPQRG6KCb3HzZQ2CEvnL+Z0ZRfX729r+28DrnZ7/4rlH8tjGQBAMAjNCIF8vYKyiCAQBGaKfz+nLtzBMMAMDkSIIBAEZoD35ta48gCQYAYHIkwQAAI+TFuGFJggEAmBxJMADACJkdYliKYACAEfJi3LC0QwAAMDmSYACAEfJi3LAUwex1LnvG45Y9BEboeuU/JlzThZdfsuwhAEuiCAYAGCEvxg1LEQwAMELaIYblxTgAACZHEgwAMEJmSBuWJBgAgMmRBAMAjJCe4GEpggEARsjsEMPSDgEAwORIggEARmhl2QPYy0mCAQCYHEkwAMAIdfQED0kSDADA5EiCAQBGaMVqGYNSBAMAjNCKdohBaYcAAGByJMEAACPkxbhhSYIBAJgcSTAAwAhZLGNYimAAgBHSDjEs7RAAAEyOJBgAYIS0QwxLEgwAwORIggEARkgSPCxFMADACHkxbljaIQAAmBxJMADACK0IggclCQYAYHIkwQAAI7SiJ3hQkmAAACZHEgwAMEK97AHs5RTBAAAjZJ7gYWmHAABgciTBAAAjtFJejBuSJBgAgMmRBAMAjJAX44alCAYAGCEvxg1LOwQAAJMjCQYAGKEV78UNShIMAMDkSIIBAEZoJaLgISmCAQBGyOwQw9IOAQDA5EiCAQBGyItxw5IEAwAwOZJgAIARsljGsCTBAABMjiQYAGCEzA4xLEUwAMAIeTFuWNohAACYHEkwAMAIeTFuWAtLgqvqN6rq5VX14qp64LWc98iqOmlRzwUAgO3acRJcVZXkTUkelvxwkesnV9W7kvxud1+47pJDktx3p88FANibSYKHtYgk+FFJjklyTpKnJ/nfk3w2yUOSnFpVt1zAMwAAJqVr8Z9FqKq/qqr3V9VXq+r7VXV+VX2yqp5VVQducs0RVfXu+bnfr6ozqurJVbXPtTznIVV1clVdVFUXV9V/q6pHLuZXLKYn+FFJLkxy1+7+VpJU1UuT/FWSpyb556o6sru/vSs3r6rTNzu2z/UP2pVbAgCw656S5BNJ/inJt5Lsn+TuSZ6d5DFVdffu/urqyVX1H5O8NcllmXUPnJ/k15O8NMk9k/zW+gdU1ROSvDLJd5K8MckVmYWur6uqn+vu43b6IxZRBP9ckresFsBJ0t1XJTmuqr6S5GWZFcL37+4LFvA8AIC93ojbIX68uy9bv7Oqnp/kaUn+c5LHzff9eJLXJLkqyf26++Pz/c9MclKSY6rq4d19wpr7HJLkRZkVy4d399nz/c9N8i9Jjq2qt3b3R3byIxbRDrFfkm9udKC7X5HkSUnukuSfquqm2715dx+22WdnwwYAYLs2KoDn3jzf3mHNvmOS/ESSE1YL4DX3eMb86x+tu8/vJ7lBkuNXC+D5NRck+Yv518fu0uDXWEQSfG6Sf7fZwe4+vqr2TfKSJO9N8uEFPBMAYK824iR4M78+356xZt+R8+2JG5x/SpJLkxxRVTfo7su3cM171p2zyxZRBH8myf2v7YTufllV3SDJXyb5xQU8EwBgrzbEssnX9q7Vdv8ve1Udl+QmSQ5IcniSe2VWAL9gzWl3nG/P3OB5V1bVl5LcKcntkvzbFq75elVdkuS2VXXj7r50O2NeaxFF8LuTHF1Vv9bd79rspO7+q6raL8lzYjlsAIA93XFJbrXm+4lJfq+7z1uz74D59qJN7rG6f23L7Fau2X9+3lKL4P+aZJ8kl1zXid395/OX5Q5ZwHMBAPZaKwua0mytRb5T1d23TpKqulWSIzJLgD9ZVQ/p7k8s6jlD2XER3N3nJ/mbbZz/Dzt9JgAA49Dd30zytqr6RGYtDK9Pcuf54dU094CNrl2zf+3iahclucX82Heu5ZrNkuItWdiyyQAALM7KAJ8hdfeXM1sw7U5VdYv57s/Nt4euP38+ccJPJbkyyVlrDl3bNbfJrBXinJ30AyeKYAAAFmd1JbOr5tuT5tsHbXDufZLcOMlpa2aGuK5rHrzunF2mCAYAGKExJsFVdWhVXaO1oaquN18s45aZFbWrC6S9Jcm3kzy8qg5fc/4Nkzxv/vVV62732iSXJ3nCfOGM1WtultliHEny6p3+lkW8GAcAwIKNdCqtX03yl1V1apIvZdaze6sk981smrNvJPmD1ZO7+7tV9QeZFcMnV9UJma0E99DMpkJ7S2ZLKWfNNV+qqj9O8ookH6+qN+XqZZNvm+TFO10tLlEEAwCwdf+c5N9nNifwL2Y2tdklmb0Q94Ykr5hPmvBD3f32qrpvkqcneViSGyb5QpKnzs+/Rr3f3a+sqrMzm4btdzPrXvhskmcsapIFRTAAwAgNMUXaTnX3vyZ5wi5c9+HMUuTtXPPOJO/c7rO2Sk8wAACTIwkGABihoac0mzpFMADACI30xbi9hnYIAAAmRxIMADBCK7LgQUmCAQCYHEkwAMAIeTFuWIpgAIAR0gwxLO0QAABMjiQYAGCEtEMMSxIMAMDkSIIBAEZopZY9gr2bIhgAYITMEzws7RAAAEyOJBgAYITkwMOSBAMAMDmSYACAETJF2rAkwQAATI4kGABghMwOMSxFMADACCmBh6UdAgCAyZEEAwCMkBfjhiUJBgBgciTBAAAj5MW4YSmCAQBGSAk8LO0QAABMjiQYAGCEvBg3LEkwAACTIwkGABih1hU8KEUwAMAIaYcYlnYIAAAmRxIMADBC5gkeliQYAIDJkQQDAIyQHHhYkmAAACZHEgwAMEJ6goelCAYAGCFTpA1LOwQAAJMjCQYAGCErxg1LEgwAwORIgtnr3OzvP7PsIQB7iEt/cPmyhwCb0hM8LEUwAMAIaYcYlnYIAAAmRxIMADBC2iGGJQkGAGByJMEAACO00nqCh6QIBgAYISXwsLRDAAAwOZJgAIARWpEFD0oSDADA5EiCAQBGyGIZw5IEAwAwOZJgAIARsljGsBTBAAAj5MW4YWmHAABgciTBAAAj5MW4YUmCAQCYHEkwAMAIeTFuWIpgAIAR6tYOMSTtEAAATI4kGABghEyRNixJMAAAkyMJBgAYIS/GDUsRDAAwQuYJHpZ2CAAAJkcRDAAwQivphX92qqqOqapXVtWHquq7VdVV9cZNzj1kfnyzzwnX8pxHVtXHquriqrqoqk6uqofs+AesoR0CAICtekaSn09ycZJzkvz0Fq75dJK3b7D/Xzc6uapelOTY+f1fk2S/JA9P8s6qemJ3H78L474GRTAAwAiNdLGMp2RWnH4hyX2TfGAL13yqu5+9lZtX1RGZFcBfTHLX7r5gvv+FSU5P8qKq+sfuPnv7Q/9R2iEAANiS7v5Ad3++h6vQHzvfPn+1AJ4/9+wkf53kBkketYgHSYIBAEZoiCnSqur0zY5192EDPDJJDqqqP0xyYJLvJPlId5+xyblHzrcnbnDsPUmeOT/nWTsdlCIYAGCE9qIp0h44//xQVZ2c5JHd/ZU1+/ZPcnCSi7v76xvc5/Pz7aGLGJQiGABgIgZMezdyaZI/z+yluLPm++6S5NlJ7p/k/VX1C919yfzYAfPtRZvcb3X/TRcxOEUwAMAILWJKs2Xq7m8l+bN1u0+pqqOSnJrkbkkeneTlu3tsiRfjAADYjbr7yiR/O/96nzWHVpPeA7Kx1f0XLmIckmAAgBEa6RRpi3LefLv/6o7uvqSqzk1ycFXdZoO+4DvMt2cuYgCSYACAERrjinELdPf59qx1+0+abx+0wTUPXnfOjiiCAQBYuKr6paq6Rq1ZVQ/IbNGNJFm/5PKr59unV9XN1lxzSJLHJ7k8yWsXMT7tEAAAIzTGKdKq6ugkR8+/3nq+vUdVvW7+79/u7uPm//6SJHeoqtMyW2Uumc0OsToX8DO7+7S19+/u06rqJUmemuSMqnpLZssm/3aSmyd54iJWi0sUwQAAbN0vJHnkun23m3+S5MtJVovgNyT5jSR3zayV4fpJvpnkzUmO7+4PbfSA7j62qj6TWfL7mMzWDflEkhd29z8u6ofUntx0ve9+B++5gwcARunKK86tZY8hSe5z8AMWXueccu77R/HbxkASDAAwQpK+YXkxDgCAyZEEAwCM0MimNNvrSIIBAJgcSTAAwAhJgoclCQYAYHIkwQAAI7QnT2O7J1AEAwCMkHaIYWmHAABgciTBAAAj1JLgQUmCAQCYnN2SBFfVzZNc1d0X7Y7nAQDs6bwYN6yFJMFVdXBVHV9V762q/1JVB873/0JVnZHkvCTnV9UpVfXTi3gmAMDebCW98A9X23ESPE95P5rk4PmuByb5D1X1K0neleTAJJ9OclCSeyX556q6c3dfuNNnAwDArlhEO8QTMiuA/yLJm5M8NMlzk7wuyfeS3KO7v5IkVfX8JP85yROT/PlWbl5Vp292bJ/rH7STcQMAjJZ2iGEtoh3i6CT/0t3P6O4zuvt5SU5N8qAkf7JaAM89I8lZSR6ygOcCAMAuWUQS/D8leeO6fR9Pcs8kp63d2d1dVR9M8ptbvXl3H7bZsX33O9hfkQCAvZIe3mEtogi+UZJL1u27KEm6+7wNzv9mkv0X8FwAgL2WeYKHtYh2iG8nueW6fZck+dYm5x+YxEtxAAAszSKK4DOT/OzaHd39ou6+zSbn/1SScxbwXACAvdZK98I/XG0RRfDpSQ6rqv2u68Sq+vHMpkk7dQHPBQCAXbLjIri7/6S7b9DdV2zh9FtlNkXa3+z0uQAAe7Me4B+utluWTV7V3Z9P8vLd+UwAAFhvtxbBAABsjR7eYSmCAQBGSPvCsBbxYhwAAOxRJMEAACOkHWJYkmAAACZHEgwAMEJ6goelCAYAGCHtEMPSDgEAwORIggEARkg7xLAkwQAATI4kGABghLpXlj2EvZoiGABghFa0QwxKOwQAAJMjCQYAGKE2RdqgJMEAAEyOJBgAYIT0BA9LEgwAwORIggEARgBx178AAAcCSURBVEhP8LAUwQAAI7SiCB6UdggAACZHEgwAMELtxbhBSYIBAJgcSTAAwAh5MW5YimAAgBEyT/CwtEMAADA5kmAAgBHSDjEsSTAAAJMjCQYAGCGLZQxLEQwAMELaIYalHQIAgMmRBAMAjJAp0oYlCQYAYHIkwQAAI6QneFiSYAAAJkcSDAAwQqZIG5YiGABghNqLcYPSDgEAwORIggEARkg7xLAkwQAATI4kGABghEyRNixFMADACHkxbljaIQAA2Jaqum1V/X1Vfa2qLq+qs6vqZVV1s2WPbaskwQAAIzTWdoiqun2S05LcMsk7kvyPJL+c5D8leVBV3bO7v7PEIW6JJBgAgO34PzIrgJ/U3Ud3959295FJXprkjkmev9TRbVGN9W8ZW7HvfgfvuYMHAEbpyivOrWWPIUmuP0Cd84Md/rZ5CvyFJGcnuX13r6w59mNJvp6kktyyuy/ZybOGJgkGABihHuCzAPefb9+3tgBOku7+XpIPJ7lxkrsv5nHD0RMMADARVXX6Zse6+7At3OKO8+2Zmxz/fJKjkhya5P3bG93utUcXwWP53xXLtvoHeot/eJkIfy7YiD8XbMSfi3Eaos65tiJ4iw6Yby/a5Pjq/pvu8DmD26OLYAAAts5fdK6mJxgAgK1aTXoP2OT46v4Ld8NYdkQRDADAVn1uvj10k+N3mG836xkeDUUwAABb9YH59qiq+pE6cj5F2j2TXJrko7t7YNulCAYAYEu6+4tJ3pfkkCSPX3f4OUn2T/KGsc8RnHgxDgCA7XlcZssmv6KqHpDk35LcLbM5hM9M8vQljm3L9ugV4wAA2P2q6ieTPDfJg5IcmNlKcW9L8pzuvmCZY9sqRTAAAJOjJxgAgMlRBAMAMDmKYAAAJkcRDADA5CiCAQCYHEUwAACTowjeg1XVbavq76vqa1V1eVWdXVUvq6qbLXtsLEdVHVNVr6yqD1XVd6uqq+qNyx4Xy1NVB1bVo6vqbVX1har6flVdVFWnVtX/tn7ZU6ajqv6qqt5fVV+d/7k4v6o+WVXPqqoDlz0+GJp5gvdQVXX7zFZruWWSdyT5H0l+ObPVWj6X5J7d/Z3ljZBlqKpPJfn5JBcnOSfJTyf5v7r7EUsdGEtTVY9N8qrMJrL/QJKvJLlVkt9MckCStyb5rfYfg8mpqiuSfCLJZ5N8K7Plbu+e5PAkX0ty9+7+6vJGCMNSBO+hquq9SY5K8qTufuWa/S9J8pQkf9Pdj13W+FiOqrp/ZsXvF5LcN7OiRxE8YVV1ZGbFzbu6e2XN/lsn+ViSn0xyTHe/dUlDZEmq6obdfdkG+5+f5GlJXtXdj9v9I4Pdw/8G2wPNU+Cjkpyd5K/XHX5WkkuS/E5V7b+bh8aSdfcHuvvzUj1WdfdJ3f3OtQXwfP83krx6/vV+u31gLN1GBfDcm+fbO+yuscAyKIL3TPefb9+3wX/Yvpfkw0lunNn/1gLYzA/m2yuXOgrG5tfn2zOWOgoY2L7LHgC75I7z7ZmbHP98ZknxoUnev1tGBOxRqmrfJL87/3riMsfCclXVcUluklmP+OFJ7pVZAfyCZY4LhqYI3jMdMN9etMnx1f033Q1jAfZML0hy5yTv7u73LnswLNVxmb0suerEJL/X3ectaTywW2iHAJiYqnpSkmMzm1Xmd5Y8HJasu2/d3ZXk1pnNGnK7JJ+sql9a7shgWIrgPdNq0nvAJsdX91+4G8YC7EGq6glJXp7ZtFj37+7zlzwkRqK7v9ndb8usne7AJK9f8pBgUIrgPdPn5ttDNzm++kbvZj3DwARV1ZOTvDLJv2ZWAH9jyUNihLr7y5n9JelOVXWLZY8HhqII3jN9YL49av1qT1X1Y0numeTSJB/d3QMDxqmq/iTJS5N8KrMC+FtLHhLjdtB8e9VSRwEDUgTvgbr7i0nel+SQJI9fd/g5mU2M/4buvmQ3Dw0Yoap6ZmYvwp2e5AHd/e0lD4klq6pDq+oaLXVVdb35Yhm3THJad1+w+0cHu4cV4/ZQGyyb/G9J7pbZHMJnJjnCssnTU1VHJzl6/vXWSX4lyVlJPjTf9+3uPm4ZY2M5quqRSV6XWaL3ymw8q8zZ3f263TgslmzeGvOXSU5N8qUk38lshoj7ZvZi3Dcy+wvTZ5c2SBiYIngPVlU/meS5SR6U2UsMX0/ytiTP8bf3aaqqZ2e2auBmvtzdh+ye0TAGW/gzkSQf7O77DT8axqKq7pzksZnNCXzbzKbUvCSzEOVdSV7hpUn2dopgAAAmR08wAACTowgGAGByFMEAAEyOIhgAgMlRBAMAMDmKYAAAJkcRDADA5CiCAQCYHEUwAACTowgGAGByFMEAAEyOIhgAgMlRBAMAMDmKYAAAJkcRDADA5CiCAQCYHEUwAACT8/8Dp29Hth65jpIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 248,
       "width": 352
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = timm.create_model('tresnet_xl', num_classes=4, checkpoint_path='/home/linh/Downloads/Covid-19/weights/TResNet_XL.pth')\n",
    "model.to(device)\n",
    "\n",
    "since = round(time.monotonic() * 1000)\n",
    "model.eval()\n",
    "y_test = []\n",
    "y_pred = []\n",
    "\n",
    "for i, (images, labels) in enumerate(data_loader['val']):\n",
    "    images = images.to(device)\n",
    "    labels = labels.to(device)\n",
    "    outputs = model(images)\n",
    "    _, predictions = outputs.max(1)\n",
    "    \n",
    "    y_test.append(labels.data.cpu().numpy())\n",
    "    y_pred.append(predictions.data.cpu().numpy())\n",
    "\n",
    "#set the end time to predict\n",
    "time_elapsed = round(time.monotonic() * 1000) - since   \n",
    "print('prediction time for complete in {:.0f} milisecond'.format(time_elapsed))\n",
    "#print('Prediction time to complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "#concat predicted results to be dataframe\n",
    "y_test = np.concatenate(y_test)\n",
    "y_pred = np.concatenate(y_pred)\n",
    "\n",
    "pd.DataFrame({'true_label':y_test,'predicted_label':y_pred}).to_csv('/home/linh/Downloads/Covid-19/results/TResNet_XL.csv',index=False)\n",
    "\n",
    "sns.heatmap(confusion_matrix(y_test, y_pred))\n",
    "# set accuracy score to control processes\n",
    "accuracy_score(y_test, y_pred)\n",
    "\n",
    "# Generate a classification report\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "report = classification_report(y_test, y_pred, digits=6)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble voting of 3 models with image size = 224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Eff_B0 = timm.create_model('efficientnet_b0', num_classes=4, checkpoint_path='/home/linh/Downloads/Covid-19/weights/EfficientNet_B0.pth')\n",
    "Eff_B4 = timm.create_model('efficientnet_b4', num_classes=4, checkpoint_path='/home/linh/Downloads/Covid-19/weights/EfficientNet_B4.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model = timm.create_model('efficientnet_b0', pretrained=True)\n",
    "#model.fc #show fully connected layer for ResNet family\n",
    "model.classifier #show the classifier layer (fully connected layer) for EfficientNets\n",
    "\n",
    "# Create classifier\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = True\n",
    "# define `classifier` for ResNet\n",
    "# Otherwise, define `fc` for EfficientNet family \n",
    "#because the definition of the full connection/classifier of 2 CNN families is differnt\n",
    "fc = nn.Sequential(OrderedDict([('fc1', nn.Linear(2048, 1000, bias=True)),\n",
    "\t\t\t\t\t\t\t     ('BN1', nn.BatchNorm2d(1000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)),\n",
    "\t\t\t\t\t\t\t\t ('dropout1', nn.Dropout(0.7)),\n",
    "                                 ('fc2', nn.Linear(1000, 512)),\n",
    "\t\t\t\t\t\t\t\t ('BN2', nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)),\n",
    "\t\t\t\t\t\t\t\t ('swish1', Swish()),\n",
    "\t\t\t\t\t\t\t\t ('dropout2', nn.Dropout(0.5)),\n",
    "\t\t\t\t\t\t\t\t ('fc3', nn.Linear(512, 128)),\n",
    "\t\t\t\t\t\t\t\t ('BN3', nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)),\n",
    "\t\t\t\t\t\t\t     ('swish2', Swish()),\n",
    "\t\t\t\t\t\t\t\t ('fc4', nn.Linear(128, 3)),\n",
    "\t\t\t\t\t\t\t\t ('output', nn.Softmax(dim=1))\n",
    "\t\t\t\t\t\t\t ]))\n",
    "# connect base model (EfficientNet_B0) with modified classifier layer\n",
    "model.fc = fc\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "#optimizer = Nadam(model.parameters(), lr=0.001)\n",
    "#optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=0.0001)\n",
    "optimizer = optim.SGD(model.parameters(), \n",
    "                      lr=0.01,momentum=0.9,\n",
    "                      nesterov=True,\n",
    "                      weight_decay=0.0001)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)\n",
    "#show our model architechture and send to GPU\n",
    "model.to(device)\n",
    "CHECK_POINT_PATH = '/home/linh/Downloads/Covide-19/weights/EfficientNet_B0_Covid-19.pth'\n",
    "try:\n",
    "    checkpoint = torch.load(CHECK_POINT_PATH)\n",
    "    print(\"checkpoint loaded\")\n",
    "except:\n",
    "    checkpoint = None\n",
    "    print(\"checkpoint not found\")\n",
    "\n",
    "def load_model(path):                                \n",
    "  model.load_state_dict(checkpoint['model_state_dict'])\n",
    "  best_model_wts = copy.deepcopy(model.state_dict())\n",
    "  optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "  scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n",
    "  best_loss = checkpoint['best_val_loss']\n",
    "  best_acc = checkpoint['best_val_accuracy']\n",
    "    \n",
    "Mod_Eff_B0 = load_model(CHECK_POINT_PATH)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Ensemble_Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Ensemble_Model, self).__init__()\n",
    "        self.Eff_B0 = Eff_B0()\n",
    "        self.Mod_Eff_B0 = Mod_Eff_B0\n",
    "        self.Eff_B4 = Eff_B4()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x1 = self.Eff_B0(x)\n",
    "        x2 = self.Mod_Eff_B0(x)\n",
    "        x3 = self.Eff_B4(x)\n",
    "        x = sum([x1, x2, x3]) / 3\n",
    "        return x\n",
    "    \n",
    "model = Ensemble_Model()\n",
    "model,to(device)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "since = round(time.monotonic() * 1000)\n",
    "model.eval()\n",
    "y_test = []\n",
    "y_pred = []\n",
    "\n",
    "for i, (images, labels) in enumerate(data_loader['data']):\n",
    "    images = images.to(device)\n",
    "    labels = labels.to(device)\n",
    "    outputs = model(images)\n",
    "    _, predictions = outputs.max(1)\n",
    "    \n",
    "    y_test.append(labels.data.cpu().numpy())\n",
    "    y_pred.append(predictions.data.cpu().numpy())\n",
    "\n",
    "#set the end time to predict\n",
    "time_elapsed = round(time.monotonic() * 1000) - since   \n",
    "print('prediction time for complete in {:.0f} milisecond'.format(time_elapsed))\n",
    "#print('Prediction time to complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "#concat predicted results to be dataframe\n",
    "y_test = np.concatenate(y_test)\n",
    "y_pred = np.concatenate(y_pred)\n",
    "\n",
    "pd.DataFrame({'true_label':y_test,'predicted_label':y_pred}).to_csv('/home/linh/Downloads/Covid-19/results/Ensemble_Eff_B0_Eff_B4_Mod_Eff_B0_Whole_Data.csv',index=False)\n",
    "\n",
    "sns.heatmap(confusion_matrix(y_test, y_pred))\n",
    "# set accuracy score to control processes\n",
    "accuracy_score(y_test, y_pred)\n",
    "\n",
    "# Generate a classification report\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "report = classification_report(y_test, y_pred, digits=6)\n",
    "print(report)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Metrics_for_EfficientNets.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}