Dataset_test_with_BlazeFace.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stuff\n",
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function\n",
    "import pandas as pd\n",
    "\n",
    "# import dataset\n",
    "from utils.dataset import VOCDataset, DatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# set up person only VOC dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val not found\n",
      "6469\n"
     ]
    }
   ],
   "source": [
    "# load files\n",
    "vocpath = \"../VOCdevkit/VOC2007\"\n",
    "train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath, cls=\"person\")\n",
    "\n",
    "vocpath = \"../VOCdevkit/VOC2012\"\n",
    "train_img_list2, train_anno_list2, _, _ = make_datapath_list(vocpath, cls=\"person\", VOC2012=True)\n",
    "\n",
    "train_img_list.extend(train_img_list2)\n",
    "train_anno_list.extend(train_anno_list2)\n",
    "\n",
    "print(len(train_img_list))\n",
    "\n",
    "# make Dataset\n",
    "voc_classes = ['person']\n",
    "color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "input_size = 128  # 画像のinputサイズを300×300にする\n",
    "\n",
    "## DatasetTransformを適応\n",
    "transform = DatasetTransform(input_size, color_mean)\n",
    "transform_anno = Anno_xml2list(voc_classes)\n",
    "\n",
    "train_dataset = VOCDataset(train_img_list, train_anno_list, phase = \"train\", transform=transform, transform_anno = transform_anno)\n",
    "val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DatasetTransform(\n",
    "    input_size, color_mean), transform_anno=Anno_xml2list(voc_classes))\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "train_dataloader = data.DataLoader(\n",
    "    train_dataset, batch_size=batch_size, shuffle=True, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "val_dataloader = data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_img_list2[0]\n",
    "\n",
    "import os\n",
    "os.path.isfile(train_img_list2[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../VOCdevkit/VOC2012/Annotations/2008_000003.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train_anno_list2[0])\n",
    "\n",
    "import os\n",
    "os.path.isfile(train_anno_list2[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 3, 128, 128])\n",
      "32\n",
      "torch.Size([3, 5])\n"
     ]
    }
   ],
   "source": [
    "# check operation\n",
    "batch_iterator = iter(dataloaders_dict[\"train\"])  # iter\n",
    "images, targets = next(batch_iterator)  # get first element\n",
    "print(images.size())  # torch.Size([4, 3, 300, 300])\n",
    "print(len(targets))\n",
    "print(targets[1].shape)  # check targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.7043, 0.4665, 0.8380, 0.6313, 0.0000],\n",
       "        [0.7997, 0.4612, 0.9142, 0.6682, 0.0000],\n",
       "        [0.8538, 0.5786, 0.9176, 0.6336, 0.0000]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test with ssd model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.blazeface import SSD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using: cuda:0\n",
      "set weights!\n"
     ]
    }
   ],
   "source": [
    "# SSD300の設定\n",
    "ssd_cfg = {\n",
    "    'num_classes': 2,  # 背景クラスを含めた合計クラス数\n",
    "    'input_size': 128,  # 画像の入力サイズ\n",
    "    'bbox_aspect_num': [4, 6],  # 出力するDBoxのアスペクト比の種類\n",
    "    'feature_maps': [16, 8],  # 各sourceの画像サイズ\n",
    "    'steps': [4, 8],  # DBOXの大きさを決める\n",
    "    'min_sizes': [30, 60],  # DBOXの大きさを決める\n",
    "    'max_sizes': [60, 128],  # DBOXの大きさを決める\n",
    "    'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "}\n",
    "\n",
    "net = SSD(phase=\"train\", cfg=ssd_cfg)\n",
    "\n",
    "# SSDのweightsを設定\n",
    "\n",
    "def weights_init(m):\n",
    "    if isinstance(m, nn.Conv2d):\n",
    "        init.kaiming_normal_(m.weight.data)\n",
    "        if m.bias is not None:\n",
    "            nn.init.constant_(m.bias, 0.0)\n",
    "\n",
    "# set inits for loc and conf\n",
    "net.loc.apply(weights_init)\n",
    "net.conf.apply(weights_init)\n",
    "\n",
    "# GPUが使えるか確認\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "\n",
    "print(\"set weights!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SSD(\n",
      "  (blaze): BlazeFace(\n",
      "    (features): Sequential(\n",
      "      (0): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "      (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace)\n",
      "      (3): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (4): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (5): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (6): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (7): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (8): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (9): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (10): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (extra): BlazeFaceExtra(\n",
      "    (features): Sequential(\n",
      "      (0): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (1): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (2): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (loc): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(96, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(96, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      "  (conf): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(96, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(96, 12, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ssd_model import MultiBoxLoss\n",
    "\n",
    "# define loss\n",
    "criterion = MultiBoxLoss(jaccard_thresh=0.5,neg_pos=2, device=device)\n",
    "\n",
    "# optim\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_lr(epoch):\n",
    "    lr = 1e-3\n",
    "    for i,lr_decay_epoch in enumerate([120,180]):\n",
    "        if epoch >= lr_decay_epoch:\n",
    "            lr *= 0.1\n",
    "    return lr\n",
    "\n",
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    lr = get_current_lr(epoch)\n",
    "    print(\"lr is:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# モデルを学習させる関数を作成\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"used device：\", device)\n",
    "\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # イテレーションカウンタをセット\n",
    "    iteration = 1\n",
    "    epoch_train_loss = 0.0  # epochの損失和\n",
    "    epoch_val_loss = 0.0  # epochの損失和\n",
    "    logs = []\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs+1):\n",
    "        \n",
    "        adjust_learning_rate(optimizer, epoch)\n",
    "        \n",
    "        # 開始時刻を保存\n",
    "        t_epoch_start = time.time()\n",
    "        t_iter_start = time.time()\n",
    "\n",
    "        print('-------------')\n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "        print('-------------')\n",
    "\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "                print('train')\n",
    "            else:\n",
    "                if((epoch+1) % 5 == 0):\n",
    "                    net.eval()   # モデルを検証モードに\n",
    "                    print('-------------')\n",
    "                    print('val')\n",
    "                else:\n",
    "                    # 検証は5回に1回だけ行う\n",
    "                    continue\n",
    "\n",
    "            # データローダーからminibatchずつ取り出すループ\n",
    "            for images, targets in dataloaders_dict[phase]:\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                images = images.to(device)\n",
    "                targets = [ann.to(device)\n",
    "                           for ann in targets]  # リストの各要素のテンソルをGPUへ\n",
    "\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # 順伝搬（forward）計算\n",
    "                    outputs = net(images)\n",
    "\n",
    "                    # 損失の計算\n",
    "                    loss_l, loss_c = criterion(outputs, targets)\n",
    "                    loss = loss_l + loss_c\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()  # 勾配の計算\n",
    "\n",
    "                        # 勾配が大きくなりすぎると計算が不安定になるので、clipで最大でも勾配2.0に留める\n",
    "                        nn.utils.clip_grad_value_(\n",
    "                            net.parameters(), clip_value=2.0)\n",
    "\n",
    "                        optimizer.step()  # パラメータ更新\n",
    "\n",
    "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
    "                            t_iter_finish = time.time()\n",
    "                            duration = t_iter_finish - t_iter_start\n",
    "                            print('Iteration {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
    "                                iteration, loss.item(), duration))\n",
    "                            t_iter_start = time.time()\n",
    "\n",
    "                        epoch_train_loss += loss.item()\n",
    "                        iteration += 1\n",
    "\n",
    "                    # 検証時\n",
    "                    else:\n",
    "                        epoch_val_loss += loss.item()\n",
    "\n",
    "        # epochのphaseごとのlossと正解率\n",
    "        t_epoch_finish = time.time()\n",
    "        print('-------------')\n",
    "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
    "            epoch+1, epoch_train_loss, epoch_val_loss))\n",
    "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
    "        t_epoch_start = time.time()\n",
    "\n",
    "        # ログを保存\n",
    "        log_epoch = {'epoch': epoch+1,\n",
    "                     'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}\n",
    "        logs.append(log_epoch)\n",
    "        df = pd.DataFrame(logs)\n",
    "        df.to_csv(\"log_output.csv\")\n",
    "\n",
    "        epoch_train_loss = 0.0  # epochの損失和\n",
    "        epoch_val_loss = 0.0  # epochの損失和\n",
    "\n",
    "        # ネットワークを保存する\n",
    "        if ((epoch+1) % 10 == 0):\n",
    "            torch.save(net.state_dict(), 'weights/blazeface128_' +\n",
    "                       str(epoch+1) + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# start training here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used device： cuda:0\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 1/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10 || Loss: 26.5654 || 10iter: 3.2959 sec.\n",
      "Iteration 20 || Loss: 19.8869 || 10iter: 1.3982 sec.\n",
      "Iteration 30 || Loss: 18.4547 || 10iter: 1.4920 sec.\n",
      "Iteration 40 || Loss: 16.9972 || 10iter: 1.5371 sec.\n",
      "Iteration 50 || Loss: 13.2807 || 10iter: 1.4463 sec.\n",
      "Iteration 60 || Loss: 14.6379 || 10iter: 1.4875 sec.\n",
      "Iteration 70 || Loss: 14.0668 || 10iter: 1.4270 sec.\n",
      "Iteration 80 || Loss: 19.2473 || 10iter: 1.4120 sec.\n",
      "Iteration 90 || Loss: 12.0091 || 10iter: 1.4075 sec.\n",
      "Iteration 100 || Loss: 12.0236 || 10iter: 1.3852 sec.\n",
      "Iteration 110 || Loss: 12.8533 || 10iter: 1.3914 sec.\n",
      "Iteration 120 || Loss: 11.6305 || 10iter: 1.4570 sec.\n",
      "Iteration 130 || Loss: 12.0704 || 10iter: 1.4312 sec.\n",
      "Iteration 140 || Loss: 11.4920 || 10iter: 1.4091 sec.\n",
      "Iteration 150 || Loss: 10.5847 || 10iter: 1.4079 sec.\n",
      "Iteration 160 || Loss: 10.8896 || 10iter: 1.3742 sec.\n",
      "Iteration 170 || Loss: 10.2025 || 10iter: 1.4522 sec.\n",
      "Iteration 180 || Loss: 10.1933 || 10iter: 1.4223 sec.\n",
      "Iteration 190 || Loss: 13.0562 || 10iter: 1.2629 sec.\n",
      "Iteration 200 || Loss: 9.4132 || 10iter: 0.9563 sec.\n",
      "-------------\n",
      "epoch 1 || Epoch_TRAIN_Loss:2900.7249 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  30.2690 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 2/200\n",
      "-------------\n",
      "train\n",
      "Iteration 210 || Loss: 13.7599 || 10iter: 2.7531 sec.\n",
      "Iteration 220 || Loss: 14.3135 || 10iter: 1.3921 sec.\n",
      "Iteration 230 || Loss: 9.5832 || 10iter: 1.5507 sec.\n",
      "Iteration 240 || Loss: 9.6896 || 10iter: 1.3667 sec.\n",
      "Iteration 250 || Loss: 9.9684 || 10iter: 1.4636 sec.\n",
      "Iteration 260 || Loss: 8.5233 || 10iter: 1.5511 sec.\n",
      "Iteration 270 || Loss: 11.1001 || 10iter: 1.4985 sec.\n",
      "Iteration 280 || Loss: 8.7031 || 10iter: 1.4885 sec.\n",
      "Iteration 290 || Loss: 8.5733 || 10iter: 1.4069 sec.\n",
      "Iteration 300 || Loss: 9.6268 || 10iter: 1.4431 sec.\n",
      "Iteration 310 || Loss: 9.1268 || 10iter: 1.5277 sec.\n",
      "Iteration 320 || Loss: 9.8540 || 10iter: 1.3496 sec.\n",
      "Iteration 330 || Loss: 7.6823 || 10iter: 1.4406 sec.\n",
      "Iteration 340 || Loss: 8.4571 || 10iter: 1.4572 sec.\n",
      "Iteration 350 || Loss: 8.7585 || 10iter: 1.5359 sec.\n",
      "Iteration 360 || Loss: 7.3348 || 10iter: 3.1434 sec.\n",
      "Iteration 370 || Loss: 8.0827 || 10iter: 1.6923 sec.\n",
      "Iteration 380 || Loss: 9.5556 || 10iter: 1.5927 sec.\n",
      "Iteration 390 || Loss: 8.6310 || 10iter: 1.6056 sec.\n",
      "Iteration 400 || Loss: 6.3734 || 10iter: 1.0828 sec.\n",
      "-------------\n",
      "epoch 2 || Epoch_TRAIN_Loss:1837.7310 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  33.2062 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 3/200\n",
      "-------------\n",
      "train\n",
      "Iteration 410 || Loss: 6.7348 || 10iter: 4.0603 sec.\n",
      "Iteration 420 || Loss: 6.8117 || 10iter: 2.7104 sec.\n",
      "Iteration 430 || Loss: 7.3585 || 10iter: 2.5084 sec.\n",
      "Iteration 440 || Loss: 8.0631 || 10iter: 2.6482 sec.\n",
      "Iteration 450 || Loss: 6.7312 || 10iter: 2.8164 sec.\n",
      "Iteration 460 || Loss: 8.5758 || 10iter: 2.7080 sec.\n",
      "Iteration 470 || Loss: 9.4725 || 10iter: 2.7379 sec.\n",
      "Iteration 480 || Loss: 6.8449 || 10iter: 2.7841 sec.\n",
      "Iteration 490 || Loss: 7.0152 || 10iter: 2.6505 sec.\n",
      "Iteration 500 || Loss: 5.9678 || 10iter: 2.5250 sec.\n",
      "Iteration 510 || Loss: 7.4550 || 10iter: 2.5234 sec.\n",
      "Iteration 520 || Loss: 6.7664 || 10iter: 2.4219 sec.\n",
      "Iteration 530 || Loss: 8.2052 || 10iter: 2.4922 sec.\n",
      "Iteration 540 || Loss: 7.7421 || 10iter: 2.3377 sec.\n",
      "Iteration 550 || Loss: 7.8491 || 10iter: 2.1715 sec.\n",
      "Iteration 560 || Loss: 6.8129 || 10iter: 2.7572 sec.\n",
      "Iteration 570 || Loss: 5.9597 || 10iter: 2.6101 sec.\n",
      "Iteration 580 || Loss: 6.6826 || 10iter: 2.5355 sec.\n",
      "Iteration 590 || Loss: 6.7011 || 10iter: 1.9947 sec.\n",
      "Iteration 600 || Loss: 5.9303 || 10iter: 1.4559 sec.\n",
      "-------------\n",
      "epoch 3 || Epoch_TRAIN_Loss:1447.3626 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  52.5926 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 4/200\n",
      "-------------\n",
      "train\n",
      "Iteration 610 || Loss: 6.4922 || 10iter: 3.1196 sec.\n",
      "Iteration 620 || Loss: 6.7688 || 10iter: 3.1598 sec.\n",
      "Iteration 630 || Loss: 6.7771 || 10iter: 2.0052 sec.\n",
      "Iteration 640 || Loss: 6.4730 || 10iter: 2.0512 sec.\n",
      "Iteration 650 || Loss: 6.2961 || 10iter: 2.4821 sec.\n",
      "Iteration 660 || Loss: 7.5306 || 10iter: 2.5211 sec.\n",
      "Iteration 670 || Loss: 8.8836 || 10iter: 3.0024 sec.\n",
      "Iteration 680 || Loss: 5.6794 || 10iter: 2.7201 sec.\n",
      "Iteration 690 || Loss: 6.1368 || 10iter: 3.1096 sec.\n",
      "Iteration 700 || Loss: 7.0351 || 10iter: 2.7325 sec.\n",
      "Iteration 710 || Loss: 6.7687 || 10iter: 2.9104 sec.\n",
      "Iteration 720 || Loss: 6.8872 || 10iter: 2.7023 sec.\n",
      "Iteration 730 || Loss: 6.6372 || 10iter: 2.5167 sec.\n",
      "Iteration 740 || Loss: 6.9626 || 10iter: 2.4311 sec.\n",
      "Iteration 750 || Loss: 5.9399 || 10iter: 2.4022 sec.\n",
      "Iteration 760 || Loss: 5.9752 || 10iter: 2.2261 sec.\n",
      "Iteration 770 || Loss: 6.2314 || 10iter: 2.3325 sec.\n",
      "Iteration 780 || Loss: 6.4372 || 10iter: 2.7090 sec.\n",
      "Iteration 790 || Loss: 6.3630 || 10iter: 2.2946 sec.\n",
      "Iteration 800 || Loss: 6.9339 || 10iter: 1.7027 sec.\n",
      "Iteration 810 || Loss: 6.8400 || 10iter: 2.2807 sec.\n",
      "-------------\n",
      "epoch 4 || Epoch_TRAIN_Loss:1339.9215 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  53.7892 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 5/200\n",
      "-------------\n",
      "train\n",
      "Iteration 820 || Loss: 6.8315 || 10iter: 4.7628 sec.\n",
      "Iteration 830 || Loss: 5.7393 || 10iter: 2.8622 sec.\n",
      "Iteration 840 || Loss: 6.8602 || 10iter: 2.5625 sec.\n",
      "Iteration 850 || Loss: 7.3645 || 10iter: 2.5190 sec.\n",
      "Iteration 860 || Loss: 5.7086 || 10iter: 2.4159 sec.\n",
      "Iteration 870 || Loss: 8.0672 || 10iter: 2.8867 sec.\n",
      "Iteration 880 || Loss: 6.6154 || 10iter: 2.3884 sec.\n",
      "Iteration 890 || Loss: 6.2310 || 10iter: 2.5925 sec.\n",
      "Iteration 900 || Loss: 6.0400 || 10iter: 2.3075 sec.\n",
      "Iteration 910 || Loss: 6.5971 || 10iter: 3.1989 sec.\n",
      "Iteration 920 || Loss: 5.8160 || 10iter: 2.4123 sec.\n",
      "Iteration 930 || Loss: 5.6461 || 10iter: 2.6774 sec.\n",
      "Iteration 940 || Loss: 5.2375 || 10iter: 2.8044 sec.\n",
      "Iteration 950 || Loss: 5.6179 || 10iter: 2.8899 sec.\n",
      "Iteration 960 || Loss: 5.1162 || 10iter: 2.2624 sec.\n",
      "Iteration 970 || Loss: 6.0427 || 10iter: 2.2621 sec.\n",
      "Iteration 980 || Loss: 6.1455 || 10iter: 1.6566 sec.\n",
      "Iteration 990 || Loss: 6.3184 || 10iter: 3.4024 sec.\n",
      "Iteration 1000 || Loss: 6.1066 || 10iter: 2.3288 sec.\n",
      "Iteration 1010 || Loss: 5.7968 || 10iter: 1.9033 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 5 || Epoch_TRAIN_Loss:1253.5928 ||Epoch_VAL_Loss:404.0365\n",
      "timer:  64.8578 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 6/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1020 || Loss: 6.1319 || 10iter: 3.8716 sec.\n",
      "Iteration 1030 || Loss: 6.2060 || 10iter: 2.3461 sec.\n",
      "Iteration 1040 || Loss: 6.8030 || 10iter: 2.5172 sec.\n",
      "Iteration 1050 || Loss: 6.8841 || 10iter: 2.5115 sec.\n",
      "Iteration 1060 || Loss: 5.5225 || 10iter: 2.8523 sec.\n",
      "Iteration 1070 || Loss: 5.6806 || 10iter: 2.6577 sec.\n",
      "Iteration 1080 || Loss: 6.0698 || 10iter: 2.8019 sec.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-b796d5f98bfa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mnum_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m200\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataloaders_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-12-1f39654e9bb6>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(net, dataloaders_dict, criterion, optimizer, num_epochs)\u001b[0m\n\u001b[1;32m     59\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphase\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m                     \u001b[0;31m# 順伝搬（forward）計算\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m                     \u001b[0;31m# 損失の計算\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/BlazeFace_Person.pytorch/utils/blazeface.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0msources\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;31m# compute extra block\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m         \u001b[0msources\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/BlazeFace_Person.pytorch/utils/blazeface.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minitialize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/BlazeFace_Person.pytorch/utils/blazeface.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_double_block\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m             \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m         \u001b[0;31m# skip connection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    336\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[1;32m    337\u001b[0m         return F.conv2d(input, self.weight, self.bias, self.stride,\n\u001b[0;32m--> 338\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    339\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "num_epochs = 200\n",
    "train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}