knowledge_distillation_for_mobilenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import sys\n",
    "sys.path.append('utils/')\n",
    "\n",
    "import keras\n",
    "from keras import optimizers\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "\n",
    "# use non standard flow_from_directory\n",
    "from image_preprocessing_ver2 import ImageDataGenerator\n",
    "# it outputs y_batch that contains onehot targets and logits\n",
    "# logits came from xception\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.layers import Lambda, concatenate, Activation\n",
    "from keras.losses import categorical_crossentropy as logloss\n",
    "from keras.metrics import categorical_accuracy, top_k_categorical_accuracy\n",
    "from keras import backend as K\n",
    "\n",
    "from mobilenet import get_mobilenet\n",
    "from keras.applications.mobilenet import preprocess_input\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '/home/dongsung/study/machine_learning/distillation/data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/home/dongsung/study/machine_learning/distillation/datatrain_logits.npy'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-ffcf2d6485d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_logits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_dir\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'train_logits.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mval_logits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_dir\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'val_logits.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/numpy/lib/npyio.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(file, mmap_mode, allow_pickle, fix_imports, encoding)\u001b[0m\n\u001b[1;32m    382\u001b[0m     \u001b[0mown_fid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbasestring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 384\u001b[0;31m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    385\u001b[0m         \u001b[0mown_fid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mis_pathlib_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/dongsung/study/machine_learning/distillation/datatrain_logits.npy'"
     ]
    }
   ],
   "source": [
    "train_logits = np.load(data_dir + 'train_logits.npy')[()]\n",
    "val_logits = np.load(data_dir + 'val_logits.npy')[()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 25600 images belonging to 256 classes.\n",
      "Found 5120 images belonging to 256 classes.\n"
     ]
    }
   ],
   "source": [
    "data_generator = ImageDataGenerator(\n",
    "    data_format='channels_last',\n",
    "    preprocessing_function=preprocess_input\n",
    ")\n",
    "\n",
    "# note: i'm also passing dicts of logits\n",
    "train_generator = data_generator.flow_from_directory(\n",
    "    data_dir + 'train', train_logits,\n",
    "    target_size=(224, 224),\n",
    "    batch_size=64\n",
    ")\n",
    "\n",
    "val_generator = data_generator.flow_from_directory(\n",
    "    data_dir + 'val', val_logits,\n",
    "    target_size=(224, 224),\n",
    "    batch_size=64\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "temperature = 5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_mobilenet(224, alpha=0.25, weight_decay=1e-5, dropout=0.1)\n",
    "\n",
    "# remove softmax\n",
    "model.layers.pop()\n",
    "\n",
    "# usual probabilities\n",
    "logits = model.layers[-1].output\n",
    "probabilities = Activation('softmax')(logits)\n",
    "\n",
    "# softed probabilities\n",
    "logits_T = Lambda(lambda x: x/temperature)(logits)\n",
    "probabilities_T = Activation('softmax')(logits_T)\n",
    "\n",
    "output = concatenate([probabilities, probabilities_T])\n",
    "model = Model(model.input, output)\n",
    "# now model outputs 512 dimensional vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)       (None, 225, 225, 3)  0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 112, 112, 8)  216         conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1_bn (BatchNormalization)   (None, 112, 112, 8)  32          conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv1_relu (ReLU)               (None, 112, 112, 8)  0           conv1_bn[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)     (None, 112, 112, 8)  72          conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormalizatio (None, 112, 112, 8)  32          conv_dw_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)           (None, 112, 112, 8)  0           conv_dw_1_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_1 (Conv2D)              (None, 112, 112, 16) 128         conv_dw_1_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormalizatio (None, 112, 112, 16) 64          conv_pw_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)           (None, 112, 112, 16) 0           conv_pw_1_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)      (None, 113, 113, 16) 0           conv_pw_1_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)     (None, 56, 56, 16)   144         conv_pad_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormalizatio (None, 56, 56, 16)   64          conv_dw_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)           (None, 56, 56, 16)   0           conv_dw_2_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_2 (Conv2D)              (None, 56, 56, 32)   512         conv_dw_2_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormalizatio (None, 56, 56, 32)   128         conv_pw_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)           (None, 56, 56, 32)   0           conv_pw_2_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)     (None, 56, 56, 32)   288         conv_pw_2_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormalizatio (None, 56, 56, 32)   128         conv_dw_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)           (None, 56, 56, 32)   0           conv_dw_3_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_3 (Conv2D)              (None, 56, 56, 32)   1024        conv_dw_3_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormalizatio (None, 56, 56, 32)   128         conv_pw_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)           (None, 56, 56, 32)   0           conv_pw_3_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)      (None, 57, 57, 32)   0           conv_pw_3_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)     (None, 28, 28, 32)   288         conv_pad_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormalizatio (None, 28, 28, 32)   128         conv_dw_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)           (None, 28, 28, 32)   0           conv_dw_4_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_4 (Conv2D)              (None, 28, 28, 64)   2048        conv_dw_4_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormalizatio (None, 28, 28, 64)   256         conv_pw_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)           (None, 28, 28, 64)   0           conv_pw_4_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)     (None, 28, 28, 64)   576         conv_pw_4_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormalizatio (None, 28, 28, 64)   256         conv_dw_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)           (None, 28, 28, 64)   0           conv_dw_5_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_5 (Conv2D)              (None, 28, 28, 64)   4096        conv_dw_5_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormalizatio (None, 28, 28, 64)   256         conv_pw_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)           (None, 28, 28, 64)   0           conv_pw_5_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pad_6 (ZeroPadding2D)      (None, 29, 29, 64)   0           conv_pw_5_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_6 (DepthwiseConv2D)     (None, 14, 14, 64)   576         conv_pad_6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_6_bn (BatchNormalizatio (None, 14, 14, 64)   256         conv_dw_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_6_relu (ReLU)           (None, 14, 14, 64)   0           conv_dw_6_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_6 (Conv2D)              (None, 14, 14, 128)  8192        conv_dw_6_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_6_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_pw_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_6_relu (ReLU)           (None, 14, 14, 128)  0           conv_pw_6_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_7 (DepthwiseConv2D)     (None, 14, 14, 128)  1152        conv_pw_6_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_7_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_dw_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_7_relu (ReLU)           (None, 14, 14, 128)  0           conv_dw_7_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_7 (Conv2D)              (None, 14, 14, 128)  16384       conv_dw_7_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_7_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_pw_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_7_relu (ReLU)           (None, 14, 14, 128)  0           conv_pw_7_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_8 (DepthwiseConv2D)     (None, 14, 14, 128)  1152        conv_pw_7_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_8_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_dw_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_8_relu (ReLU)           (None, 14, 14, 128)  0           conv_dw_8_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_8 (Conv2D)              (None, 14, 14, 128)  16384       conv_dw_8_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_8_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_pw_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_8_relu (ReLU)           (None, 14, 14, 128)  0           conv_pw_8_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_9 (DepthwiseConv2D)     (None, 14, 14, 128)  1152        conv_pw_8_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_9_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_dw_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_9_relu (ReLU)           (None, 14, 14, 128)  0           conv_dw_9_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_9 (Conv2D)              (None, 14, 14, 128)  16384       conv_dw_9_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_9_bn (BatchNormalizatio (None, 14, 14, 128)  512         conv_pw_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_9_relu (ReLU)           (None, 14, 14, 128)  0           conv_pw_9_bn[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_10 (DepthwiseConv2D)    (None, 14, 14, 128)  1152        conv_pw_9_relu[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_10_bn (BatchNormalizati (None, 14, 14, 128)  512         conv_dw_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_10_relu (ReLU)          (None, 14, 14, 128)  0           conv_dw_10_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_10 (Conv2D)             (None, 14, 14, 128)  16384       conv_dw_10_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_10_bn (BatchNormalizati (None, 14, 14, 128)  512         conv_pw_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_10_relu (ReLU)          (None, 14, 14, 128)  0           conv_pw_10_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_11 (DepthwiseConv2D)    (None, 14, 14, 128)  1152        conv_pw_10_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_11_bn (BatchNormalizati (None, 14, 14, 128)  512         conv_dw_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_11_relu (ReLU)          (None, 14, 14, 128)  0           conv_dw_11_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_11 (Conv2D)             (None, 14, 14, 128)  16384       conv_dw_11_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_11_bn (BatchNormalizati (None, 14, 14, 128)  512         conv_pw_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_11_relu (ReLU)          (None, 14, 14, 128)  0           conv_pw_11_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_pad_12 (ZeroPadding2D)     (None, 15, 15, 128)  0           conv_pw_11_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_12 (DepthwiseConv2D)    (None, 7, 7, 128)    1152        conv_pad_12[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_12_bn (BatchNormalizati (None, 7, 7, 128)    512         conv_dw_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_12_relu (ReLU)          (None, 7, 7, 128)    0           conv_dw_12_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_12 (Conv2D)             (None, 7, 7, 256)    32768       conv_dw_12_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_12_bn (BatchNormalizati (None, 7, 7, 256)    1024        conv_pw_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_12_relu (ReLU)          (None, 7, 7, 256)    0           conv_pw_12_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_13 (DepthwiseConv2D)    (None, 7, 7, 256)    2304        conv_pw_12_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_13_bn (BatchNormalizati (None, 7, 7, 256)    1024        conv_dw_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_dw_13_relu (ReLU)          (None, 7, 7, 256)    0           conv_dw_13_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_13 (Conv2D)             (None, 7, 7, 256)    65536       conv_dw_13_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_13_bn (BatchNormalizati (None, 7, 7, 256)    1024        conv_pw_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_pw_13_relu (ReLU)          (None, 7, 7, 256)    0           conv_pw_13_bn[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 256)          0           conv_pw_13_relu[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 256)          0           global_average_pooling2d_1[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          65792       dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 256)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 256)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 256)          0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 512)          0           activation_2[0][0]               \n",
      "                                                                 activation_3[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 284,336\n",
      "Trainable params: 65,792\n",
      "Non-trainable params: 218,544\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create custom loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def knowledge_distillation_loss(y_true, y_pred, lambda_const):    \n",
    "    \n",
    "    # split in \n",
    "    #    onehot hard true targets\n",
    "    #    logits from xception\n",
    "    y_true, logits = y_true[:, :256], y_true[:, 256:]\n",
    "    \n",
    "    # convert logits to soft targets\n",
    "    y_soft = K.softmax(logits/temperature)\n",
    "    \n",
    "    # split in \n",
    "    #    usual output probabilities\n",
    "    #    probabilities made softer with temperature\n",
    "    y_pred, y_pred_soft = y_pred[:, :256], y_pred[:, 256:]    \n",
    "    \n",
    "    return lambda_const*logloss(y_true, y_pred) + logloss(y_soft, y_pred_soft)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For testing use usual output probabilities (without temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(y_true, y_pred):\n",
    "    y_true = y_true[:, :256]\n",
    "    y_pred = y_pred[:, :256]\n",
    "    return categorical_accuracy(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def top_5_accuracy(y_true, y_pred):\n",
    "    y_true = y_true[:, :256]\n",
    "    y_pred = y_pred[:, :256]\n",
    "    return top_k_categorical_accuracy(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def categorical_crossentropy(y_true, y_pred):\n",
    "    y_true = y_true[:, :256]\n",
    "    y_pred = y_pred[:, :256]\n",
    "    return logloss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# logloss with only soft probabilities and targets\n",
    "def soft_logloss(y_true, y_pred):     \n",
    "    logits = y_true[:, 256:]\n",
    "    y_soft = K.softmax(logits/temperature)\n",
    "    y_pred_soft = y_pred[:, 256:]    \n",
    "    return logloss(y_soft, y_pred_soft)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lambda_const = 0.07\n",
    "\n",
    "model.compile(\n",
    "    optimizer=optimizers.SGD(lr=1e-1, momentum=0.9, nesterov=True), \n",
    "    loss=lambda y_true, y_pred: knowledge_distillation_loss(y_true, y_pred, lambda_const), \n",
    "    metrics=[accuracy, top_5_accuracy, categorical_crossentropy, soft_logloss]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "400/400 [==============================] - 24s - loss: 5.8086 - accuracy: 0.2173 - top_5_accuracy: 0.4171 - categorical_crossentropy: 4.0499 - soft_logloss: 5.5225 - val_loss: 5.6917 - val_accuracy: 0.4475 - val_top_5_accuracy: 0.7285 - val_categorical_crossentropy: 2.6774 - val_soft_logloss: 5.5013\n",
      "Epoch 2/30\n",
      "400/400 [==============================] - 23s - loss: 5.6726 - accuracy: 0.4953 - top_5_accuracy: 0.7514 - categorical_crossentropy: 2.4414 - soft_logloss: 5.4984 - val_loss: 5.6416 - val_accuracy: 0.5660 - val_top_5_accuracy: 0.8070 - val_categorical_crossentropy: 2.0523 - val_soft_logloss: 5.4944\n",
      "Epoch 3/30\n",
      "400/400 [==============================] - 24s - loss: 5.6378 - accuracy: 0.5785 - top_5_accuracy: 0.8072 - categorical_crossentropy: 2.0036 - soft_logloss: 5.4938 - val_loss: 5.6240 - val_accuracy: 0.5951 - val_top_5_accuracy: 0.8303 - val_categorical_crossentropy: 1.8349 - val_soft_logloss: 5.4916\n",
      "Epoch 4/30\n",
      "400/400 [==============================] - 23s - loss: 5.6212 - accuracy: 0.6181 - top_5_accuracy: 0.8376 - categorical_crossentropy: 1.7899 - soft_logloss: 5.4917 - val_loss: 5.6167 - val_accuracy: 0.6207 - val_top_5_accuracy: 0.8410 - val_categorical_crossentropy: 1.7269 - val_soft_logloss: 5.4914\n",
      "Epoch 5/30\n",
      "400/400 [==============================] - 23s - loss: 5.6116 - accuracy: 0.6418 - top_5_accuracy: 0.8532 - categorical_crossentropy: 1.6668 - soft_logloss: 5.4904 - val_loss: 5.6128 - val_accuracy: 0.6244 - val_top_5_accuracy: 0.8428 - val_categorical_crossentropy: 1.6673 - val_soft_logloss: 5.4915\n",
      "Epoch 6/30\n",
      "400/400 [==============================] - 23s - loss: 5.6048 - accuracy: 0.6636 - top_5_accuracy: 0.8650 - categorical_crossentropy: 1.5771 - soft_logloss: 5.4896 - val_loss: 5.6079 - val_accuracy: 0.6213 - val_top_5_accuracy: 0.8496 - val_categorical_crossentropy: 1.6315 - val_soft_logloss: 5.4888\n",
      "Epoch 7/30\n",
      "400/400 [==============================] - 23s - loss: 5.6001 - accuracy: 0.6760 - top_5_accuracy: 0.8740 - categorical_crossentropy: 1.5145 - soft_logloss: 5.4891 - val_loss: 5.6029 - val_accuracy: 0.6369 - val_top_5_accuracy: 0.8518 - val_categorical_crossentropy: 1.5752 - val_soft_logloss: 5.4875\n",
      "Epoch 8/30\n",
      "400/400 [==============================] - 24s - loss: 5.5966 - accuracy: 0.6841 - top_5_accuracy: 0.8798 - categorical_crossentropy: 1.4656 - soft_logloss: 5.4888 - val_loss: 5.6024 - val_accuracy: 0.6422 - val_top_5_accuracy: 0.8596 - val_categorical_crossentropy: 1.5532 - val_soft_logloss: 5.4884\n",
      "Epoch 9/30\n",
      "400/400 [==============================] - 23s - loss: 5.5935 - accuracy: 0.6929 - top_5_accuracy: 0.8843 - categorical_crossentropy: 1.4258 - soft_logloss: 5.4883 - val_loss: 5.6016 - val_accuracy: 0.6430 - val_top_5_accuracy: 0.8539 - val_categorical_crossentropy: 1.5479 - val_soft_logloss: 5.4878\n",
      "Epoch 10/30\n",
      "400/400 [==============================] - 23s - loss: 5.5913 - accuracy: 0.7013 - top_5_accuracy: 0.8882 - categorical_crossentropy: 1.3932 - soft_logloss: 5.4882 - val_loss: 5.6010 - val_accuracy: 0.6479 - val_top_5_accuracy: 0.8586 - val_categorical_crossentropy: 1.5299 - val_soft_logloss: 5.4882\n",
      "Epoch 11/30\n",
      "400/400 [==============================] - 23s - loss: 5.5893 - accuracy: 0.7086 - top_5_accuracy: 0.8922 - categorical_crossentropy: 1.3639 - soft_logloss: 5.4881 - val_loss: 5.6000 - val_accuracy: 0.6496 - val_top_5_accuracy: 0.8643 - val_categorical_crossentropy: 1.5198 - val_soft_logloss: 5.4878\n",
      "Epoch 12/30\n",
      "400/400 [==============================] - 23s - loss: 5.5876 - accuracy: 0.7136 - top_5_accuracy: 0.8962 - categorical_crossentropy: 1.3396 - soft_logloss: 5.4879 - val_loss: 5.6018 - val_accuracy: 0.6475 - val_top_5_accuracy: 0.8602 - val_categorical_crossentropy: 1.5318 - val_soft_logloss: 5.4886\n",
      "Epoch 13/30\n",
      "400/400 [==============================] - 23s - loss: 5.5863 - accuracy: 0.7170 - top_5_accuracy: 0.8991 - categorical_crossentropy: 1.3194 - soft_logloss: 5.4879 - val_loss: 5.5964 - val_accuracy: 0.6559 - val_top_5_accuracy: 0.8660 - val_categorical_crossentropy: 1.4852 - val_soft_logloss: 5.4863\n",
      "Epoch 14/30\n",
      "400/400 [==============================] - 24s - loss: 5.5855 - accuracy: 0.7220 - top_5_accuracy: 0.8981 - categorical_crossentropy: 1.3094 - soft_logloss: 5.4877 - val_loss: 5.5999 - val_accuracy: 0.6492 - val_top_5_accuracy: 0.8609 - val_categorical_crossentropy: 1.4970 - val_soft_logloss: 5.4889\n",
      "Epoch 15/30\n",
      "400/400 [==============================] - 24s - loss: 5.5845 - accuracy: 0.7254 - top_5_accuracy: 0.9034 - categorical_crossentropy: 1.2905 - soft_logloss: 5.4879 - val_loss: 5.5961 - val_accuracy: 0.6584 - val_top_5_accuracy: 0.8604 - val_categorical_crossentropy: 1.4825 - val_soft_logloss: 5.4860\n",
      "Epoch 16/30\n",
      "400/400 [==============================] - 24s - loss: 5.5835 - accuracy: 0.7258 - top_5_accuracy: 0.9059 - categorical_crossentropy: 1.2741 - soft_logloss: 5.4879 - val_loss: 5.5975 - val_accuracy: 0.6564 - val_top_5_accuracy: 0.8686 - val_categorical_crossentropy: 1.4793 - val_soft_logloss: 5.4875\n",
      "Epoch 17/30\n",
      "400/400 [==============================] - 23s - loss: 5.5794 - accuracy: 0.7401 - top_5_accuracy: 0.9128 - categorical_crossentropy: 1.2293 - soft_logloss: 5.4868 - val_loss: 5.5948 - val_accuracy: 0.6627 - val_top_5_accuracy: 0.8682 - val_categorical_crossentropy: 1.4618 - val_soft_logloss: 5.4860\n",
      "Epoch 18/30\n",
      "400/400 [==============================] - 24s - loss: 5.5788 - accuracy: 0.7466 - top_5_accuracy: 0.9130 - categorical_crossentropy: 1.2241 - soft_logloss: 5.4866 - val_loss: 5.5970 - val_accuracy: 0.6592 - val_top_5_accuracy: 0.8641 - val_categorical_crossentropy: 1.4787 - val_soft_logloss: 5.4870\n",
      "Epoch 19/30\n",
      "400/400 [==============================] - 23s - loss: 5.5783 - accuracy: 0.7441 - top_5_accuracy: 0.9142 - categorical_crossentropy: 1.2181 - soft_logloss: 5.4866 - val_loss: 5.5931 - val_accuracy: 0.6641 - val_top_5_accuracy: 0.8715 - val_categorical_crossentropy: 1.4433 - val_soft_logloss: 5.4856\n",
      "Epoch 20/30\n",
      "400/400 [==============================] - 23s - loss: 5.5787 - accuracy: 0.7448 - top_5_accuracy: 0.9116 - categorical_crossentropy: 1.2227 - soft_logloss: 5.4866 - val_loss: 5.5967 - val_accuracy: 0.6607 - val_top_5_accuracy: 0.8658 - val_categorical_crossentropy: 1.4738 - val_soft_logloss: 5.4871\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fb007d570f0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(\n",
    "    train_generator, \n",
    "    steps_per_epoch=400, epochs=30, verbose=1,\n",
    "    callbacks=[\n",
    "        EarlyStopping(monitor='val_accuracy', patience=4, min_delta=0.01), \n",
    "        ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=2, epsilon=0.007)\n",
    "    ],\n",
    "    validation_data=val_generator, validation_steps=80, workers=4\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss/epoch plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4XNV9//H3V9JoGe0ayZZtCUngBS9gG4TjBExoIMQQ\nAqRhMSUJadISWtKEpGlDnrZpml8X0o2WNC2BhAQSSiCGNJRCWMJOACPANjY22MYylldtlrWv5/fH\nvZLG8kga25oZSfN5Pc88c+fOuaOvRtJ8dM8991xzziEiIgKQkugCRERk8lAoiIjIEIWCiIgMUSiI\niMgQhYKIiAxRKIiIyBCFgoiIDFEoiIjIEIWCiIgMSUt0AcequLjYVVZWJroMEZEp5fXXX29wzpWM\n127KhUJlZSU1NTWJLkNEZEoxs13RtFP3kYiIDFEoiIjIEIWCiIgMmXLHFEREjkdvby91dXV0dXUl\nupSYyszMpKysjEAgcFzbKxREJCnU1dWRm5tLZWUlZpbocmLCOUdjYyN1dXVUVVUd12uo+0hEkkJX\nVxehUGjaBgKAmREKhU5ob0ihICJJYzoHwqAT/R6TJhS27j/MLY9tpaWzN9GliIhMWjEPBTNLNbM3\nzeyRCM9lmNn9ZrbdzF41s8pY1bG7qZPbn9vBzob2WH0JEZFRHTp0iP/8z/885u0uvvhiDh06FIOK\nIovHnsJXgC2jPPcFoNk5Nxe4FfhurIqoDAUBqFUoiEgCjBYKfX19Y2736KOPUlBQEKuyjhLTUDCz\nMuDjwA9HaXIZcLe/vBY432LU6VdeFMQMahsVCiISfzfffDM7duxg2bJlnHXWWaxatYpLL72URYsW\nAXD55Zdz5plnsnjxYu64446h7SorK2loaKC2tpaFCxfyh3/4hyxevJgLL7yQzs7OCa8z1kNS/w34\ncyB3lOfnALsBnHN9ZtYChICGiS4kM5DK7Pws7SmICH/zv5t5e+/hCX3NRbPz+OtPLB71+VtuuYVN\nmzaxfv16nn32WT7+8Y+zadOmoaGjd911F0VFRXR2dnLWWWfxqU99ilAodMRrbNu2jfvuu48777yT\nq666igcffJBPf/rTE/p9xGxPwcwuAQ46516fgNe63sxqzKymvr7+uF+nsjhIbWPHiZYjInLCVqxY\nccS5BLfddhtLly5l5cqV7N69m23bth21TVVVFcuWLQPgzDPPpLa2dsLriuWewtnApWZ2MZAJ5JnZ\nz5xz4bG2BygH6swsDcgHGke+kHPuDuAOgOrqane8BVWGsvm/t/Yd7+YiMk2M9R99vGRnZw8tP/vs\nszz11FO8/PLLBINBzjvvvIjnGmRkZAwtp6amxqT7KGZ7Cs65bzrnypxzlcAa4OkRgQDwMHCdv3yF\n3+a4P/THUxnK5lBHL4c6emL1JUREIsrNzaW1tTXicy0tLRQWFhIMBtm6dSuvvPJKnKsbFvdpLszs\nO0CNc+5h4EfAT81sO9CEFx4xU1nsJXNtYwfLgumx/FIiIkcIhUKcffbZLFmyhKysLGbOnDn03OrV\nq7n99ttZuHAhCxYsYOXKlQmr02L4j3lMVFdXu+O9yM62A6189Nbn+fc1y7hs2ZwJrkxEJrMtW7aw\ncOHCRJcRF5G+VzN73TlXPd62SXNGMwwPS9UJbCIikSVVKAwOS92lEUgiIhElVSiANyxVewoiIpEl\nXShUhLLZpbOaRUQiSrpQqApl09zRS0uHZksVERkp6UKhYnBiPO0tiIgcJelCoWroXAWFgohMXjk5\nOQn5ukkXCkOzpTZoBJKIyEhxP6M50YZmS9WegojE0c0330x5eTk33ngjAN/+9rdJS0vjmWeeobm5\nmd7eXv72b/+Wyy67LKF1Jl0ogHdcQcNSRZLYYzfD/rcm9jVLT4OLbhn16auvvpqbbrppKBQeeOAB\nHn/8cb785S+Tl5dHQ0MDK1eu5NJLL03otaSTMhQqi7N5TLOlikgcLV++nIMHD7J3717q6+spLCyk\ntLSUr371qzz//POkpKSwZ88eDhw4QGlpacLqTM5QCAWHhqXmBwOJLkdE4m2M/+hj6corr2Tt2rXs\n37+fq6++mnvvvZf6+npef/11AoEAlZWVEafMjqekO9AM3hTaoBFIIhJfV199NT//+c9Zu3YtV155\nJS0tLcyYMYNAIMAzzzzDrl27El1ikoaChqWKSAIsXryY1tZW5syZw6xZs7j22mupqanhtNNO4557\n7uHUU09NdInJ2X10koalikiCvPXW8AHu4uJiXn755Yjt2tra4lXSEZJyTyEzkMqsvEztKYiIjJCU\noQBeF5JCQUTkSMkdCjpXQSSpTLUrTR6PE/0ekzcUwoalisj0l5mZSWNj47QOBuccjY2NZGZmHvdr\nJOWBZjhyWOrSYEGCqxGRWCsrK6Ouro76+vpElxJTmZmZlJWVHff2yRsKYcNSl5YrFESmu0AgQFVV\nVaLLmPSStvtIw1JFRI6WtKEwOCxVl+YUERmWtKEAXhfSToWCiMiQpA6FilA2uxrVfSQiMiipQ6Gq\nOEhTew8tnRqWKiICMQwFM8s0s3VmtsHMNpvZ30Ro8zkzqzez9f7tD2JVTyQV/rBUHVcQEfHEckhq\nN/AR51ybmQWAF83sMefcKyPa3e+c+1IM6xhVlT8sdWdDO6eXaViqiEjMQsF5pw0OTvMX8G+T6lTC\nk4qCADquICLii+kxBTNLNbP1wEHgSefcqxGafcrMNprZWjMrj2U9I2UGUpmdn6k5kEREfDENBedc\nv3NuGVAGrDCzJSOa/C9Q6Zw7HXgSuDvS65jZ9WZWY2Y1E32KekVIs6WKiAyKy+gj59wh4Blg9Yj1\njc65bv/hD4EzR9n+DudctXOuuqSkZEJr86bQVveRiAjEdvRRiZkV+MtZwEeBrSPazAp7eCmwJVb1\njKYypGGpIiKDYjn6aBZwt5ml4oXPA865R8zsO0CNc+5h4MtmdinQBzQBn4thPRENToy3q1EjkERE\nYjn6aCOwPML6b4UtfxP4ZqxqiMbgFNoalioikuRnNANUhDQsVURkUNKHQmYglVkalioiAigUAK8L\nScNSRUQUCoCGpYqIDFIooGGpIiKDFAocOSxVRCSZKRQYHpaqLiQRSXYKBYaHpWoEkogkO4UCYcNS\n1X0kIklOoeCrDGVrT0FEkp5CwVdZHNRZzSKS9BQKvspQNo3tPRzu0rBUEUleCgVfhT8CaVeD9hZE\nJHkpFHxV/rkKO3WwWUSSmELBd1KRP1uqDjaLSBJTKPiy0r1hqdpTEJFkplAIUxHSCCQRSW4KhTBV\nxTpXQUSSm0IhTIWGpYpIklMohKnUsFQRSXIKhTCVxf7EeDrYLCJJSqEQpqLIn0JbxxVEJEkpFMJk\npadSmqdhqSKSvBQKI2hiPBFJZgqFETQsVUSSmUJhBA1LFZFkFrNQMLNMM1tnZhvMbLOZ/U2ENhlm\ndr+ZbTezV82sMlb1REvDUkUkmcVyT6Eb+IhzbimwDFhtZitHtPkC0OycmwvcCnw3hvVERcNSRSSZ\nxSwUnKfNfxjwb25Es8uAu/3ltcD5ZmaxqikaGpYqIskspscUzCzVzNYDB4EnnXOvjmgyB9gN4Jzr\nA1qAUCxrGs/gsNRajUASkSQU01BwzvU755YBZcAKM1tyPK9jZtebWY2Z1dTX109skRFUFgfVfSQi\nSSkuo4+cc4eAZ4DVI57aA5QDmFkakA80Rtj+DudctXOuuqSkJNblUhnKZpdCQUSSUCxHH5WYWYG/\nnAV8FNg6otnDwHX+8hXA0865kccd4q6yOJuGth5aNSxVRJJMLPcUZgHPmNlG4DW8YwqPmNl3zOxS\nv82PgJCZbQe+Btwcw3qiVhnyL82p4woikmTSYvXCzrmNwPII678VttwFXBmrGo5XZbE3AmlnQztL\n5uQnuBoRkfjRGc0RDA5L1XEFEUk2CoUIhmZL1VnNIpJkFAqjqAgFtacgIklHoTCKquJsnasgIklH\noTCKipCGpYpI8jnmUDCzFDPLi0Uxk0lVsYalikjyiSoUzOy/zSzPzLKBTcDbZvZnsS0tsSr8KbTV\nhSQiySTaPYVFzrnDwOXAY0AV8JmYVTUJVPgnsGm2VBFJJtGGQsDMAnih8LBzrpejp8GeVoLpaczM\ny9BsqSKSVKINhR8AtUA28LyZVQCHY1XUZFEZ0vWaRSS5RBUKzrnbnHNznHMX+xfP2QX8ToxrS7jK\nULb2FEQkqUR7oPkr/oFmM7MfmdkbwEdiXFvCebOldmtYqogkjWi7jz7vH2i+ECjEO8h8S8yqmiQ0\nLFVEkk20oTB43eSLgZ865zaHrZu2NCxVRJJNtKHwupk9gRcKj5tZLjAQu7ImBw1LFZFkE+31FL4A\nLAPec851mFkI+P3YlTU5aFiqiCSbaEcfDQBlwF+a2T8DH/IvojN11L4EP74Yuo5tJK2GpYpIMol2\n9NEtwFeAt/3bl83s72NZ2IRLy4BdL8GWh49pMw1LFZFkEu0xhYuBjzrn7nLO3QWsBi6JXVkxMOdM\nCM2FDT8/ps00LFVEksmxzJJaELY89S5cbAZL10DtC3Do/ag3qwxpWKqIJI9oQ+EfgDfN7Cdmdjfw\nOvB3sSsrRk6/2rvfeH/Um1QWa1iqiCSPaA803wesBB4CHgQ+6JyL/pN1sig4CSpXeV1ILrr5/Cq0\npyAiSWTMUDCzMwZvwCygzr/N9tdNPUvXQON22PN6VM0Hh6Xu1AgkEUkC452n8C9jPOeYivMfLbwU\n/u/rsOE+KKuOapOKUDa71H0kIklgzFBwzk2/mVAz82DhJbDpQfjY33tDVcdRFcrmN1sPxqE4EZHE\niuqMZjP73QirW4C3nHNT79Ny6Rp46xew7QlY+Ilxm1cUB2lo66atu4+cjGhPAhcRmXqiHX30BeCH\nwLX+7U7gG8BLZhbxspxmVm5mz5jZ22a22cy+EqHNeWbWYmbr/du3jvP7ODZV50FOadTnLFQNToyn\n4woiMs1F+29vGrDQOXcAwMxmAvcAHwCeB34aYZs+4E+dc2/4E+i9bmZPOufeHtHuBedcfE+ES02D\n06+EV26H9kbIDo3ZfHC21F2NHSyZM/VO0RARiVa0ewrlg4HgO+ivawIinurrnNvnnHvDX24FtgBz\nTqTYCbX0Ghjohc0Pjdu00r+ugs5VEJHpLtpQeNbMHjGz68zsOuBhf102cGi8jc2sElgOvBrh6Q+a\n2QYze8zMFo+y/fVmVmNmNfX19VGWPI6Zi6H0NG8U0jiC6WnMyM1Q95GITHvRhsKNwI/xps9eBtwN\n3Oicax9vhJKZ5eCd8HaTf/W2cG8AFc65pcD3gP+J9BrOuTucc9XOueqSkpIoS47C0mu88xXq3x23\naWVxtvYURGTai/aMZge8CDwN/AZ43l83JjML4AXCvc65o/ppnHOHnXNt/vKjQMDMio+h/hOz5Aqw\nFNg4/gHnylBQs6WKyLQX7dTZVwHrgCuAq4BXzeyKcbYx4EfAFufcv47SptRvh5mt8OtpjL78E5Q7\nE045HzbcDwNjX0iusjib+lZvWKqIyHQVbffRXwBnOeeuc859FlgB/NU425wNfAb4SNiQ04vN7AYz\nu8FvcwWwycw2ALcBa6LZA5lQS9fA4TrY9eKYzU4pyQHgtZ1N8ahKRCQhoh2SmjLiJLVGxgkU59yL\ngI3T5j+A/4iyhtg49eOQkeeds1B17qjNzltQwpyCLG596l3OW1CCv4MjIjKtRLun8Gsze9zMPmdm\nnwP+D3g0dmXFUSALFl0Gb/8KekY/kJyRlspNF8xjY10Lj28+MGo7EZGpLNoDzX8G3AGc7t/ucM59\nI5aFxdXSa6CnDbb+35jNPrl8DqeUZPMvT7xD/0B8e7lEROIh6iuvOecedM59zb/9MpZFxd1JH/Su\ntTDOOQtpqSn86YUL2Hawjf95c0+cihMRiZ/xrqfQamaHI9xazWzkOQdTV0oKnL4G3nsWDu8ds+lF\nS0o5bU4+tz71Lj19Y49YEhGZasY7WJzrnMuLcMt1zuXFq8i4WLoG3IA3e+oYzIyvf2wBdc2d3P9a\n9Nd6FhGZCqLuPpr2QqdA2QpYf9+4l+o8d14xK6qKuO3p7XT29MepQBGR2FMohFu6Buq3wP6NYzYz\nM/7sYwuob+3m7pdr41KaiEg8KBTCLf4kpKZHdZ2FsyqL+J0FJfzXszto6Yw4UayIyJSjUAgXLIL5\nq73jCv3jf9D/6YULaOns5YcvvBeH4kREYk+hMNLSa6C9HnY8PW7TJXPyueT0WfzoxZ00tHXHoTgR\nkdhSKIw09wIIhqK6zgLA1z46n+6+Af7zmR0xLkxEJPYUCiOlpXtTam99FDrHvX4QJ5fkcMUZZfzs\nlV3sOdQZhwJFRGJHoRDJ0jXQ3w1vR7zmz1G+fME8AG57alssqxIRiTmFQiSzl0PxgqhGIQHMKcji\n2pUnsfaNOnbUt8W4OBGR2FEoRGLm7S28/zI0RTey6MbfmUtGWgq3Pjn+pT1FRCYrhcJoTr8KMNj4\nQFTNi3My+MI5VTyycR+b97bEtjYRkRhRKIwmv8y76M6G8ae9GPQHq04mPyvAvzyhvQURmZoUCmNZ\neg0018LuV6Nqnp8V4IYPn8LTWw9SU6vLdorI1KNQGMvCT0AgGPU5CwDXfaiCktwM/vHX7xDvy02L\niJwohcJYMnJg4aWw6ZfQ2xXVJsH0NP7kI3NZV9vE89saYlygiMjEUiiMZ+ka6G6Bdx+LepM1Z51E\nWWEW//T4Vu0tiMiUolAYT9W5kDs76nMWANLTUvjqBfPZtOcwv960P4bFiYhMLIXCeFJSveGp256E\ntvqoN7t8+Rzmzsjhn594h75+XbZTRKYGhUI0lq4B1w+b1ka9SWqK8fUL57Ojvp1fvrknhsWJiEwc\nhUI0ZiyEWcuOaRQSwMcWl3J6WT7/9tQ2uvt02U4RmfwUCtFaeg3s2wAb7o96k8HLdu451MnP1+2O\nYXEiIhMjZqFgZuVm9oyZvW1mm83sKxHamJndZmbbzWyjmZ0Rq3pO2BmfhcpV8MsvQs2Po97snLnF\nrDy5iO89vZ2Onr4YFigicuJiuafQB/ypc24RsBK40cwWjWhzETDPv10P/FcM6zkx6UG49hcw76Pw\nyE3w2+9Ftdng3kJDWzc/fqk2tjWKiJygmIWCc26fc+4Nf7kV2ALMGdHsMuAe53kFKDCzWbGq6YQF\nsuDqe2HR5fDEX8Iz/xDVvEhnVhRx/qkz+MFzO9jfEt1JcCIiiRCXYwpmVgksB0ZOIjQHCO9sr+Po\n4MDMrjezGjOrqa+PflhoTKSlwxV3wbJPw3O3eOEQRTB846JT6R9wXPmD37KrsT0OhYqIHLuYh4KZ\n5QAPAjc55w4fz2s45+5wzlU756pLSkomtsDjkZIKl34PVnwRXv4P+N8vw8DYo4vmz8zl3j9cSWtX\nH1fc/jJb9x/XWyEiElMxDQUzC+AFwr3OuYciNNkDlIc9LvPXTX4pKXDRd2HV1+GNe+Ch66G/d8xN\nlpUX8IsvfpAUg6t/8ApvvN8cp2JFRKITy9FHBvwI2OKc+9dRmj0MfNYfhbQSaHHO7YtVTRPODM7/\nK7jg296JbQ98dtyJ8+bNzGXtDR+iIBjg2jtf5YVtCe4OExEJE8s9hbOBzwAfMbP1/u1iM7vBzG7w\n2zwKvAdsB+4E/jiG9cTOOV+Fi/8Z3nkU/vsq6B77Os3lRUF+ccMHqQgF+fxPXuOxt6ZODorI9GZT\nbRbP6upqV1NTk+gyIlt/H/zqj6HsLPi9ByCrYMzmLR29fP7u13jz/WZu+d3Tueqs8jHbi4gcLzN7\n3TlXPV47ndE8kZZdA1f+BPa8AXdfAu1jX08hPxjgp19YwTnzSvjzBzdy5/PvxadOEZFRKBQm2qLL\n4Pd+Dg3b4ccXweG9YzYPpqfxw89W8/HTZvF3j27RNRhEJKEUCrEw9wL49INweB/ctRqado7ZPD0t\nhduuWc41K8r5/jM7+KtfbWJgQMEgIvGnUIiVyrPhul9B92Fvj6H+nTGbp6YYf//J0/jih0/mZ6+8\nz033r6dX12EQkThTKMTSnDPhc496J7b9+CJvltUxmBnfvGgh31h9Kg9v2Mv199TQ2aMpt0UkfhQK\nsTZzEXz+1xAIwk8ugef/GToPjbnJH513Cn//ydN49t16rrtrHYe7xj4pTkRkoigU4iF0ihcM5Svg\n6f8Hty6BJ78FrQdG3eT3PnASt61Zzpu7m7nmjldoaOuOY8EikqwUCvGSX+YdfP7iC97027/9Hvzb\nafC/N0FT5KGon1g6mzs/W82O+jauuv1l9hzqjHPRIpJsFArxNut0uPLH8KUa77yG9ffC986EtZ+H\n/W8d1fy8BTP42Rc+QH1bN1f812956u0DGrIqIjGjM5oTrXU/vPx9qLkLetpg7kdh1deg4kNHNHt7\n72Fu/O832NnQTnVFITdfdCrVlUUJKlpEpppoz2hWKEwWnc3w2g/hlduhowHKV3pzKs3/mDfxHtDb\nP8ADNbv5t6e2Ud/azQULZ/Lnqxcwf2ZugosXkclOoTBV9XTAmz/zjjm0vA8zFnnhsPh3ITUNgI6e\nPn78Ui23P7uD9p4+PnVGGV/96HxmF2QluHgRmawUClNdfy9seghevBXqt0DBSfDBP4FTPw753sXp\nmtt7+P4z27nn5V1g8LkPVfLH551CQTA9wcWLyGSjUJguBgZg2+Pwwr9C3TpvXWguVH0Yqs6FqnOp\n687k1ie38dCbdeRkpPFH553C73+oiqz01MTWLiKThkJhunEODmyC956Fnc/Drt96B6YxKF0CVR9m\nd8FZ/MPmIh59t5WZeRl85fz5XFVdRlqqBpmJJDuFwnTX3+tN0b3zOS8kdr8K/T2QkkZr8VJ+3T6f\nB5tPobloGTd9bAmrl5Ri/gFrEUk+CoVk09PhBcPO52Hnc7i9b2JugG7SWdc/n5151Sxb9QlOW74S\ny8hJdLUiEmcKhWTX1QK1LzGw8zkOv/0bClq3DT3VlllKRumpBGYsgOJ5UDzfu+WWDg1/FZHpRaEg\nR+hq3se65x+ldut6stveY17KPuan7iNzoGO4UXpuWEjMG14uOhnSMhJXvIicMIWCjGrTnhYeqNnN\nL9+sI6urgbMLGrm8vJ2zshsIHn4PGrbB4brhDSwFCiuheIE3qV/lOTBrGaRp6KvIVKFQkHF19fbz\n2KZ9/Hzdbl7d2URqivGRU2ew5qxyPlyZRVqzHxAN73q3g2979+BNBV52lhcQFWd7144IZCb2GxKR\nUSkU5Ji8V9/GAzV1rH29joa2bkrzMrmyuoyrqsspLwoON2yrh/d/C7Uvwa6X4MBmwEFqBpRVewFR\nebYXGOnZCft+RORICgU5Lr39Azy99SD3v7abZ985yICDc+YWc/VZ5Vy4eCYZaSNOiOtogvdf8QKi\n9kXYvxHcAKSkwewzvICoOAdO+gBkaI4mkURRKMgJ29fSydqaOu6v2U1dcyf5WQE+PL+EVfOKWTWv\nhNL8CN1FXYe9obG1L3pBsfdNGOjzjkuUngYFFZAzE3JnQk5p2PJMyC6BFJ2FLRILCgWZMAMDjt/u\naOShN+p4flvD0FXg5s3IYdW8ElbNL+YDVUUE09OO3rinHXav8wKi7jU4vA/a9ntDZkeyFC8YcmaM\nCIxSf90MCGRBSgBS0yF18D7dmyxwcDklTUNrRUZQKEhMOOfYur+VF7bV88K2BtbtbKK7b4D01BTO\nrChk1fxizp1XwqJZeaSkjPHB3NsFbQeGb637oe2gFxhtB8MeHwDXf+yFhgfE4HJaOmQWQHYxBEP+\nrShsOQRB/7msAu21yLSS8FAws7uAS4CDzrklEZ4/D/gVsNNf9ZBz7jvjva5CYXLp6u3ntdomXtjW\nwPPv1rN1fysARdnpnDO3mHPmFbNqXjGz8o9zWu+BAeho9MKh/SD09XjTefT3eFN9hC8P9I5YH/58\nH/R1Quch7/U6mrzrVvR2jPKFDbIKh8Miu9h7nJHrjbxKD0Ig2zuYPrQ8yrq0zCP3XJzz9qC6W735\nq7pbw5bboPtw2HJYm95O73rfs5d7t9BcBZdEbTKEwrlAG3DPGKHwdefcJcfyugqFye1gaxcvbW/g\nhXcbjupqOmdeMWdWFLKsvIA5BVmTYy6mng7obPKDohHaG4eXI9162r0bx/B3YyleQAQyvT2knrbo\ntrcU74TCjFzIyPH2dhq3DwdZeg7MWjocErOXQ2EVpGgCRDlawkPBL6ISeEShkJycc7xzoNUPiPqh\nriaA4pwMlpXns7SsgGUnFXB6WQH5WYEEVxwl56CvywuU3nbvvqd9eHnovsMLgMHl3g5Iyxr+kE/P\ngYy8sGU/AAaXA1lHHxsZ6PfOFdn75vBt/1tePQAZ+TB7RFAUVBzbMZbB72/kXktvh7fXk5nvda9l\n5nuhpRCaEqZKKDwI1AF78QJi8yivcz1wPcBJJ5105q5du2JUscRST98A7+xvZf3uZtbvbmH97mZ2\n1LcPPX9ySTbL/JBYWlbAwll5pKfpA2dc/b1Qv3VEUGzyutPA6/aavRxKT/e6m7pbvQ/5nsH7tqPX\nRX0cxyAzzwuIzHzvmM0R92EBkpHnHdexVO9YT4p/b6n+8liP/fYZeUNXIIy7gf4p3V03FUIhDxhw\nzrWZ2cXAvzvn5o33mtpTmF4Od/WycXcLG+oO8eb7h1i/+9BQl1N6agqLZuexrLyAZeUFLC0voKIo\nOPYBbPH0dXtnoIcHxcEt3l5ARo7fLZUTtmeSc2RXVXrOkXstGTleF1hfpzdybPDWeejIx0fcDvld\nZRMsM98fDFA0YsBAUYR1IS8UU8P2Qp3z6upo8rsOm7xrpB/xOGx9ZxN0NEN3i/d6oXlHzg0WmudN\nAzORYTXQD4f3QHPtkbd5H4OlVx/XS076UIjQthaods41jNVOoTC9OefY29LFht1eQKzffYi36lro\n7PX+cw2mpzJvZi4LZuawoDSPBTNzmV+aQ0lOxuQ4RjGZDfR7xyni+T7193ldUF1+ePT3enUM9Hl7\nIwN9/uPwdSMf9w0/7moJGyjQOPzh3dE4xqABvG61rAKvW6yz2Rt8MGrbPH+QwWDI+PeZ+d6Ah8bt\nXhdee/0CvQtJAAANlElEQVTwNikBKKryQ2Lu8MzDxXO914qkq+XoD/3B26Hdw3t64O0tFZTDiuvh\ngzdG++4fIdpQSNB+GJhZKXDAOefMbAWQAjQmqh6ZHMyMOQVZzCnI4uLTZgHQ1z/AtoNtbKw7xNb9\nrbyzv5Wntx7kgZrhSfuKstOZPzOHU0vzmD8zlwWlucyfmUNu5hQ5ThEPiej6SE0b/i8+1no7R4TF\nYHgMrmv2DvaHf9Af9eE/Yq9iLJ3N0OAHROM2f56wbfDu40d+oGeXeHsToVO8Y0/NtdC809s+XFah\nt8cxayksusxbHrzllcWt2yyWo4/uA84DioEDwF8DAQDn3O1m9iXgj4A+oBP4mnPut+O9rvYUZFBD\nWzfv7m9l6/5W3j0wfN/RM9wfPqcgyw+IXE4tzaWqOJvKUDb5QYWFxEh/Hxza5U8k6U8o2bjdu2Xk\nHvlhP3grqPD2ZGJoUnQfxYJCQcYyMODYc6iTd/a38s4Bb6/inf2t7Khvo29g+He9MBigIpRNVXE2\nFaEglaFsKouzqQwFKQhqSnCZfiZ995FILKSkGOVFQcqLglywaObQ+p6+AWob29nZ0M6uxnZ2NnSw\nq7GddTub+J/1ewj/36hgMDBCQSpC2VQW+6ERyqYgGNCxC5nWFAqSFNLTUpg/0+tGGqmrt5/dTR3U\nNnZQ29BObWM7uxo7eK22mV9t2HtEYORmpDGnMMsLnsIgZYPLRVmUFQbJydCflExt+g2WpJcZ8EY0\nzYsQGN19/exu6hwKi7rmTnY3dfB+YwcvbW844vgFeN1S5UV+WBQGKQtfLswiMzB1x7lLclAoiIwh\nIy2VuTNymDsj56jnnHM0tfd4QdHcwe6mTuqaO9jd3MnWfa089fZBevoHjtimKDudmXmZlOZlUJqf\n6S9nMjPfuy/Ny1QXlSSUQkHkOJkZoZwMQjkZLC0/euTIwICjvq2b3U0dQ3sY+w93ceBwF/taunhr\nTwsNbUePl89ISzkiLGaFhUdxTjrFuRkUZ2eQl5Wm8JAJp1AQiZGUFGNmnveBXl0ZuU1P3wAHW72g\n2N/SPRQa+1u62H+4i411h3hic9fQnFHhAqlGKDuD4tx07z4nwwuNnAxCYfclORkUZqcTSNWUITI+\nhYJIAqWnpVBWGKSsMDhqG+ccLZ297GvporGth4a2bv/WQ2PY8rYDrTS09RzVZTWoIBggJyONYHoq\nwfTB+1Sy0tPITk8ly3888rlgwF+fkUZhMEBRdjo5GdpLma4UCiKTnJlREEyP6vwJ5xyt3X00tHbT\n2N5DQ2s3Df59Y3s37d39dPT00dHTT0dPP80dvXT6jzt7+uno7ad/YPxzl9JTUyjKTqcoO51QTvrw\ncnY6RdkZR6wPZaeTlxnQnFVThEJBZBoxM/IyA+RlBji55Ni3d87R3TcwFBCdYQHS3t3HoY5eGtu9\nwGlq66GpvYfG9h52NXbQ1N5DW3dfxNdNTTEKg+kUBgMUBtPJDwYoyApQmJ1OflaAAn99QVaA/MHl\nYICsQKr2SOJMoSAiQ8yMzEAqmYFURpnGbUxdvf00d/TQ6AfGYGg0tXfT1N5Dc3svhzp72N3UwVsd\n3nJXb+TuLvC61wr80CgIpnNKSQ5L5uSxZHY+C0pzNcQ3BhQKIjJhMgOpzMrPOqbLr3b19nPID4hD\nHb0c6vDvO3tp7uihpaN3aA/l0bf2cd+69wFv72PejByWzMlnyew8lszJZ+GsPLJ1AuEJ0bsnIgmV\nGUilND+V0vzMcds656hr7mTz3hY27TnMpr0tPPvOQda+7s2YawYnF2f7QZHP4jl5LJ6VrwkQj4FC\nQUSmDLPhua1WL/GmVnfOcbC1m017hoPitZ1N/Gr93qHtyouyWDwrn8LsdNJTjUBqCoG0FAIpYcup\nKaSnGmmp3nIg1UhPTfEfe8uBtBTvPjWF9LQUMtK8+8HH6X7b4z0OMjDg6B0YoK/f0ds/QG+/oy/s\ncV5WgOKcjAl5L0ejUBCRKc1s+HyQ8xcOT4LY2NbN5r1eSGzec5gt+w7T2t3nfdj2eR+4ow3fPVHp\nqYNhYV5Y+MHhHPT2ex/yfQP+h37/AL0D3v14A7/+6LxT+MbqU2NS8yCFgohMS6GcDM6dX8K580cf\nhuWco3/ADQXE4Ad2b//A0OPePu+/954+7/HgfXff4LKjp6/fb+/C1nv3Q+37B0gxI5BipA3ukaR4\n92mpRiDFv09NIc1fP7jnkubv0cybefR0KxNNoSAiScts8AMastBIJvAugSkiIgIoFEREJIxCQURE\nhigURERkiEJBRESGKBRERGSIQkFERIYoFEREZIg5N/4FNSYTM6sHdh3n5sVAwwSWM9Eme30w+WtU\nfSdG9Z2YyVxfhXNu3KtsTLlQOBFmVuOcq050HaOZ7PXB5K9R9Z0Y1XdiJnt90VD3kYiIDFEoiIjI\nkGQLhTsSXcA4Jnt9MPlrVH0nRvWdmMle37iS6piCiIiMLdn2FEREZAzTMhTMbLWZvWNm283s5gjP\nZ5jZ/f7zr5pZZRxrKzezZ8zsbTPbbGZfidDmPDNrMbP1/u1b8arP//q1ZvaW/7VrIjxvZnab//5t\nNLMz4ljbgrD3Zb2ZHTazm0a0ifv7Z2Z3mdlBM9sUtq7IzJ40s23+feEo217nt9lmZtfFsb5/MrOt\n/s/wl2ZWMMq2Y/4+xLC+b5vZnrCf48WjbDvm33sM67s/rLZaM1s/yrYxf/8mlHNuWt2AVGAHcDKQ\nDmwAFo1o88fA7f7yGuD+ONY3CzjDX84F3o1Q33nAIwl8D2uB4jGevxh4DDBgJfBqAn/W+/HGXyf0\n/QPOBc4ANoWt+0fgZn/5ZuC7EbYrAt7z7wv95cI41XchkOYvfzdSfdH8PsSwvm8DX4/id2DMv/dY\n1Tfi+X8BvpWo928ib9NxT2EFsN05955zrgf4OXDZiDaXAXf7y2uB8+14r7R9jJxz+5xzb/jLrcAW\nYE48vvYEugy4x3leAQrMbFYC6jgf2OGcO96TGSeMc+55oGnE6vDfs7uByyNs+jHgSedck3OuGXgS\nWB2P+pxzTzjn+vyHrwBlE/11ozXK+xeNaP7eT9hY9fmfHVcB9030102E6RgKc4DdYY/rOPpDd6iN\n/0fRAoTiUl0Yv9tqOfBqhKc/aGYbzOwxM1sc18LAAU+Y2etmdn2E56N5j+NhDaP/ISby/Rs00zm3\nz1/eD8yM0GayvJefx9v7i2S834dY+pLfvXXXKN1vk+H9WwUccM5tG+X5RL5/x2w6hsKUYGY5wIPA\nTc65wyOefgOvS2Qp8D3gf+Jc3jnOuTOAi4AbzezcOH/9cZlZOnAp8IsITyf6/TuK8/oRJuVQPzP7\nC6APuHeUJon6ffgv4BRgGbAPr4tmMrqGsfcSJv3fU7jpGAp7gPKwx2X+uohtzCwNyAca41Kd9zUD\neIFwr3PuoZHPO+cOO+fa/OVHgYCZFcerPufcHv/+IPBLvF30cNG8x7F2EfCGc+7AyCcS/f6FOTDY\nrebfH4zQJqHvpZl9DrgEuNYPrqNE8fsQE865A865fufcAHDnKF830e9fGvC7wP2jtUnU+3e8pmMo\nvAbMM7Mq/7/JNcDDI9o8DAyO8rgCeHq0P4iJ5vc//gjY4pz711HalA4e4zCzFXg/p7iElpllm1nu\n4DLewchNI5o9DHzWH4W0EmgJ6yaJl1H/O0vk+zdC+O/ZdcCvIrR5HLjQzAr97pEL/XUxZ2argT8H\nLnXOdYzSJprfh1jVF36c6pOjfN1o/t5j6QJgq3OuLtKTiXz/jluij3TH4oY3OuZdvFEJf+Gv+w7e\nLz9AJl63w3ZgHXByHGs7B68bYSOw3r9dDNwA3OC3+RKwGW8kxSvAh+JY38n+193g1zD4/oXXZ8D3\n/ff3LaA6zj/fbLwP+fywdQl9//ACah/Qi9ev/QW841S/AbYBTwFFfttq4Idh237e/13cDvx+HOvb\njtcfP/h7ODgibzbw6Fi/D3Gq76f+79dGvA/6WSPr8x8f9fcej/r89T8Z/L0Laxv3928ibzqjWURE\nhkzH7iMRETlOCgURERmiUBARkSEKBRERGaJQEBGRIQoFkTjyZ3B9JNF1iIxGoSAiIkMUCiIRmNmn\nzWydPwf+D8ws1czazOxW866D8RszK/HbLjOzV8KuS1Dor59rZk/5E/O9YWan+C+fY2Zr/WsZ3Buv\nGXpFoqFQEBnBzBYCVwNnO+eWAf3AtXhnUtc45xYDzwF/7W9yD/AN59zpeGfgDq6/F/i+8ybm+xDe\nGbHgzYx7E7AI74zXs2P+TYlEKS3RBYhMQucDZwKv+f/EZ+FNZjfA8MRnPwMeMrN8oMA595y//m7g\nF/58N3Occ78EcM51Afivt875c+X4V+uqBF6M/bclMj6FgsjRDLjbOffNI1aa/dWIdsc7R0x32HI/\n+juUSUTdRyJH+w1whZnNgKFrLVfg/b1c4bf5PeBF51wL0Gxmq/z1nwGec95V9erM7HL/NTLMLBjX\n70LkOOg/FJERnHNvm9lf4l0tKwVvZswbgXZghf/cQbzjDuBNi327/6H/HvD7/vrPAD8ws+/4r3Fl\nHL8NkeOiWVJFomRmbc65nETXIRJL6j4SEZEh2lMQEZEh2lMQEZEhCgURERmiUBARkSEKBRERGaJQ\nEBGRIQoFEREZ8v8BO6upO01BxZIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb007d6a128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['categorical_crossentropy'], label='train');\n",
    "plt.plot(model.history.history['val_categorical_crossentropy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('logloss');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8XNWZ//HPo9673CT3BjYYA6YklNBDWVoKEEKWlA3J\nBgLsZjdLdn+bsGz5JeyGzZIly5IN/NLohOBkTcdAChALAm64SdhYctFIlmSNejm/P+7VeCyrjI3u\njKT5vl+veemWM6NHI+k+c88597nmnENERAQgJdEBiIjI+KGkICIiEUoKIiISoaQgIiIRSgoiIhKh\npCAiIhFKCiIiEqGkICIiEUoKIiISkZboAA5XWVmZmzNnTqLDEBGZUN58880G51z5aO0mXFKYM2cO\nVVVViQ5DRGRCMbMdsbRT95GIiEQoKYiISISSgoiIREy4MYWh9PT0UFtbS2dnZ6JDCVRWVhaVlZWk\np6cnOhQRmaQmRVKora0lPz+fOXPmYGaJDicQzjkaGxupra1l7ty5iQ5HRCapSdF91NnZSWlp6aRN\nCABmRmlp6aQ/GxKRxJoUSQGY1AlhQDL8jCKSWJOi+0hEJAjhrl62N7RRHQqzc1876akp5GWlkZ+V\nTn5WGgVRy/lZ6eRmpE74D29KCmOgubmZBx98kK985SuH9byLL76YBx98kKKiooAiE5HR9Pb1U9vU\nwXv+wb+moY33Qm3UNITZu7/rsF4rxSA3M42CSKI4kDTyMtP87+fo7Xf09vd7X/v66ev3t/X52/02\nff2Onqj9t563kMuXVwTxNkQoKYyB5uZmfvCDHxySFHp7e0lLG/4tXrVqVdChiYivqa2bbaEwNf6B\nvybURk0ozPv72unpc5F2RTnpzCvL5fQF5cwrz2V+eS7zyvOYVZJDX78j3NVLa2cP+zt7ae30lg/+\n2st+fznc2Ut9ayfVIW/ZDNJSUkhNMdJSjbQUIy0lJbLsbU8hIy2FnNSUyLb0VCM1JYXS3MzA3ycl\nhTFw2223UV1dzfLly0lPTycrK4vi4mI2bdrEli1buOKKK9i5cyednZ3ccsst3HDDDcCBkh3hcJiL\nLrqI008/nd///vdUVFTw1FNPkZ2dneCfTGRi27u/k1XrdrNq3W6qdjTh/GN/eqoxuzSX+eV5nL9k\nGvPKc5lX5h38S3IzRnzN3Mw0phZkxSH6xJh0SeEffrWBjbv2j+lrLplRwLcuXTrs/m9/+9usX7+e\nt99+m5dffplLLrmE9evXR6aO3n///ZSUlNDR0cFJJ53Exz/+cUpLSw96ja1bt/LQQw/xwx/+kKuu\nuoonnniC6667bkx/DpFksHd/J0+v283/RiWCxVPzueXchRxXWcS88lwqirJJS50082zG1KRLCuPB\nySeffNC1BHfffTdPPvkkADt37mTr1q2HJIW5c+eyfPlyAE488US2b98et3hFJrr6/Z08vX4P/7t2\nN2t27IskglvPXcQly6axYEp+okOcMCZdUhjpE3285ObmRpZffvllXnjhBV577TVycnI466yzhrzW\nIDPzQF9hamoqHR0dcYlVZKKKJIJ1u1mz3UsEi6bmKRF8QJMuKSRCfn4+ra2tQ+5raWmhuLiYnJwc\nNm3axOuvvx7n6EQmj/rWTp5Zv4dfrz2QCBZOyeOWcxdyybHTWThVieCDUlIYA6WlpZx22mkcc8wx\nZGdnM3Xq1Mi+Cy+8kHvvvZejjz6axYsXc+qppyYwUpFgdfb00dzew762bprau9nX1k1zezf72npo\n7uimt8/R5xzOedMt+/q9Ei59ztHvoL/f0e/v6/e3DSy3dvbyTm1zJBHcfM5CLlk2nUVKBGPKnHOj\ntxpHVqxY4QbfZOfdd9/l6KOPTlBE8ZVMP6skXn+/oyHcRV1zB7tbOmls66apLepg394TWW9q76a9\nu2/Y18rPTCM9LYUUgxQzUsybbmkGqSnmb4teNlJSINUMMyMjLYUPzStVIjhCZvamc27FaO10piCS\nxNq7e9nV3EFdcye7mjv85Q5/uZPdLR0HzeEfkJ+ZRnFuBsU56ZTmZbBwSh7FuRmU5GZQnONtj14v\nykknXbN9JgQlBZFJxjlHa1fvQZ/gG8Pe8u6WzqiDfgdN7T0HPTfFYFpBFjOKslk+s4iLj51ORZG3\nPr0wm7K8DIpyMshI0wF+slJSEBnnnHM0tnWzZ1D3zUCffeRrWw/72r39vf1DdwvnZ6VRUZTNjKJs\njp9VxIyi7Mj6jKJspuZnav5+klNSEEmwzp4+drd0Duq68bpvBrZ19fYf8jwzKM7xumhKcjKYU5bD\nCblFkW2Rr/7+4tx08rN0gyYZmZKCSMB6+/rZ3tjOtvrwQQf9geWGcPchz5mSn8mMomyOnl7AuUdP\n8btvsijLy4wc5Auy00lNmdgVOWX8CTQpmNmFwH8AqcD/OOe+PWj/vwNn+6s5wBTnnEqGyoTU3+/Y\n2dTOlr1htuxtZfOeVrbsbaUm1EZ334FP+tnpqVQUe901S2cUMKPwQPdNRVE2UwszyUxLTeBPIsks\nsKRgZqnAPcD5QC2wxsxWOuc2DrRxzv1FVPuvAscHFc94kpeXRzgcTnQYcoScc+xu6WTz3la27m1l\n8x4vCWyrD9PRc2BKZkVRNoun5fORxeUsnprPwin5zCzJpjA7fcLX3JfJK8gzhZOBbc65GgAzexi4\nHNg4TPtPAd8KMB6Rw9bZ08e7u/ezftd+Nu5qYfOeVrbuDdPa1RtpM7Ugk0VT87n2lFksmprHoqn5\nLJyaH6mfLzKRBPlXWwHsjFqvBU4ZqqGZzQbmAi8FGE9gbrvtNmbOnMmNN94IwO23305aWhqrV6+m\nqamJnp4e/umf/onLL788wZHKSFo7e9i4y0sAG+paWL+rhepQG33+TJ6inHQWT83nyhMqWDQ133/k\nUZQzcqllkYlkvHyUuQZ43Dk35OWQZnYDcAPArFmzRn6lp2+DPevGNrppx8JF3x5299VXX82tt94a\nSQqPPvoozz77LDfffDMFBQU0NDRw6qmnctlll6nbYJzY19bNhl0trK/bz/pdLWzctZ/3Gtoi+6cW\nZHLMjEIuPGY6x8wo4JiKQqYXZun3J5NekEmhDpgZtV7pbxvKNcCNw72Qc+4+4D7wylyMVYBj5fjj\nj6e+vp5du3YRCoUoLi5m2rRp/MVf/AWvvvoqKSkp1NXVsXfvXqZNm5bocJNOY7iLtbUtrKtrYX1d\nCxt27aeu+UAV2sribI6ZUcjHT6hgaUUhS2cUMCV/8t5ERWQkQSaFNcBCM5uLlwyuAa4d3MjMjgKK\ngdfG5LuO8Ik+SJ/85Cd5/PHH2bNnD1dffTU///nPCYVCvPnmm6SnpzNnzpwhS2bL2Nrf2cP62hbe\nqW1hbW0za2tbIgnADOaW5XLi7GKu//BsjplRyJIZBer+EYkSWFJwzvWa2U3As3hTUu93zm0wszuA\nKufcSr/pNcDDbqJV5hvk6quv5otf/CINDQ288sorPProo0yZMoX09HRWr17Njh07Eh3ipNPR3ceG\nXV4CWOcngJqoLqBZJTksn1XE9R+ezbLKIpbOKNDFWyKjCHRMwTm3Clg1aNs3B63fHmQM8bJ06VJa\nW1upqKhg+vTpfPrTn+bSSy/l2GOPZcWKFRx11FGJDnFC6+rtY8ueMO/UNkfOALbWhyODwFMLMllW\nWcSVx1ewbGYRyyoKKR7lXrsicqjxMtA8Kaxbd2CAu6ysjNdeG7pHTNcojKylw5sFtHH3fjb4g8Db\n6sORej7FOekcW1nE+UumsqyyiGWVhZP6Ruoi8aSkIAkzcBHYxl372bBrPxt3e4PAtU0HBoHL8zNZ\nOqOAc46awtIZhSyrLKSyOFuzgEQCoqQgceGcozoUZoOfAAbOAAZKN5vB3NJcjptZxLWnzGLJ9AKW\naBaQSNxNmqTgnJv0nx4n2lj8/s4efre1gdWb63l5c4j61i4AMlJTWDwtnwuWTGNpRQFLZxSweFqB\nrgAWGQcmxX9hVlYWjY2NlJaWTtrE4JyjsbGRrKzx+8nZOceWvWFe3lzP6s31VG1vorffkZ+VxpmL\nyvnIwnKWzSxkfnme7sIlMk5NiqRQWVlJbW0toVAo0aEEKisri8rKykSHcZC2rl5+X93onQ1sqmdX\ni3ctxlHT8vnimfM4e/EUTphVpBu3iEwQkyIppKenM3fu3ESHkRScc7zX0MbqzSFe3lzPGzX76O7r\nJzcjldMXlvHVcxdy1uJyphdmJzpUETkCkyIpSPC21bfys9ffZ/XmenY0tgOwYEoe1394NmcvnsKK\nOSW6b6+MXz2dsL8O0nMgpxTSAryGpb8PwnuhpQ721/pf66B1t//9S7wYBh7ZUevZRZCS2HtpKCnI\niGpCYe5+cStPvbOLjNQUTltQxp+dPpezFk9hZklOosMTOaCvB5p2wL5qaNwGjdX+cjW01AJREzUy\nCyG3FHLKILfMOyDnlkWtlx28P90/83UO2vf5B/vaQw/8LXXQugv6ew+OLT0X8qdBbxe0N0DvcCVv\nDLKLByWOkgPJY8F5MO2YIN69CCUFGdL2hjbufmkrv/xjHZlpqXzpzPnccOY8SnSVsBwO57yDct2b\nsHe99yk4I9c7SGYM8Ri8PTXDm688oL8PWnZ6rxl90N9X7SWE6ELLWYVQugBmfQhK50PRbOjtgLZG\n78Dc1uB9bX4f6t6C9kbo7xn650jP9T7Ft+/zXiNaagYUzICCSpj9YSisgIIKKKz0v1ZAVtHBP0d3\nu/f9Bh4dTQevDzyad8Kut704+7q9GJQUJJ527mvn+y9t5Ym36khLMb5w+ly+9JH5lOVlJjo0ORJd\nrV4p+V1ve5+eC2ZA2UIoWwQl8yBtjH+vbY1eAqirgtoqb7mz2duXmgGu/9BP0SNJSTuQLFLTvS6Y\nvqh7WqfnQuk8mH4cLP2Yd/AvXQAl871P2IczG9E56GzxDsYDCSPytdH7ObKLvYN95IBf6Z1NpBxm\n12lGjvcomjl624HYutvi0rWkpCAA1DV38J8vbeOxqp2kpBh/+qHZ/PlH5jNF5SMmjo4m2P3OwY/G\nbQf2ZxZA1/4D65bifXouW+QnCj9ZlC3yuitGO6D2dHoJJ5IAqqBp+4HXLj8allwGFSugcgWUH+Ud\n1Hq7oafNO8gNfoy0vafTS2qlC7yDf8l8r0tmrKahm3mfxLOLvNcfT8wgMy8u30pJIcntbungB6ur\neXjN+xjGtafM4itnLWBaoZLBEenv9/qLe9q9R3f7geXeLm+gMSPX+wfPyPe+pmUd/oEtHPIP/G8f\nSADNUZV4C2d6n56XXe19nX6cdwDtCnuJomErNGzxHo3b4L1XDu7nzio6kCDKFnhfCyogtOnAGcCe\ndQe6W/JnQOWJcOLnoOJEmHH88AextAzvkV18eD+zxIVNtKtkV6xY4aqqqhIdxoRXv7+TH7xczYN/\neB/nHFetmMmNZy9gRpGmkuKc96l7YOBwYFCxdS90h/2DfIf/CbbjwEF/YPlwWaqfJPzHwHJmvt+3\n7m+zVO+gvOttbzBzQMm8Awf+6cfBtOO8QdLDMdBX37DNTxRbDySO8N6D26bnQsUJ3sG/4kTvLKBg\nxuH/3BJXZvamc27FaO10ppBkQq1d3PtKNT97fQe9/Y5PnljJjWcvSK6ZRN1t3sG+ZeegA3+dd/Df\nX3fowT0lDfKmeQfn9BzvkTfFm5WSnut9zcg5sC/d7zOO3p+W6b1uV9hLLt1tXp9/d/jAtq5Wv8sk\n7PVnd7ce2NfX431in3N6VBJY5g2oflApqVA8x3ssPO/gfZ0tXrJo2el1MQ10A8mkpKSQJJraurn3\nlWp+/Np2unv7+dgJlXz1nAXMLs1NdGjD6273DkTN73uP1j3Q1+V9qu3r8bou+nuhrzdq2f8aWe7z\n9vX1eN0j+3cdGPiMMMib6s0SmXI0LLzg0BkkeVMSfyDs7z/8Ac2xkFXodQ1Vnhj/7y1xp6QwyXX2\n9PHj32/nP1dvo62rlyuWV/DVcxcyt2wcJIPO/VEH/Z1en3j0envDoCeYN4MlNd07QKek+8tp3uOQ\nZX89PdsbZE3L9KYMFlR4fe4DB/786cFezDRWEpEQJOkoKUxS/f2OX63dxZ3PbKauuYOzF5fzjYuP\nZtHU/AQE0wfbfwNbnvMO/M07vIP+4E/saVnewbrIHyQtnOnNjima6S3nT0v8p3WRSU5JYRJ6vaaR\nf1n1LmtrW1gyvYA7P7GM0xaUxT+QvRth7cOw9jFvYDQt2+uzLpoJM0/xD/qzDjxyy8dueqGIHBEl\nhUlkW32Ybz+9iRfe3cuMwizuuuo4rlheQUpKHA+0rXth/ePwzsOwZ63XfbPgPPjoP8Piiw6UCxCR\ncUlJYRJoCHfxvRe28NAfdpKTnsrXL1zM50+bS1Z6nLpautth8yp45yGoXu2VGphxPFx0Jxzzca92\njIhMCEoKE1hHdx8/+m0N975SQ2dPH9edMoubz11IaTxKUvT3w47femcEG1d6UycLZ8Lpt8Kya6B8\nUfAxiMiYU1KYgPr6Hb94q5bvPreFPfs7uWRJMV8/vZTZWe2w+1VoC0U9GiBc7y13tXrTC3NKvIqL\nA9UYD1ku8pYzCw+d8VK/6cA4wf5a76rcpZd7iWD2aZohIzLBKSlMFI3VsO4xdtdup2bHduZ0NfJE\nepip+a2k1bRCzRDPScuC3Cle903+dO/Cp84W6Njn1ahp3+etM8xV7ZbilTsYSBY97V6lS0uFBefC\nBXfA4os1TiAyiSgpjHfd7fCb7+J+fzeur4d0l8/0lEIKpsygdOoxWG455JV7M3cijzLva0be6LN5\n+vv8ypD7vGTR0TT8clomXPhtb5wgb0p8fn4RiSslhfHKOXj3V/Ds30LLTp5LOYPvuE9z7Xmn8JkP\nzSYzbYwGkVNS/Zt4lIzN64nIhKakMB41bIWnvw7VL9FRfBQ32j+wPvUYfnLDyRw1rSDR0YnIJKak\nMJ50heHVf4XX7oH0bKpP/HuuWHM0Jfk5PP75U5hVmkRF60QkIQKdKmJmF5rZZjPbZma3DdPmKjPb\naGYbzOzBIOMZt5yD9b+Ae06G330Pjv0kL5y3igtfX0JFST6PfflDSggiEheBnSmYWSpwD3A+UAus\nMbOVzrmNUW0WAt8ATnPONZlZ8o1e1m+Cp/8a3nsVpi2DTzzAQ3um87e/WMeJs4r50WdPojA7PdFR\nikiSCLL76GRgm3OuBsDMHgYuBzZGtfkicI9zrgnAOVcfYDzjS+d+eOU78Ma93o1ULv433Imf479+\ns507n1nH2YvL+cGnTyQ7QwXgRCR+gkwKFcDOqPVa4JRBbRYBmNnvgFTgdufcMwHGlHjOwbrH4Lm/\n9+5odcJn4Nxv4XJK+b9Pb+K+V2u4fPkM/u2Tx5GeqgvBRCS+Ej3QnAYsBM4CKoFXzexY59xBNZXN\n7AbgBoBZs2bFO8axs3cDrPpr2PE7rzbQNQ9C5Yn09vXzjcfX8tibtVz/odl869Kl8S1iJyLiCzIp\n1AEzo9Yr/W3RaoE3nHM9wHtmtgUvSayJbuScuw+4D7x7NAcWcVB6u2D1v8Dvv++Vmbj0P+D4z0BK\nKp09fdz80B95buNebj1vIbecuxBT+WgRSZAgk8IaYKGZzcVLBtcA1w5q80vgU8ADZlaG1500VMGG\niWvX2/DLP4f6jXD8dXD+P0YuFGvt7OGGn7zJazWN3H7pEj572twEBysiyS6wpOCc6zWzm4Bn8cYL\n7nfObTCzO4Aq59xKf98FZrYR6AP+2jnXGFRMcdXXA7+5C169E3LK4NrHYNEFkd2N4S4++8AaNu7e\nz/euXs4Vx1ckMFgREY85N7F6Y1asWOGqqqoSHcbI6t+FJ78Mu9+GY6+Ci75zUBmJuuYOPvOjN6hr\n6uC/rjuBc46amsBgRSQZmNmbzrkVo7VL9EDz5NLfB6/9J7z0T5CZD1f9BJZcflCTbfVhPvOjNwh3\n9vLTL5zCyXNVc0hExg8lhbHSWO2NHex8A46+FC75d696aZS1tc189oE1pJjx8JdOZemMwgQFKyIy\nNCWFD6q/H9b8Dzz/TUjLgI/9EI795CElq9fVtvCp+16nODeDn33hFOaU5SYoYBGR4SkpfBDN78NT\nN3olKhacD5fdDQUzDmnmnOMff72RnMw0Hv/yh5lWmJWAYEVERqekcCScgz/+FJ75W8DBZd/3rjsY\n5vqC32xt4A/b93HH5UuVEERkXFNSOFz7d8Ovboatz8GcM+Dye6B49rDNnXN897nNVBRlc/VJM4dt\nJyIyHigpxMo5WPc4rPor7wrli+6Ek7446o3qX3i3nndqW/jOx48du7uliYgEREkhVn/8Gay8CSpP\nhivvhdL5oz6lv987S5hTmsPHTqiMQ5AiIh+MkkKs3nwApiyFzz/j3dc4BqvW72bTnla+d/VyVTwV\nkQlBR6pYNGyFujdh+adiTgi9ff3c9fwWFk7J49LjDp2RJCIyHikpxGLto2ApcMwnYn7KU2/voibU\nxl+ev4hUlcEWkQlCSWE0zsHaR2DuR6BgekxP6enr53svbmHpjAI+unRawAGKiIwdJYXR7HwDmnfA\nsqtjfspjVbXs3NfB1y5YpJvliMiEoqQwmncehvQcr55RDDp7+vj+S1s5flYRZy+eEnBwIiJjS0lh\nJL1dsOFJOOpPIDMvpqc8/If32d3SyV9dsFh3UBORCUdJYSRbn4PO5pi7jjq6+/jP1dWcOq+ED88v\nDTg4EZGxp6QwkncehtwpMO+smJr/5LXtNIS7+JrOEkRkglJSGE77PtjyrFcGO3X0a/xaO3u495Vq\nPrKonJPm6MY5IjIxKSkMZ+Mvob8Hll0VU/MHfredpvYevnbBooADExEJTkxJwcx+YWaXmFnyJJF3\nHoHyo2D6caM2bW7v5oev1nDBkqksqyyKQ3AiIsGI9SD/A+BaYKuZfdvMFgcYU+Ltew92vu6dJcQw\nNvDD39QQ7u7lL3WWICITXExJwTn3gnPu08AJwHbgBTP7vZl9zszSgwwwIdY95n09dvSuo4ZwFw/8\nbjt/smwGR00rCDgwEZFgxdwdZGalwGeBPwP+CPwHXpJ4PpDIEsU5b9bRnDOgaPSb4tz7cjWdPX3c\net7COAQnIhKsmEpnm9mTwGLgp8Clzrnd/q5HzKwqqOASou4t2FcNp986atM9LZ389PUdfOyESuaX\nx3Zxm4jIeBbr/RTuds6tHmqHc27FGMaTeGsfhtRMWHL5qE3vWb2Nvn7HLefqLEFEJodYu4+WmFlk\nWo2ZFZvZVwKKKXH6emD9E7D4IsgqHLHpzn3tPLzmfa4+aSYzS3LiFKCISLBiTQpfdM41D6w455qA\nLwYTUgJtexHaG+G4a0Zt+v2XtmJm3HTOgjgEJiISH7EmhVSLqttgZqlARjAhJdDahyG7BBacN2Kz\nmlCYJ96q47pTZjO9MDtOwYmIBC/WMYVn8AaV/9tf/5K/bfLobIHNT8Pxn4HUkWfZ/seLW8lITeHP\nz5ofp+BEROIj1jOFvwFWA3/uP14Evj7ak8zsQjPbbGbbzOy2IfZ/1sxCZva2//izwwl+TG1cCb2d\no3Ydbd7Tysp3dvHZ0+ZQnp8Zp+BEROIjpjMF51w/8F/+IyZ+F9M9wPlALbDGzFY65zYOavqIc+6m\nWF83MGsfgZL5UHHiiM3+/fkt5GWk8aUz58UpMBGR+Im19tFCM3vczDaaWc3AY5SnnQxsc87VOOe6\ngYeB0ed5JkLzTtj+G+++CSOUtVhX28IzG/bwhTPmUpQz+YZURERi7T56AO8soRc4G/gJ8LNRnlMB\n7Ixar/W3DfZxM1vrJ50hLyE2sxvMrMrMqkKhUIwhH4aBshajVES96/nNFOWk8/nT5459DCIi40Cs\nSSHbOfciYM65Hc6524FLxuD7/wqY45xbhlcu48dDNXLO3eecW+GcW1FeXj4G3/agF/e6jmaeAiXD\nH+zfer+J1ZtDfOnM+RRkTb5yTyIiEHtS6PLLZm81s5vM7EpgtLoOdUD0J/9Kf1uEc67ROdflr/4P\nMHKHfhD2rIXQplFvubl6Uz0pBn/6odlxCkxEJP5iTQq3ADnAzXgH7uuA60d5zhpgoZnNNbMM4Bpg\nZXQDM5setXoZ8G6M8Yyddx6BlHRYeuWIzWpCbcwqySE3M9ZZvCIiE8+oRzh/FtHVzrm/AsLA52J5\nYedcr5ndBDwLpAL3O+c2mNkdQJVzbiVws5ldhjdWsQ+vCmv89PXC+sdh0UchZ+RbaFaHwip6JyKT\n3qhJwTnXZ2anH8mLO+dWAasGbftm1PI3gG8cyWuPifdehvDeUbuO+vod7zW0ccbCsvjEJSKSILH2\nhfzRzFYCjwFtAxudc78IJKp4WfuoV/hu0UdHbLaruYOu3n6dKYjIpBdrUsgCGoFzorY5YOImha4w\nvPsrbxpq2shXJleHwgDMU1IQkUku1iuaYxpHmFA2/Rp62mHZ6BVRq0PeydH88tygoxIRSahY77z2\nAN6ZwUGcc58f84jiZe0jUDTLuz5hFDWhMIXZ6ZTk6ipmEZncYu0++nXUchZwJbBr7MOJk9Y9UPMy\nnPE1SBl9Vq438ygXG6EEhojIZBBr99ET0etm9hDw20Aiiod1j4HrH3XW0YCaUBtnLhrjK6lFRMah\nWC9eG2whMGUsA4mrtY/AjBOgbPR7K7d29lDf2qWZRyKSFGIdU2jl4DGFPXj3WJh49m6EPevgojtj\nal7jDzLP0yCziCSBWLuP8oMOJG7WPgKWCks/FlPzgemoOlMQkWQQ6/0UrjSzwqj1IjO7IriwAtLf\n740nLDgP8mIbI6gJtZGaYswqyQk4OBGRxIt1TOFbzrmWgRXnXDPwrWBCCtD238D+OjgutgFm8M4U\nZpfkkJF2pMMvIiITR6xHuqHaTbxyoWsfhYx8WHxxzE+pCbVpPEFEkkasSaHKzO4ys/n+4y7gzSAD\nG3Pd7bDxKVhyOaRnx/SUvn7He41tGk8QkaQRa1L4KtANPIJ3r+VO4MagggrE5lXQ3XpYXUd1TR10\n9/brTEFEkkass4/agNsCjiVYKWkw7yyYHXsVcM08EpFkE+vso+fNrChqvdjMng0urAAsvQL+9KmY\nyloMUHUIP7amAAAOsUlEQVRUEUk2sR4hy/wZRwA455qYyFc0x6g61EZxjgrhiUjyiDUp9JvZrIEV\nM5vDEFVTJ5uaUFhnCSKSVGKdVvp3wG/N7BXAgDOAGwKLapyoDrVxzlEqhCciySPWgeZnzGwFXiL4\nI/BLoCPIwBKtpaOHhnCXzhREJKnEWhDvz4BbgErgbeBU4DUOvj3npFKjmUcikoRiHVO4BTgJ2OGc\nOxs4Hmge+SkTm6qjikgyijUpdDrnOgHMLNM5twlYHFxYiVcdCpOmQngikmRiHWiu9a9T+CXwvJk1\nATuCCyvxakJtzCrNIT1VhfBEJHnEOtB8pb94u5mtBgqBZwKLahzw7sus8QQRSS6HXenUOfdKEIGM\nJ719/exobOecoyf99XkiIgdR38gQaps66O7r15mCiCQdJYUh1DQMTEfVzCMRSS6BJgUzu9DMNpvZ\nNjMbtsqqmX3czJx/gVzCVdf701HLdKYgIsklsKRgZqnAPcBFwBLgU2a2ZIh2+XjXQbwRVCyHq6Yh\nTEluBsUqhCciSSbIM4WTgW3OuRrnXDfezXkuH6LdPwLfwbtxz7hQXd+mriMRSUpBJoUKYGfUeq2/\nLcLMTgBmOuf+N8A4DltNQ1hdRyKSlBI20GxmKcBdwNdiaHuDmVWZWVUoFAo0rpb2HhrC3cyfojMF\nEUk+QSaFOmBm1Hqlv21APnAM8LKZbccrsrdyqMFm59x9zrkVzrkV5eXBlrKu9mce6UxBRJJRkElh\nDbDQzOaaWQZwDbByYKdzrsU5V+acm+OcmwO8DlzmnKsKMKZRVdf701GnKCmISPIJLCk453qBm4Bn\ngXeBR51zG8zsDjO7LKjv+0HVNLSRnmrMLM5OdCgiInF32GUuDodzbhWwatC2bw7T9qwgY4lVdX2Y\n2aW5pKkQnogkIR35BqlpaGNemQaZRSQ5KSlE8QrhtWk8QUSSlpJClJ1NHfT0OZ0piEjSUlKIoplH\nIpLslBSiRKqj6hoFEUlSSgpRquvbKMvLoDAnPdGhiIgkhJJCFNU8EpFkp6QQpTrUpppHIpLUlBR8\nTW3d7Gvr1pmCiCQ1JQVfZJBZZwoiksSUFHzVId2CU0REScFXHQqTkZpCpQrhiUgSU1Lw1YTamF2a\no0J4IpLUdAT0VYfCzC9X15GIJDclBaCnr5/3G9uZV65BZhFJbkoKwPv72untdzpTEJGkp6SAN54A\n6ExBRJKekgLeeALAPJ0piEiSU1IAakJhyvIyKcxWITwRSW5KCvg1j9R1JCKipADemYK6jkRElBTY\n19ZNU3uPzhRERFBSoMYfZNZ0VBERJYWomUc6UxARSfqkUBNq8wvh5SQ6FBGRhEv6pFAdCjOnLIfU\nFEt0KCIiCZf0SaEm1KbxBBERX1Inhe7efnbsUyE8EZEBgSYFM7vQzDab2TYzu22I/V82s3Vm9raZ\n/dbMlgQZz2Dv72unT4XwREQiAksKZpYK3ANcBCwBPjXEQf9B59yxzrnlwJ3AXUHFMxTVPBIROViQ\nZwonA9ucczXOuW7gYeDy6AbOuf1Rq7mACzCeQ6g6qojIwdICfO0KYGfUei1wyuBGZnYj8JdABnBO\ngPEcojoUpjw/k4IsFcITEYFxMNDsnLvHOTcf+Bvg/wzVxsxuMLMqM6sKhUJj9r1rQmGVtxARiRJk\nUqgDZkatV/rbhvMwcMVQO5xz9znnVjjnVpSXl49JcM45qkNtGk8QEYkSZFJYAyw0s7lmlgFcA6yM\nbmBmC6NWLwG2BhjPQfa1ddPS0aOZRyIiUQIbU3DO9ZrZTcCzQCpwv3Nug5ndAVQ551YCN5nZeUAP\n0ARcH1Q8g1VrkFlE5BBBDjTjnFsFrBq07ZtRy7cE+f1HMlAddYHOFEREIhI+0Jwo1aEwGWkpzCjK\nTnQoIiLjRtImhZpQG/PKclUIT0QkStImhepQWOMJIiKDJGVS6OrtY2dTh2YeiYgMkpRJ4f1GrxCe\nzhRERA6WlEmhWvdlFhEZUpImBe8ahbllOlMQEYmWpEkhzNSCTPJVCE9E5CBJmRS86ajqOhIRGSzp\nkoJXCC/M/CnqOhIRGSzpkkJDuJvWzl6dKYiIDCHpkkJk5tEUJQURkcGSLilEbsGpmUciIodIuqRQ\nHQqTmZZChQrhiYgcIumSQk0ozNyyXFJUCE9E5BBJlxSqQ20aTxARGUZSJYXOnj5qm9qZr/EEEZEh\nJVVS2NHYTr/TzCMRkeEkVVIYuAWnrlEQERlaUiWFgWsUVDJbRGRoSZUUakJtTCvIIjczLdGhiIiM\nS0mVFFTzSERkZEmTFJxzqo4qIjKKpEkKodYuWrt6ma/xBBGRYSVNUhi429o83YJTRGRYSZQUVB1V\nRGQ0SZMUpuRncv6SqUwvyEp0KCIi41bSzM28YOk0Llg6LdFhiIiMa0lzpiAiIqMLNCmY2YVmttnM\ntpnZbUPs/0sz22hma83sRTObHWQ8IiIyssCSgpmlAvcAFwFLgE+Z2ZJBzf4IrHDOLQMeB+4MKh4R\nERldkGcKJwPbnHM1zrlu4GHg8ugGzrnVzrl2f/V1oDLAeEREZBRBJoUKYGfUeq2/bThfAJ4eaoeZ\n3WBmVWZWFQqFxjBEERGJNi4Gms3sOmAF8K9D7XfO3eecW+GcW1FeXh7f4EREkkiQU1LrgJlR65X+\ntoOY2XnA3wEfcc51BRiPiIiMIsgzhTXAQjOba2YZwDXAyugGZnY88N/AZc65+gBjERGRGJhzLrgX\nN7sY+B6QCtzvnPtnM7sDqHLOrTSzF4Bjgd3+U953zl02ymuGgB1HGFIZ0HCEz40HxffBKL4PbrzH\nqPiO3Gzn3Kj974EmhfHGzKqccysSHcdwFN8Ho/g+uPEeo+IL3rgYaBYRkfFBSUFERCKSLSncl+gA\nRqH4PhjF98GN9xgVX8CSakxBRERGlmxnCiIiMoJJmRRiqM6aaWaP+PvfMLM5cYxtppmt9qvDbjCz\nW4Zoc5aZtZjZ2/7jm/GKz//+281snf+9q4bYb2Z2t//+rTWzE+IY2+Ko9+VtM9tvZrcOahP398/M\n7jezejNbH7WtxMyeN7Ot/tfiYZ57vd9mq5ldH6fY/tXMNvm/vyfNrGiY5474txBwjLebWV3U7/Hi\nYZ474v97gPE9EhXbdjN7e5jnxuU9HDPOuUn1wLsmohqYB2QA7wBLBrX5CnCvv3wN8Egc45sOnOAv\n5wNbhojvLODXCXwPtwNlI+y/GK9OlQGnAm8k8He9B2/+dULfP+BM4ARgfdS2O4Hb/OXbgO8M8bwS\noMb/WuwvF8chtguANH/5O0PFFsvfQsAx3g78VQx/AyP+vwcV36D93wW+mcj3cKwek/FMYdTqrP76\nj/3lx4FzzcziEZxzbrdz7i1/uRV4l5ELBY5HlwM/cZ7XgSIzm56AOM4Fqp1zR3ox45hxzr0K7Bu0\nOfrv7MfAFUM89aPA8865fc65JuB54MKgY3POPeec6/VXE16heJj3Lxax/L9/YCPF5x87rgIeGuvv\nmwiTMSnEUp010sb/x2gBSuMSXRS/2+p44I0hdn/IzN4xs6fNbGlcAwMHPGdmb5rZDUPsP9wKuEG5\nhuH/ERP5/g2Y6pwbuFp/DzB1iDbj4b38PMNUKGb0v4Wg3eR3cd0/TPfbeHj/zgD2Oue2DrM/0e/h\nYZmMSWFCMLM84AngVufc/kG738LrEjkO+D7wyziHd7pz7gS8GyTdaGZnxvn7j8qvp3UZ8NgQuxP9\n/h3Cef0I426qn5n9HdAL/HyYJon8W/gvYD6wHK8Uznfj+L0Px6cY+Sxh3P8/RZuMSSGW6qyRNmaW\nBhQCjXGJzvue6XgJ4efOuV8M3u+c2++cC/vLq4B0MyuLV3zOuTr/az3wJN4perSYKuAG7CLgLefc\n3sE7Ev3+Rdk70K3mfx2q6GPC3ksz+yzwJ8Cn/aR1iBj+FgLjnNvrnOtzzvUDPxzmeyf0b9E/fnwM\neGS4Nol8D4/EZEwKo1Zn9dcHZnl8AnhpuH+Kseb3P/4IeNc5d9cwbaYNjHGY2cl4v6e4JC0zyzWz\n/IFlvAHJ9YOarQT+1J+FdCrQEtVNEi/DfjpL5Ps3SPTf2fXAU0O0eRa4wMyK/e6RC/xtgTKzC4Gv\n41Uobh+mTSx/C0HGGD1OdeUw3zuW//cgnQdscs7VDrUz0e/hEUn0SHcQD7zZMVvwZiX8nb/tDrx/\nAIAsvG6HbcAfgHlxjO10vG6EtcDb/uNi4MvAl/02NwEb8GZSvA58OI7xzfO/7zt+DAPvX3R8hnf/\n7WpgHd59tuP5+83FO8gXRm1L6PuHl6B2Az14/dpfwBunehHYCrwAlPhtVwD/E/Xcz/t/i9uAz8Up\ntm14ffEDf4MDs/FmAKtG+luI4/v3U//vay3egX764Bj99UP+3+MRn7/9/w383UW1Tch7OFYPXdEs\nIiIRk7H7SEREjpCSgoiIRCgpiIhIhJKCiIhEKCmIiEiEkoJIHPkVXH+d6DhEhqOkICIiEUoKIkMw\ns+vM7A9+Dfz/NrNUMwub2b+bdx+MF82s3G+73Mxej7o3QbG/fYGZveAX5nvLzOb7L59nZo/79zP4\nebwq9IrEQklBZBAzOxq4GjjNObcc6AM+jXcldZVzbinwCvAt/yk/Af7GObcM7wrcge0/B+5xXmG+\nD+NdEQteZdxbgSV4V7yeFvgPJRKjtEQHIDIOnQucCKzxP8Rn4xWz6+dA4bOfAb8ws0KgyDn3ir/9\nx8Bjfr2bCufckwDOuU4A//X+4PxaOf7duuYAvw3+xxIZnZKCyKEM+LFz7hsHbTT7+0HtjrRGTFfU\nch/6P5RxRN1HIod6EfiEmU2ByL2WZ+P9v3zCb3Mt8FvnXAvQZGZn+Ns/A7zivLvq1ZrZFf5rZJpZ\nTlx/CpEjoE8oIoM45zaa2f/Bu1tWCl5lzBuBNuBkf1893rgDeGWx7/UP+jXA5/ztnwH+28zu8F/j\nk3H8MUSOiKqkisTIzMLOubxExyESJHUfiYhIhM4UREQkQmcKIiISoaQgIiIRSgoiIhKhpCAiIhFK\nCiIiEqGkICIiEf8fCnWPO1wdV+UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb004fedf28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['accuracy'], label='train');\n",
    "plt.plot(model.history.history['val_accuracy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEKCAYAAAAW8vJGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXHWZ7/HP0/u+pJfsGxBCwhYgBJRFFhcQFRQE3O6o\nM+IMoIjOdXB0cBlm1Ll3cMa5OIIOd3RkFUUjEwUXlouKZgFDSEISQpbuTtLppNd09VrP/eOc7lR3\nutOVpE9Vp+v7fr3Oq6pOnVP1VHX1ec7vd36LuTsiIiIDstIdgIiITCxKDCIiMoQSg4iIDKHEICIi\nQygxiIjIEEoMIiIyhBKDiIgMocQgIiJDKDGIiMgQOekO4GhUV1f7vHnz0h2GiMhxZfXq1U3uXjPW\ndsdlYpg3bx6rVq1KdxgiIscVM9uezHaqShIRkSGUGEREZAglBhERGeK4vMYwkt7eXurq6ujq6kp3\nKJEqKChg1qxZ5ObmpjsUEZmkJk1iqKuro7S0lHnz5mFm6Q4nEu7Ovn37qKurY/78+ekOR0QmqUlT\nldTV1UVVVdWkTQoAZkZVVdWkLxWJSHpNmsQATOqkMCATPqOIpNekqUoSERlP7o47eHg/7uA4PX1x\nugeW3n56+uN09w6s6094PuF+b/C4t98pyc+hrDCHsoJcygpzKS/MDe/nUJKfQ052+s/XlRjGSUtL\nCw8++CA333zzEe339re/nQcffJCKioqIIhPJPO7O7rYuNu3pYNPudl7d087mPe283nSAvnhwwI+7\n4wDhAT/uYTIA3NMXe0l+DmUFOZQlJIyBJFJWmMu7zpzOSbWlkcagxDBOWlpa+Na3vnVIYujr6yMn\nZ/SvecWKFVGHJjKpNXV0s2lPe5gAOti8J0gE7V19g9vUlOazcGop15w1k/ycLMwMMzAGbsEMssww\nggcD6wwjy8L7ZuTnZJGXk0V+Thb5OdkJj7PJz80iLzuL/Nzg8cHtgm1ys7I40NNHW1cfrZ29tHX1\n0hbrpa2rL7ztpS3WR1tXL62x4Ln6li42xNpp6+qlvauPM2eVH/+JwcyuAP4VyAa+6+5fG/b8XOB+\noAbYD3zQ3euijmu83XHHHbz22mssWbKE3NxcCgoKqKysZOPGjWzatIlrrrmGnTt30tXVxW233cZN\nN90EHBzeo6OjgyuvvJILL7yQ3/3ud8ycOZOf/vSnFBYWpvmTiaSPu9PdFx88aO4/0MvmxiAJbNrT\nwaY97ew70DO4fXlhLgunlnL1khksnFrKgqmlnDy1lCnFeWn8FEOVFuRSWpDLzIoj/9/uj6emKBNp\nYjCzbOAe4C1AHbDSzJa7+/qEzf438H13/56ZXQZ8FfjQsbzvl3/2Cusb2o7lJQ6xeEYZX3znqaM+\n/7WvfY1169bx0ksv8cwzz3DVVVexbt26wWal999/P1OmTCEWi3Huuedy7bXXUlVVNeQ1Nm/ezEMP\nPcR3vvMdrr/+en70ox/xwQ9+cFw/h0g69MedPW1d1LfEBs+U2xPOktu7+gbPltu7hp5B9/YfejAs\nzstmwdRSLl9Uy8lTS1k4rZSFU0upKc2f1A00srNS89miLjEsA7a4+1YAM3sYuBpITAyLgU+H958G\nfhJxTCmxbNmyIX0NvvnNb/L4448DsHPnTjZv3nxIYpg/fz5LliwB4JxzzmHbtm0pi1fkWLV39bJz\nf4wd+zvZub+THeGyc38ndc0xevrjI+5XmJtNWWEOpQW5lBXkUFGUx5yqYsoKwnVhHXtp+NwJ1cXM\nrCgkK0UHyUwUdWKYCexMeFwHnDdsmz8B7yGobno3UGpmVe6+72jf9HBn9qlSXFw8eP+ZZ57hV7/6\nFb///e8pKirikksuGbEvQn5+/uD97OxsYrFYSmKVyaU/7rTFemmJ9dLc2UNrZy8tsR6aD/TS0d1H\nTraRl51FbnZY752dRe4I6/JyLHwuWJeXnYU71LUkHvgPJoL9CVU6EFTrzJlSxKLpZbz11GnMmVLE\nzMpCKotyBy+mlhbkkDsBWuHIUBPh4vNfA//HzD4MPAfUA/3DNzKzm4CbAObMmZPK+JJSWlpKe3v7\niM+1trZSWVlJUVERGzdu5IUXXkhxdHK8cndaY700tnfT2NZNU0c3zZ09tHT20tLZQ0us95D7bV29\nKWlVk5NlzKwsZM6UIq44LTjwDyyzK4soL9KwLcerqBNDPTA74fGscN0gd28gKDFgZiXAte7eMvyF\n3P0+4D6ApUuXprEx2ciqqqq44IILOO200ygsLGTq1KmDz11xxRV8+9vfZtGiRSxcuJDzzz8/jZHK\nRBCPO/sO9NDY3kVjezd727oH7zcm3m/vpqdv5CqYgWqXyqJcyovymFtVPHi/ojCXyuJcKgrzKC/K\npTJcV1KQQ3/c6e2P09MXp7c/vN8fp7c/Tm+f09PfT09fsH5g6Q63dXdmVhQye0oR08sLJkSbexl/\n5hGeWphZDrAJuJwgIawE3u/uryRsUw3sd/e4mf0D0O/udx7udZcuXerDJ+rZsGEDixYtGu+PMCFl\n0mc93vTHnX0HDh7c97R1s6dt4IAf3O5p66Kpo2fEFiZlBTnUlhVQW5ofLAP3w9ua0nwqi/IoL8xN\n2YVImTzMbLW7Lx1ru0hLDO7eZ2a3Ak8SNFe9391fMbOvAKvcfTlwCfBVM3OCqqRbooxJ5Gj0x519\nHd3hWfyRH/CnFOcNHuBPnlo65MA/tSyf2tICakrzKcjNTsOnExkq8msM7r4CWDFs3Z0J9x8DHos6\nDpGRdPX2szc82O8Nq24ah1Xr7O3oZl9HNyM1Ia8qzqOmNJ+pZQUsnFrK1LICasMD/dSy4MBfU5JP\nXo6qXOT4MREuPouMu3jcaTrQze7WLna1drG7tYuG1hh7Wg/W3Te2ddGW0Dt2QJZBdUk+tWX5TCsv\n4IxZ5UE1TniQn1oWJIJqHfBlklJikONOPO40dXSzKzzo72qNDSaAXa0xdrV2saet65COUXnZWdSG\nB/UFtSW88cSqsEqngJqy/MH7U4rzVH8vGU2JQSaknr44O/Z38nrTAbY1HeD1fcHt9n2d7Gnrom9Y\nvU5eThbTywuYVlbA0rmVTK8oHHw8vbyQ6RUFTCnKU6cokSQoMUja9PXHqWuODR70tzUdYGvTAbbt\nO0B9c2xInX5FUS7zq4s5d14lMwYO+uXB7fTy4Cx/Mg+FIJJKSgxpUlJSQkdHR7rDSAl3Z8f+Ttbs\naGZdfdtgKWDH/s4hZ/6l+TnMqy5myexK3r1kJvNriplXVcz86mIqiibOIGgik50Sg4y7WE8/f6pr\nYc2OZtZsb+HFHc2DI2AW5GYxr6qYhdNKueK0acyrDg7886qKqS7RWb9MQv19sG8z7F4H+1+DklqY\nckKwlM2ErInXRFmJYZzccccdzJ49m1tuCbphfOlLXyInJ4enn36a5uZment7ueuuu7j66qvTHOn4\ncnd27o8FSSBcNuxqH2zLf0J1MZcsrOXsuRWcPaeSk6eW6sKuHLn+Pti7MVjcw8kTssGywiW8n5Ud\nTpyQ+Dhhm+wcKK6BkqmQHcGQHbEW2LMuSAJ7XobdL0PjRujvHnn77DyonHcwUSQu5bODeNMg0p7P\nURmz5/PP7wj+IONp2ulw5ddGffrFF1/kU5/6FM8++ywAixcv5sknn6S8vJyysjKampo4//zz2bx5\nM2Z2TFVJ6ez53NXbz9q6VtbsaGb19mZe3NFCU0fwoy/Ky+bMWRWDSeCsOZUTahz8jOUOnftg32vQ\nsRtKp0PFHCiuhawJ2Nw23g/7tkDDi1C/Jrjd/TL0jeegkhYkiNJpwfcx2m1x9chn9PE4NL+ekATW\nBTG2JowZWlQVHDemnnbwtuok6GyC/VuHLa8Ht72dB/fPyoGKuYcmjJnnQHHVoTEl86knQs/nTHLW\nWWfR2NhIQ0MDe/fupbKykmnTpnH77bfz3HPPkZWVRX19PXv27GHatGnpDjdp7s5rezv49YZGfrOx\nkdXbmwevC8ytKuLiBdWcNbeSs+dUsHBqqcbOSRSPQ3vD0ANAR2NwwBg4+JTNCG5LpkFe0bG9X6wZ\n9m0Nqiv2vRYcXPe/Fqzrbj10+5yC4Ky0Ys7BpXJucDCqmBMcOKOu2nMPvpeGFw8uu/4EPeFJU24R\nTD8Tln4EZpwFU0+F7HzwOHh/cBvvT3jswx7HE7bx4My9oxHad0P7roO3DS/Cgb0EMzwnsOygdDHw\ntyooD77XPa8cjNGyoGoBzF4GSz96MAmUThv5+yufFSzzLz70u+jYM0LS2Ao7XoCecJDOG34Ai945\nrn+G4SZnYjjMmX2U3vve9/LYY4+xe/dubrjhBh544AH27t3L6tWryc3NZd68eSMOtz3RdPX288LW\nfTy9sZHfvNrIzv3Bmdop00r584vms3TuFM6aU0F1Sf4Yr5QB+vuCs8T9W4MzyIEzv4GzwMQqhOy8\n4Cy9swn6RvgdFJQf/uy1dBrkl0LLjvDA/9rBJLD/taBUMMiCg37VCXDGe2HKiVB1YvAa7XugZXu4\n7AiWXS8N2x/IKRyaNCrmBAe07LzgbDYrO1xygsWyR16flRNW6+RAf09wZj2YCF46mLSy84OD6pL3\nB0lgxllQfXLq6uD7e4cljV1Dk8f+rdC5PzhrX/L+sCRwGtQuhtxxmGnRLPxbT4O5bxz63ECpb//W\noNQRscmZGNLkhhtu4GMf+xhNTU08++yzPProo9TW1pKbm8vTTz/N9u3b0x3iqHa3dvGbjUGp4Ldb\nmoj19lOQm8UFJ1bz8YtP5NJTao9qKsJjFmuB3WuDs8hda6GtPjjgDSlez4eiKdG8v3twJt66E1rr\ng4No4plcy3aIJ/SezikMYqo6CRa8dWicZTOCg5w7dLWGB5yGQ89e23fDtueD+/FDe2YPUTo9eK9T\n3hEc+KtOCpJA5TzILRh5n+mjvFZ3O7TsDJPF9qG3dSuh65BBj49eVm5w9n/aew4mgdpF0dT7Jys7\nF8pnBstEYxZUaxVXp+TtlBjG0amnnkp7ezszZ85k+vTpfOADH+Cd73wnp59+OkuXLuWUU05Jd4iD\n+uPOSztbglLBxkbW7wqmQp1ZUch158zislNqecOJVakd1K2jMUwACUtLQjItmxkkhW3Pw9qHh+5b\nUDHyBbwpJwT/TKNVifR2BcmmtS5h2RncDqxPrPcFyCsJXnfa6bD46qHvNVr1QSIzKKwIltrD/Cbi\ncYjth7aE5NHVGpy5V50YvF9e8ej7H6n8Upi6OFhG0tUKbbsg3htUzcT7g8QV7wuqbeJ9CesSnov3\nH3zesoIEMPU0yFGJc6KanBefJ7mj/awHuvv4zcZGnt7YyDOb9rL/QA/ZWcY5cyq59JRaLl9Uy4La\nkuibjLoHB9+BUsBAEujYfXCbKScEdcvTzghup5859GypNwbN20euj23dGdQrD8grhSnzwiRRG7zP\nQBI4sPfQ+EqmhkloVpCIBuqEBx4fLtGITGC6+CxAcPF4zY5mHlm5kyfW7qKzp5+KolwuObmGyxZN\n5U0Lao5+pq14PLgA19MRVEMMX3o6oLstfBxu07EnqBqKNQevYVlQcwqccMnBBDDttKC+/XByC4Oz\n7ZHOuPt6Dq3y2b81aD1yoCk4qy+fFSSd8tlh9UF44C+bqTNZyXhKDJPU3vZuHn+xjkdX1bGlsYOi\nvGzeccZ0rjtnNufMrUyuL0FPJzRtgsYNsHdDcNu87eCBvmfkqUwPkZ0fVFPklwbXAha962ASqF18\n7K1xhsvJg+qTgkVEjtikSgzuPul7zh6u6q+vP86zm/byyMqd/GZjI31x5+w5FXz92tO56owZlOSP\n8ufu64amzUHnocb1QYecxvVBEhhovpeVG7QQqVkY1OcPHOgHlrwSyC8LH5eEt2XB+hz1ZRA5nkya\nxFBQUMC+ffuoqqqatMnB3dm3bx8FBUNbm7zedIAfrtrJj9bUsaetm+qSPD564XyuXzqLk2pLgzr9\nvi440Bo0lWzcMLQUsO+14OIgBE0Oq06E6WfAGTcEFwprFwX18+lsMSIiKTNpEsOsWbOoq6tj794R\nLiZOBu7Q30NBXwuzDqyjd0sz23Y1sm33Xjra2ziTbq4qdmbMiFOR04Nt7oT1nUF1UO+BoRdjAbCg\nmWfNoqBqZyABVJ2kOnaRDDdpEkNubi7z589Pdxjjxz2o2nntadj6DGz/7cGelkDcjenkU5VVQHZx\nMUUl5eQWlAQtcHKLg3r73KKgKmfwfnFQDVSzMKgWGu+6fRGZFCZNYpgU2hpg67OwNUwGHXuC9VNO\npPvU6/m/DXP5j+3V9OSWcvlpc7j+3DmcN3/KpK06E5H0UGJIp662oCSw9ZmgZND0arC+qBpOeBOc\ncCmc8CZe7ijn5gdXs6uli9vfdjIfesNcygpU3y8i0VBiSKV4P+z8Y5AItj4NdauCi745hcHYKGd/\nKGjPX3sqZGXh7vzXC9u564nfUV2SxyMffwPnzK1M84cQkclOiSFVXn8OfvG5YHheywrGhrnw9iAR\nzF52yAXf9q5e7vjRy/z3y7u4dGENd1+/hEoNYS0iKaDEELX9r8Mv/w42/CwY4+bd98LJb4PC0c/8\nX2lo5ZYH1rCzOcbfXHEKH7/4BE1iLyIpo8QQle4OeP5u+N3/CUbUvOwL8IZbDzs8r7vz4B938OWf\nraeyKJeHPnY+y+ZHNGqoiMgolBjGWzwOax+BX30pGKztjBvhzV8Mhlw+jAPdffzt4y/z05cauGhB\nNf9ywxKqNN+BiKSBEsN42rkSfvE3UL86mH7vhh/A7HPH3G3j7jZufmAN25oO8Jm3nMwtl56kqiMR\nSRslhvHQ1hCUENY+EkzR+O574fTrx5xP19354ao67ly+jtKCXH7wF+fxxhNTMxGHiMholBiORW8s\nuIbw/N1BU9SLPgMXfjoYRG4MnT19fOEn6/jxmnreeGIV/3LjEmpLR5lxS0QkhZQYjoY7rP8pPPV3\n0LojGGvorX8fTKeYhM172rn5gTVs2dvBbZcv4JOXL0huGGwRkRRQYjhSu9bCL+4IeixPPQ2ueQLm\nX5T07j9eU8fnH19HUV423//oMi5aUBNhsCIiR+7wleDjwMyuMLNXzWyLmd0xwvNzzOxpM3vRzNaa\n2dujjumovfgDuPfiYKjqd3wDPv7cESWFbz2zhU8/+idOn1XOitsuUlIQkQkp0hKDmWUD9wBvAeqA\nlWa23N3XJ2z2BeBRd/93M1sMrADmRRnXUenugF9+EWafB+9/+LAd1Eayevt+/vmpTVx1xnT+9YYl\n5GRHnpNFRI5K1EenZcAWd9/q7j3Aw8DVw7ZxoCy8Xw40RBzT0fnjfcEkN2+964iTQmusl08+9BIz\nKgr42ntOV1IQkQkt6msMM4GdCY/rgPOGbfMl4Ckz+wRQDLw54piOXFcr/PZfYcHbkuqXkMjd+fzj\nL7O7rYsf/uUbKNWoqCIywU2EU9f3Af/p7rOAtwP/ZWaHxGVmN5nZKjNblfJZ2l74d+hqgUv/9oh3\nfWx1HU+s3cXtb17A2XM0MqqITHxRJ4Z6YHbC41nhukR/DjwK4O6/BwqAQ3p5uft97r7U3ZfW1KTw\nom3nfvj9PXDKO2DGkiPa9fWmA3xx+SucN38Kf3XJSREFKCIyvqJODCuBBWY238zygBuB5cO22QFc\nDmBmiwgSw8SZuPl3/wbd7UdcWujpi/PJh14kNzuLb9ywRP0UROS4EWlicPc+4FbgSWADQeujV8zs\nK2b2rnCzzwAfM7M/AQ8BH3Z3jzKupB1ogj/cC6e9B6aeekS7/vNTr/JyfStfv/YMZlSMPqKqiMhE\nE3kHN3dfQdAENXHdnQn31wMXRB3HUXn+G9AXg0s+d2S7bW7i3ue28v7z5nDFadMiCk5EJBoT4eLz\nxNS2C1Z+Nxg2u3pB0rvt6+jm9kdf4qTaEv7uqsURBigiEg0NiTGa5++GeB+86bNJ7+LufPaxtbR2\n9vK9jyyjMC87wgBFRKKhEsNIWnbC6v+EJR+AKfOT3u37v9/Orzc2cseVp7B4RtnYO4iITEBKDCP5\nf/87uL34fya9y4ZdbfzDig1curCGj1wwL5q4RERSQIlhuP2vB4PlnfNhqJg95uYAXb39fPKhFykr\nyOV/vfdMzNQ0VUSOX7rGMNyz/wRZOcGkO0m667/Xs7mxg+9/dBnVmqdZRI5zKjEkatoMax+Gc/8C\nSpNrZvrUK7v5wQs7+NhF87n4ZA2jLSLHPyWGRM98FXIK4cLbk9p8d2sXn/3RWk6dUcZfv21hxMGJ\niKSGEsOAPeth3Y/hvI9D8SFDNR2iP+7c/shLdPfG+eb7ziI/R01TRWRy0DWGAc/8I+SXwhs/kdTm\n9z73Gr/fuo+vX3s6J9aURByciEjqqMQA0PASbPgZvOEWKJoy5uYv7Wzh7qc2cdXp07l+aXItl0RE\njhdKDABP/yMUVMD5fzXmph3dfXzyoReZWlbAP777dDVNFZFJR4lh50rY/CRccBsUlI+5+Z0/WUdd\ncyf/cuMSyos0G5uITD5KDE/fBUXVsOymMTf9xbpd/PjFej5x2QLOnTd2lZOIyPEoqcRgZqdHHUha\nbPstbH0maJ6aP/YF5Gc37aWiKJdPXKbZ2ERk8kq2xPAtM/ujmd1sZmPXtxwP3OHpf4CSaXDunye1\nS11zjDlTisjJVkFLRCavpI5w7n4R8AGC+ZtXm9mDZvaWSCOL2tZnYPtv4eK/htzkZlirb4kxU7Ox\nicgkl/Spr7tvBr4A/A3wJuCbZrbRzN4TVXCRcYff3AVls+Ds/5HkLk6DEoOIZIBkrzGcYWbfIJi3\n+TLgne6+KLz/jQjji8bmp6B+VTAJT05yg97tO9BDV29c8zeLyKSXbM/nfwO+C/ytu8cGVrp7g5l9\nIZLIojJQWqicB0ven/Ru9c3Bx55ZqcQgIpNbsonhKiDm7v0AZpYFFLh7p7v/V2TRRWHDz2D3Wrjm\n25CdfD+EhpYwMajEICKTXLLXGH4FJB4Ri8J1x5d4PBhBtfpkOOP6I9q1PkwMs1RiEJFJLtnEUODu\nHQMPwvtF0YQUoVd+DI3r4ZI7IOvIRkOta45RnJdNeaF6O4vI5JZsYjhgZmcPPDCzc4DYYbafmNrq\nYcZZsPjdR7xrfUuMmZWFGhtJRCa9ZK8xfAr4oZk1AAZMA26ILKqoXHAbvOFWyDryDmr1zTG1SBKR\njJBUYnD3lWZ2CjAwTdmr7t4bXVgROsIqpAH1LTHOmlMxzsGIiEw8RzJRz0JgMVAAnG1muPv3owlr\nYuno7qM11qumqiKSEZJKDGb2ReASgsSwArgSeB7IiMSgpqoikkmSrWy/Drgc2O3uHwHOBCbHYHpJ\nGOjcpqaqIpIJkk0MMXePA31mVgY0EgyolxHqBksMx18LXRGRI5XsNYZVZlYBfAdYDXQAv48sqgmm\nvjlGbrZRW5rcuEoiIsezMRODBQ33v+ruLcC3zewXQJm7r03mDczsCuBfgWzgu+7+tWHPfwO4NHxY\nBNS6+4Rq/lPfEmNaeQFZWerDICKT35iJwd3dzFYAp4ePtyX74maWDdwDvAWoA1aa2XJ3X5/w+rcn\nbP8J4Kyko0+R+uZOXXgWkYyR7DWGNWZ27lG8/jJgi7tvdfce4GHg6sNs/z7goaN4n0g1tHTp+oKI\nZIxkrzGcB3zAzLYDBwh6P7u7nzHGfjOBnQmP68LXOoSZzQXmA78Z5fmbgJsA5syZk2TYx66nL86e\n9i71YRCRjJFsYnhbpFEEbgQeGxjaezh3vw+4D2Dp0qWegngA2N3ahTvMUlWSiGSIZKuSfJRlLPUM\nbdY6K1w3khuZgNVIdS2dgCboEZHMkWyJ4b8JEoERDIkxH3gVOHWM/VYCC8xsPkFCuBE4ZNq0cBym\nSiZgE9jBmdtUYhCRDJHsIHqnJz4Oh+C+OYn9+szsVuBJguaq97v7K2b2FWCVuy8PN70ReNjdU1ZF\nlKyBCXqmlRekORIRkdQ4kkH0Brn7GjMb8SLyCNuuIBhfKXHdncMef+lo4kiF+uYYNaX5FOQe3ais\nIiLHm2QH0ft0wsMs4GygIZKIJpiG1piqkUQkoyR78bk0YcknuOZwuP4Ik0Z9c0wXnkUkoyR7jeHL\nUQcyEcXjTkNLF287dVq6QxERSZmkSgxm9stwEL2Bx5Vm9mR0YU0MTR3d9PTHVWIQkYySbFVSTTiI\nHgDu3gzURhPSxFGnCXpEJAMlmxj6zWxwHIpw+IoJ17R0vA30YZihxCAiGSTZ5qqfB543s2cJOrld\nRDhu0WQ20IdBVUkikkmSvfj8i7BT2/nhqk+5e1N0YU0MDS0xSgtyKCvITXcoIiIpk+zF53cDve7+\nhLs/QTDF5zXRhpZ+9c3qwyAimSfZawxfdPfWgQfhhegvRhPSxFHfEmOWqpFEJMMkmxhG2u6ohtM4\nnqjEICKZKNnEsMrM7jazE8PlbmB1lIGlW2usl/buPrVIEpGMk2xi+ATQAzwSLt3ALVEFNREMDret\nqiQRyTDJtko6ANwRcSwTSr06t4lIhkp2dNUa4LMEE/MMTkzg7pdFFFfaNagPg4hkqGSrkh4ANhLM\n3PZlYBvB7GyTVn1LjLycLKqL89MdiohISiWbGKrc/T8I+jI86+4fBSZtaQEOtkjKyrJ0hyIiklLJ\nNjntDW93mdlVBJP0TIkmpImhrkVNVUUkMyWbGO4ys3LgM8C/AWXA7ZFFNQHUN8e47JSadIchIpJy\nybZKeiK82wpcOvx5M/ucu391PANLp67efpo6uplZUZTuUEREUi7Zawxjee84vc6EoBZJIpLJxisx\nTKortA0tXYD6MIhIZhqvxDCpJu2pb+kE0AB6IpKRVGIYQX1zjCyDaeUFY28sIjLJjFdi+OE4vc6E\nUNcSY2pZAbnZ4/X1iIgcPw575DOz6mGPP2hm3zSzm8xssJTg7v8YVYDpUN8c06iqIpKxxjolfmrg\njpl9AfgQwXDbbwHujjCutKpX5zYRyWBj9WNIvHbwHuAidz9gZg8Ca6ILK336487u1i41VRWRjDVW\nYig0s7MIShbZ4fDbuHuvmfVHHl0aNLZ30Rd3lRhEJGONlRh2cbDKaL+ZTXf3XWZWBfRFG1p6aIIe\nEcl0h039JtvyAAAOd0lEQVQM7n7I8BehZuDi8Q8n/QYm6JmlEoOIZKik22Oa2XvCeZ//Gbja3TuT\n3O8KM3vVzLaY2YizwJnZ9Wa23sxeCa9fpE1dWGJQqyQRyVTJzuD2LeAk4KFw1cfN7M3ufth5n80s\nG7iHoBVTHbDSzJa7+/qEbRYAnwMucPdmM6s9is8xbupbYlQU5VKcn+zAsyIik0uyR7/LgEXu7gBm\n9j3glST2WwZscfet4X4PA1cD6xO2+Rhwj7s3A7h7Y5IxRWJggh4RkUyVbFXSFmBOwuPZ4bqxzAR2\nJjyuC9clOhk42cx+a2YvmNkVI71Q2KlulZmt2rt3b5JhH7kG9WEQkQyXbGIoBTaY2TNm9gzBGX+Z\nmS03s+XHGEMOsAC4BHgf8B0zqxi+kbvf5+5L3X1pTU00E+i4e9C5TS2SRCSDJVuVdOdRvn49Qeli\nwKxwXaI64A/u3gu8bmabCBLFyqN8z6PW0tlLZ0+/SgwiktGSKjG4+7PARoKSQymwwd2fHVgOs+tK\nYIGZzTezPOBGYHgJ4ycEpYWBsZlOBrYe0acYJ4NNVVViEJEMllRiMLPrgT8SzNR2PfAHM7turP3c\nvQ+4FXgS2AA86u6vmNlXzOxd4WZPAvvMbD3wNPA/3X3fkX+UYzfQVFVTeopIJku2KunzwLkDLYbM\nrAb4FfDYWDu6+wpgxbB1dybcd+DT4ZJWAyWGGRWah0FEMleyF5+zhjUj3XcE+x436ptjFORmMaU4\nL92hiIikTbIlhp+b2ZMc7OB2A8NKAZPBQFPVhKkmREQyTrJn/Q7cC5wRLvdFFlEaBU1VdX1BRDJb\nsonhLe7+Y3f/dLg8DlwZZWDpoAl6RETGqEoys78CbgZOMLO1CU+VAr+NMrBU6+zpY/+BHjVVFZGM\nN9Y1hgeBnwNfBRJHRm139/2RRZUGDWqRJCICjD0fQyvQSjBUxaSmPgwiIoFJ1+T0aA30YdA4SSKS\n6ZQYQg0tMbKzjKml+ekORUQkrZQYQvXNMaaVFZCTra9ERDKbjoIhDbctIhJQYgjVN8eYpT4MIiJK\nDAC9/XF2t3UxQ4lBRESJAWB3axdxV4skERFQYgASmqqqxCAiosQAB3s9q8QgIqLEAAQXnkElBhER\nUGIAgqqk6pI8CnKz0x2KiEjaKTGg4bZFRBIpMRBUJampqohIIOMTg7urxCAikiDjE8O+Az1098XV\nIklEJJTxiUEtkkREhlJiUB8GEZEhlBjCEsMszdwmIgIoMVDfEqMkP4eywrGmvxYRyQwZnxjqmmPM\nqCjAzNIdiojIhJDxiUFNVUVEhsr4xNCgmdtERIbI6MTQ0d1Ha6yXmbrwLCIyKPLEYGZXmNmrZrbF\nzO4Y4fkPm9leM3spXP4i6pgGDPZhUIlBRGRQpE1xzCwbuAd4C1AHrDSz5e6+ftimj7j7rVHGMpL6\nlk5AndtERBJFXWJYBmxx963u3gM8DFwd8XsmbbAPg0oMIiKDok4MM4GdCY/rwnXDXWtma83sMTOb\nHXFMB4NpiZGbbdSU5KfqLUVEJryJcPH5Z8A8dz8D+CXwvZE2MrObzGyVma3au3fvuLxxfXOM6eWF\nZGWpD4OIyICoE0M9kFgCmBWuG+Tu+9y9O3z4XeCckV7I3e9z96XuvrSmpmZcgmtQHwYRkUNEnRhW\nAgvMbL6Z5QE3AssTNzCz6QkP3wVsiDimQfXqwyAicohIWyW5e5+Z3Qo8CWQD97v7K2b2FWCVuy8H\nPmlm7wL6gP3Ah6OMaUBPX5zG9m6VGEREhol85Dh3XwGsGLbuzoT7nwM+F3Ucw+1qjeGuPgwiIsNN\nhIvPaaEJekRERpaxiaGuRYlBRGQkGZsYBkoM0ysK0hyJiMjEkrGJoaElRm1pPvk52ekORURkQsnY\nxKCmqiIiI8vsxKDrCyIih8jIxBCPO7taulRiEBEZQUYmhr0d3fT0x1ViEBEZQUYmhjr1YRARGVVG\nJob6Fs3cJiIymoxMDA3q3CYiMqqMTAz1zTHKCnIoLchNdygiIhNOZiaGlhgzK4vSHYaIyISUmYmh\nWX0YRERGk3GJwd3Dzm0aI0lEZCQZlxjaYn10dPepRZKIyCgyLjHUtXQCMLNC1xhEREaScYmhoaUL\nUB8GEZHRZFxiqG8eKDEoMYiIjCTzEkNLjPycLKpL8tIdiojIhJSRiWFmRSFmlu5QREQmpMxLDM0x\nZqgaSURkVJmXGDRBj4jIYWVUYujq7aepo0ctkkREDiOjEoNGVRURGVtGJQbNwyAiMrbMSgyauU1E\nZEyZlRhaYmQZTCvXAHoiIqPJqMQwt6qYa5bMJDc7oz62iMgRyUl3AKl03TmzuO6cWekOQ0RkQtOp\ns4iIDBF5YjCzK8zsVTPbYmZ3HGa7a83MzWxp1DGJiMjoIk0MZpYN3ANcCSwG3mdmi0fYrhS4DfhD\nlPGIiMjYoi4xLAO2uPtWd+8BHgauHmG7vwe+DnRFHI+IiIwh6sQwE9iZ8LguXDfIzM4GZrv7f0cc\ni4iIJCGtF5/NLAu4G/hMEtveZGarzGzV3r17ow9ORCRDRZ0Y6oHZCY9nhesGlAKnAc+Y2TbgfGD5\nSBeg3f0+d1/q7ktramoiDFlEJLNFnRhWAgvMbL6Z5QE3AssHnnT3Vnevdvd57j4PeAF4l7uvijgu\nEREZRaQd3Ny9z8xuBZ4EsoH73f0VM/sKsMrdlx/+FUa2evXqJjPbfpRhVQNNR7lvKii+Y6P4jo3i\nO3YTOca5yWxk7h51IBOKma1y9wnbV0LxHRvFd2wU37E7HmIci3o+i4jIEEoMIiIyRCYmhvvSHcAY\nFN+xUXzHRvEdu+MhxsPKuGsMIiJyeJlYYhARkcOYtIlhrFFdzSzfzB4Jn/+Dmc1LYWyzzexpM1tv\nZq+Y2W0jbHOJmbWa2Uvhcmeq4gvff5uZvRy+9yH9SizwzfD7WxsObZKq2BYmfC8vmVmbmX1q2DYp\n/f7M7H4zazSzdQnrppjZL81sc3hbOcq+fxZus9nM/iyF8f0vM9sY/v0eN7OKUfY97G8hwvi+ZGb1\nCX/Dt4+yb1IjOEcQ3yMJsW0zs5dG2Tfy72/cufukWwj6TLwGnADkAX8CFg/b5mbg2+H9G4FHUhjf\ndODs8H4psGmE+C4Bnkjjd7gNqD7M828Hfg4YQY/1P6Txb70bmJvO7w+4GDgbWJew7p+AO8L7dwBf\nH2G/KcDW8LYyvF+ZovjeCuSE978+UnzJ/BYijO9LwF8n8fc/7P96VPENe/6fgTvT9f2N9zJZSwzJ\njOp6NfC98P5jwOVmZqkIzt13ufua8H47sIFhgwseB64Gvu+BF4AKM5uehjguB15z96Pt8Dgu3P05\nYP+w1Ym/se8B14yw69uAX7r7fndvBn4JXJGK+Nz9KXfvCx++QDBkTVqM8v0lI9kRnI/J4eILjxvX\nAw+N9/umy2RNDGOO6pq4TfjP0QpUpSS6BGEV1lmMPBfFG8zsT2b2czM7NaWBgQNPmdlqM7tphOeT\n+Y5T4UZG/4dM5/cHMNXdd4X3dwNTR9hmonyPHyUoAY5krN9ClG4Nq7ruH6UqbiJ8fxcBe9x98yjP\np/P7OyqTNTEcF8ysBPgR8Cl3bxv29BqC6pEzgX8DfpLi8C5097MJJlm6xcwuTvH7jykcf+tdwA9H\neDrd398QHtQpTMgmgGb2eaAPeGCUTdL1W/h34ERgCbCLoLpmInofhy8tTPj/peEma2IYa1TXIduY\nWQ5QDuxLSXTBe+YSJIUH3P3Hw5939zZ37wjvrwByzaw6VfG5e3142wg8TlBkT5TMdxy1K4E17r5n\n+BPp/v5Cewaq18LbxhG2Sev3aGYfBt4BfCBMXodI4rcQCXff4+797h4HvjPK+6b7+8sB3gM8Mto2\n6fr+jsVkTQyHHdU1tBwYaAFyHfCb0f4xxltYJ/kfwAZ3v3uUbaYNXPMws2UEf6uUJC4zK7ZgulXM\nrJjgIuW6YZstB/5H2DrpfKA1odokVUY9U0vn95cg8Tf2Z8BPR9jmSeCtZlYZVpW8NVwXOTO7Avgs\nwYjGnaNsk8xvIar4Eq9ZvXuU903mfz1KbwY2unvdSE+m8/s7Jum++h3VQtBqZhNBi4XPh+u+QvBP\nAFBAUAWxBfgjcEIKY7uQoFphLfBSuLwd+EvgL8NtbgVeIWhl8QLwxhTGd0L4vn8KYxj4/hLjM4L5\nvF8DXgaWpvjvW0xwoC9PWJe2748gQe0Cegnquf+c4JrVr4HNwK+AKeG2S4HvJuz70fB3uAX4SArj\n20JQPz/wGxxopTcDWHG430KK4vuv8Le1luBgP314fOHjQ/7XUxFfuP4/B35zCdum/Psb70U9n0VE\nZIjJWpUkIiJHSYlBRESGUGIQEZEhlBhERGQIJQYRERlCiUEkxcKRX59Idxwio1FiEBGRIZQYREZh\nZh80sz+G4+jfa2bZZtZhZt+wYB6NX5tZTbjtEjN7IWFug8pw/Ulm9qtwML81ZnZi+PIlZvZYOB/C\nA6ka2VckGUoMIiMws0XADcAF7r4E6Ac+QNDjepW7nwo8C3wx3OX7wN+4+xkEvXUH1j8A3OPBYH5v\nJOg9C8GIup8CFhP0jr0g8g8lkqScdAcgMkFdDpwDrAxP5gsJBsGLc3DAtB8APzazcqDC3Z8N138P\n+GE4Rs5Md38cwN27AMLX+6OH4+uEM3/NA56P/mOJjE2JQWRkBnzP3T83ZKXZ3w3b7mjHlOlOuN+P\n/hdlAlFVksjIfg1cZ2a1MDh/81yC/5nrwm3eDzzv7q1As5ldFK7/EPCsB7Pz1ZnZNeFr5JtZUUo/\nhchR0FmKyAjcfb2ZfYFg5q0sglE1bwEOAMvC5xoJrkNAMKz2t8MD/1bgI+H6DwH3mtlXwtd4bwo/\nhshR0eiqIkfAzDrcvSTdcYhESVVJIiIyhEoMIiIyhEoMIiIyhBKDiIgMocQgIiJDKDGIiMgQSgwi\nIjKEEoOIiAzx/wEl4REedkiXWwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb004f23550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['top_5_accuracy'], label='train');\n",
    "plt.plot(model.history.history['val_top_5_accuracy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('top5_accuracy');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 5120 images belonging to 256 classes.\n"
     ]
    }
   ],
   "source": [
    "val_generator_no_shuffle = data_generator.flow_from_directory(\n",
    "    data_dir + 'val', val_logits,\n",
    "    target_size=(224, 224),\n",
    "    batch_size=64, shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5.595249432325363, 0.66171875000000002, 0.86679687500000002, 1.4644993908703328, 5.4862456738948824]\n"
     ]
    }
   ],
   "source": [
    "print(model.evaluate_generator(val_generator_no_shuffle, 80))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}