CIFAR10_Keras_MobileNetV2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import os \n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/nicolasmaquaire/Dropbox/KLoop/Dev/190209_MLEnvs'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir_path = os.getcwd()\n",
    "dir_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir = \"{}/run-{}/\".format(root_logdir, now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpickle(file):\n",
    "    import pickle\n",
    "    with open(file, 'rb') as fo:\n",
    "        dict = pickle.load(fo, encoding='bytes')\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_1 = unpickle('datasets/cifar-10/data_batch_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_dataset_folder_path = 'datasets/cifar-10/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[b'batch_label', b'labels', b'data', b'filenames']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(cifar10_1.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 10000)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cifar10_1[b'data']),len(cifar10_1[b'labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'training batch 1 of 5'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cifar10_1[b'batch_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[b'airplane',\n",
       " b'automobile',\n",
       " b'bird',\n",
       " b'cat',\n",
       " b'deer',\n",
       " b'dog',\n",
       " b'frog',\n",
       " b'horse',\n",
       " b'ship',\n",
       " b'truck']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cifar10_Label_names = unpickle('datasets/cifar-10/batches.meta')\n",
    "cifar10_Label_names[b'label_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = []\n",
    "for x in cifar10_Label_names[b'label_names']:\n",
    "    label = x.decode()\n",
    "    label_names.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['airplane',\n",
       " 'automobile',\n",
       " 'bird',\n",
       " 'cat',\n",
       " 'deer',\n",
       " 'dog',\n",
       " 'frog',\n",
       " 'horse',\n",
       " 'ship',\n",
       " 'truck']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_cfar10_batch(cifar10_dataset_folder_path, batch_id):\n",
    "    with open(cifar10_dataset_folder_path + '/data_batch_' + str(batch_id), mode='rb') as file:\n",
    "        # note the encoding type is 'latin1'\n",
    "        batch = pickle.load(file, encoding='latin1')\n",
    "        \n",
    "    features = batch['data'].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1)\n",
    "    labels = batch['labels']\n",
    "        \n",
    "    return features, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_stats(cifar10_dataset_folder_path, batch_id, sample_id):\n",
    "    features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_id)\n",
    "    \n",
    "    if not (0 <= sample_id < len(features)):\n",
    "        print('{} samples in batch {}.  {} is out of range.'.format(len(features), batch_id, sample_id))\n",
    "        return None\n",
    "\n",
    "    print('\\nStats of batch #{}:'.format(batch_id))\n",
    "    print('# of Samples: {}\\n'.format(len(features)))\n",
    "    \n",
    "    label_counts = dict(zip(*np.unique(labels, return_counts=True)))\n",
    "    for key, value in label_counts.items():\n",
    "        print('Label Counts of [{}]({}) : {}'.format(key, label_names[key].upper(), value))\n",
    "    \n",
    "    sample_image = features[sample_id]\n",
    "    sample_label = labels[sample_id]\n",
    "    \n",
    "    print('\\nExample of Image {}:'.format(sample_id))\n",
    "    print('Image - Min Value: {} Max Value: {}'.format(sample_image.min(), sample_image.max()))\n",
    "    print('Image - Shape: {}'.format(sample_image.shape))\n",
    "    print('Label - Label Id: {} Name: {}'.format(sample_label, label_names[sample_label]))\n",
    "    \n",
    "    plt.imshow(sample_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Stats of batch #3:\n",
      "# of Samples: 10000\n",
      "\n",
      "Label Counts of [0](AIRPLANE) : 994\n",
      "Label Counts of [1](AUTOMOBILE) : 1042\n",
      "Label Counts of [2](BIRD) : 965\n",
      "Label Counts of [3](CAT) : 997\n",
      "Label Counts of [4](DEER) : 990\n",
      "Label Counts of [5](DOG) : 1029\n",
      "Label Counts of [6](FROG) : 978\n",
      "Label Counts of [7](HORSE) : 1015\n",
      "Label Counts of [8](SHIP) : 961\n",
      "Label Counts of [9](TRUCK) : 1029\n",
      "\n",
      "Example of Image 7000:\n",
      "Image - Min Value: 24 Max Value: 252\n",
      "Image - Shape: (32, 32, 3)\n",
      "Label - Label Id: 0 Name: airplane\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAH0CAYAAADVH+85AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmUrHV95/H3t3q5G1yEyyZuLLIormAigYRFEkbjuOMMf7iMJzqJiWMwMiczURNM9MScMyeuiebEhYnmBD04kpPE4AaKiokRVyKKyK5sF4QLd+3u+s0fz9Om6dt9732+XV3V/Pr9Oueeul1V3/796ldP17eeWp5PlFKQJEl16o16ApIkafnY6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmq2PioJ7AcIuJGYCNw04inIklS1pHAllLKUUv5JSNt9BHxaOCPgWcDm4DbgUuBt5ZSfraEX70xxtcctO7gIw/qXFmWMKoaxUXUnuS2j1SVm+I8D4e1r/NOy9yqXT+7lTKza8ljj6zRR8QxwFXAocDfAz8AfhH4XeDZEXFaKeWe5K+/ad3BRx705Ff9386Fpd/vXpP+44nONVG61wCJkdq6fvfbVkr3NWwrk3UrXPJmZcqya18ST86y21RmLMiuR3KsRF12rEgvZPeSfpnJDZW4bf0hbotNYa4sNVRm+0hM8Mb/dz47N//4ps6F84zyPfq/pGnyry+lvLCU8r9KKc8C3gkcD7x9hHOTJKkKI2n07d78OTTvof/FvIv/CNgKvDwiNgx5apIkVWVUe/RntaefLfNeayylPAB8FVgPnDLsiUmSVJNRNfrj29PrFrn8R+3pcUOYiyRJ1RrVh/EOaE/vX+Ty2fMfsadfEhFXL3LRCZlJSZJUGw+YI0lSxUa1Rz+7x37AIpfPnn/fnn5JKeXkhc5v9/RPyk1NkqR6jGqP/oft6WLvwR/bni72Hr4kSdoHo2r0V7Sn50TEQ+YQEfsDpwHbgH8Z9sQkSarJSBp9KeXHwGdpjuP7O/MufiuwAfhoKWXrkKcmSVJVRnms+9+mOQTueyLibOBa4Jk037G/DnjTCOcmSVIVRvap+3av/hnARTQN/o3AMcC7gVOWcJx7SZLUGml6XSnlVuBVy/G7g2B8fLJzXelnQh+yqRSZ0IdkcEY28CETMJEM3kmFAw05KS813BDXo2THyoTaJDf7dAhUalscYqhNaiSGOsf040Divs5ui4lcsdkRExXDC9DJ3M3pwKN5/B69JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUsZGG2iyniGBibKx7YS+TIpANLUmkN5Rc4kM2GyETgpG6XUA/lRSRGiotFXbSH1Ayxb6MlQ75ySRu5O7n7BxzmTZDDNAZZlhPsq6QeEwEMgFc2U2xl96GM3Mc3t+LoTaSJGlZ2OglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSKVZxeB+NjieifIaahlUymXMnFGWVDkHqJyn5yDfvJ1Lth6meS6FKJiDllqAlqydTGFZ4Y1tQNLwktvRyp9RjefTbc5EBSaYqZ+7mt7FzRH+Lf2Hzu0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWrN9QGGE/cun6/e8hBPnage2U2lCIr80ww0sE7w3vemQ3OiNRNW/nPp3PrMeTQkqGG2mT+NrNjDW8dSxkb2ljDVsp0omYmOVb39YhE2lcYaiNJkvbGRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFVsZOl1EXET8LhFLr6zlHL4EgegN949qSnY2b0mmQgVpfvzrH4yGa4kn9MF3dOdgu4JgJBbx5JMUMuXjS6Bal+kM8Yy6WQldz+nU95SaY/ZwRLb4sMivW54KXTp25UdsJ94HE48Bjd1ibXvJR47cnGZuxl1TO39wLsWOP/BYU9EkqQajbrR31dKuXDEc5AkqVq+Ry9JUsVGvUe/JiJeBjwW2Ap8F7iylNL9jWFJkrSbUTf6w4GPzjvvxoh4VSnlS3srjoirF7nohCXPTJKkCozypfuPAGfTNPsNwJOBvwKOBP45Ip46uqlJklSHke3Rl1LeOu+sa4DfiogHgTcCFwIv2svvOHmh89s9/ZMGME1Jkh7WVuKH8T7Qnp4+0llIklSBldjo725PN4x0FpIkVWAlNvpT2tMbRjoLSZIqMJJGHxFPiIjd9tgj4kjgfe2PHxvmnCRJqtGoPoz3X4E3RsSVwM3AA8AxwHOBtcCngf8zorlJklSNUTX6K4DjgacDp9G8H38f8BWa79V/tAwzfUGSpEqNpNG3B8PZ6wFxlqIXwZrxyc51pd89LajP2s41ACW6J+WNlR2pscb6uXdpSpnoXNOP3HO0VN0QU+hguOlfGfm0tsxY2Xf+Vn6CGomUyPQ21R9iel1qJHJpben0uuQsE+l1JZvAmPqD6V4yoPC6FflhPEmSNCA2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkio2qvS6ZRcE63rdb97M9Eznmp1Md64B6K/pHrozkQx8mJjO1c2U7ms43cuN1aP72mdlsyKGGiSSMcwgkWwgSDoYKBOskh0qE2qTHCq3jKnxhhvykxwqMRZAv3R//Ognx0rdtkyRoTaSJGlvbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLF6k2vCxgf7x79s2m/7glI+2/Y2bkG4M4H1naueXBH9xoAJnMRWf3onsw3lsyt6mWSpLJhXMnCXLLWEPPrhpjGFcn9hJJNvUuUpcPahlYEjGXT2rrXlEwRw05tzFVmblpyOVLbVeYxJ0yvkyRJe2OjlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWLVhtoQwET3skMP7l501hMO6T4QcPd93VM6PvOtu1Jj3c/+qbqJXvcghl5/KjVWlLFUXUYmpGMpdcMyzNsVQw0GAsYSwSpDXI9sUFK6LJHy0+9ng4gyKS7JtU9VQSmZ25bdPhJrnxjKUBtJkrRXNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliA0mvi4hzgTOApwFPBfYH/raU8rI91JwKvBk4BVgH/Aj4MPDeUsrM0ucEvcnuz2Ompronrx00M925BuCxGx/sXPOTQx5IjfXNe5LJcL31iaJcIlRmFSMd7zTMVLPkHDNjPSzS61JlybGyg3VPJxt2ImKmrJ+JUAP6Q9wWSSTDNXWZmuzaZ/aRu9+uXvaxY55BxdS+mabBPwjcBpywpytHxAuATwI7gI8D9wLPA94JnAa8dEDzkiRpVRvUS/dvAI4DNgKv3dMVI2Ij8NfADHBmKeU3Sin/k+bVgK8B50bEeQOalyRJq9pAGn0p5YpSyo/Kvr1Wcy5wCHBxKeUbc37HDppXBmAvTxYkSdK+GcWH8Z7Vnl62wGVXAtuAUyNizfCmJElSnQb1Hn0Xx7en182/oJQyHRE3AicCRwPX7ukXRcTVi1y0x88ISJK0Woxij/6A9vT+RS6fPf8RQ5iLJElVG8Ue/cCUUk5e6Px2T/+kIU9HkqQVZxR79LN77Acscvns+fcNYS6SJFVtFI3+h+3pcfMviIhx4CiaY6fcMMxJSZJUo1E0+svb02cvcNnpwHrgqlLKzuFNSZKkOo2i0V8CbAbOi4hnzJ4ZEWuBt7U/vn8E85IkqTqDOtb9C4EXtj8e3p7+UkRc1P5/cynlAoBSypaIeA1Nw/9iRFxMcwjc59N89e4SmsPiSpKkJRrUp+6fBrxy3nlHt/8AbgYumL2glHJpRJwBvAl4CbAWuB74PeA9+3iEPUmStBcDafSllAuBCzvWfBX49UGMv5AImEjcup0z3VPe7rprc/eBgJ9+5/K9X2meR+13cGqsmcOelKr7/j3dPyrRH59MjRXs6lyTTYaLZEJWJrFtpiSTAxNPd4eblJcbKhHi1YyXuG3ZOUbiXc2S3KZKOgWw+3r0EzVNXaYmF0Iaye0jc8tKybXAXApg4oYNJrzOPHpJkmpmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKlig0qvW3EiYDKRrTKVCH24fcvW7gMBcUf3MJwDD1+TGus5v/q4VN2O793Zuebmn21PjVXGut+2mWRIRySTVXqJ4JKxZLJKriw7ViLUJhXsQXaKqbLsHEsiWSUdajPMsM7sWIl1jOxYybrM6ufCaaD0U6N1rujFYLYN9+glSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSapY3el1E4m0oEQo0YPJBLXDjz66c82jjjgiNdbRB21I1Z3z1EM71/zjN25JjXXPjrHONWV8IjVWNkEtk6zVTw82PJnAsCGHkw11jpn7LH0/Jx8/+okEtdKfSY3Vm57qXBMz3WsA+pFNpOxeNx65fd3o/lDFTGKszG1aiHv0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxeoNtaHPZNnRue7BB+7pXPPAul2dawCOO/GEzjXrNm1MjTXd356qO/bg7mE4ZzzhkamxvnH93Z1rdkzlgjNKMu2kH93rppLPp6dnugeQlETQSVZJhrj0Z7IhP93XMXs/TzGdqssoyVCbkgh/GRvPhdo8Yr/uKS7rxxLJL8B0MqhqOrGOO7d27xEA9z3QvW5HJG7XYDJt3KOXJKlmNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliA0mvi4hzgTOApwFPBfYH/raU8rIFrnskcOMeft3HSynnLXVOPQobet2TzTbfe3vnms9/56rONQD/Nralc81TnnR8aqxfeeYzU3XHHHls55oTDz8gNdamdd3Trh7YmUsZS4aaMTPTfbyZXi6Cat26dZ1rSjK8bqbfPdWsnxwsEcoH5G7b9ExujiW638/ZFLrsetx04y3di7Y+kBrr0b3ureLAyVx7iY259LpDjz+6c839yfS6r3/v+s41123e1rlmLAaTRjmomNo30zT4B4HbgH3JX/0OcOkC518zoDlJkrTqDarRv4GmwV9Ps2d/xT7UfLuUcuGAxpckSQsYSKMvpfy8sUfkXr6SJEmDN6g9+owjIuI3gU3APcDXSinfHeF8JEmqzigb/a+1/34uIr4IvLKUsk+fMomIqxe5aF8+IyBJUvVG8fW6bcCfACcDB7b/Zt/XPxP4QkRsGMG8JEmqztD36EspdwF/OO/sKyPiHOArwDOBVwPv3offdfJC57d7+ictcaqSJD3srZgD5pRSpoEPtj+ePsq5SJJUixXT6Ft3t6e+dC9J0gCstEZ/Snt6w0hnIUlSJYbe6CPipIjYbdyIOJvmwDsAHxvurCRJqtOgjnX/QuCF7Y+Ht6e/FBEXtf/fXEq5oP3/nwPHRsRVNEfTA3gK8Kz2/28ppeQOHi9Jkh5iUJ+6fxrwynnnHd3+A7gZmG30HwVeBPwC8BxgArgT+ATwvlLKlwcxoV4P1k12rzv2yMd0rll3/5HdBwKu/crnOtf88/V7ygNa3K033rb3Ky3g9F85s3PNE4/tHi4BMDbR/aiKY91ziwCYTiaJbLnnrs41m+++IzXW4x73uM41Bx9ycGqsjRs3dq5Zvz73UZoFXtDbR93reuSO1NlPhIlEcqzt23al6qZu7B6K1d++OTXWzK13dq65d2p7aqz9DjssVXfIMZs61xx28H6psTb94uM71xz84+5r+PdrBtOiB3UI3AuBC/fxuh8CPjSIcSVJ0p6ttA/jSZKkAbLRS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVLFBpdetOAGM9aY715Xonmo2kYnJA04/+8zONVvvy6VP7dy+NVX3ta9+sXPNVYkagP0f0T196tBHHpEa65GHJ1Pe9lvXuWZizdrUWH/3iU90rrnhhh+nxnrKU57aueaJT3xKaqxHPebRqbr1a7qvfa9fUmOVibHONePjuYfTteNrUnWPeXT3bX/siENTY83sOLJ7zfTO1FgbDzwgVbe9dE8c7G/dlhprIrrf108/8qDONesHlF7nHr0kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWrN71uZpp1W7onvd1w0w861/zLFZ/qXAPwpKO6p089+tBc6trdt92Qqlu3vnvy2lTk0vx2zOzoXHPTT3JpbVO7cqlVh27qnrC3/8bcfbbl/gc712y9b3tqrM9f9qXONXfck1vDU375tFRdme6eLPmtr/9baqxjjj+6c81jH/vY1FiHbzokVbdje/f1H5/snsoHcPc9d3eumZqaSo01eXcuoXPylp90rlk7mUsOZKb7bdt/Xff0xV07uj8mLsQ9ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWLVhtpse+BnXP25T3Su+/a13+xcs3XLnZ1rAK7d1j2E4Z67uoeqANx3d/dQCoCx8e6byNjaDamx9nvEoZ1rds30U2PdeXv3tQe4/oc7O9dsfXBXaqy1491DMJ7w+Cemxvr3a7qHOX3p859NjXX99d3HApgcn+hc89Nbbk2NdfOt13eueeKJubU/4vBHpup+eG33dbztJzenxrrzrrs610xP5/42p3Z1Dy8CWLN+feea9YnQLoCx6e5/07+aCHN6YMv9nWsW4h69JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVW3J6XURsAl4EPBd4MvAoYBfwPeAjwEdKKbvFGEXEqcCbgVOAdcCPgA8D7y2l5OKL5pie2sHdd1zXuW4iug+9cWMuUa43Oda5Zlc/d5cddMhjU3W9iURi2E9ziWG7prunAG7bMZ0aa3pn9xQ6gP03dE+72m9D9xQ6gJjp/jy8lG2psR71yAM710zf9tPUWLf88JpU3Zo1k51rNu63MTXWXT/tnm44tWN7aqyfbMo9fsxMd9/2dz74YGqsqS3d6yYm1qTGKtO5h/+xfve0vOlduceBbYlUuW98/V+7j7N1a+eahQwipvalwPuB24ErgFuAw4AXAx8EnhMRLy2llNmCiHgB8ElgB/Bx4F7gecA7gdPa3ylJkpZoEI3+OuD5wD/N3XOPiD8Avg68hKbpf7I9fyPw18AMcGYp5Rvt+W8BLgfOjYjzSikXD2BukiStakt+j76Ucnkp5R/mvzxfSrkD+ED745lzLjoXOAS4eLbJt9ffQfNSPsBrlzovSZK0/B/Gm2pP576Z9Kz29LIFrn8lsA04NSJyb/BIkqSfG8RL9wuKiHHgFe2Pc5v68e3pbp+UK6VMR8SNwInA0cC1exnj6kUuOqHbbCVJqtNy7tG/A3gS8OlSymfmnH9Ae7rYxxZnz3/Eck1MkqTVYln26CPi9cAbgR8AL1+OMQBKKScvMv7VwEnLNa4kSQ8XA9+jj4jXAe8Gvg+cVUq5d95VZvfYD2Bhs+ffN+i5SZK02gy00UfE+cB7gWtomvwdC1zth+3pcQvUjwNH0Xx474ZBzk2SpNVoYI0+In6f5oA336Zp8nctctXL29NnL3DZ6cB64KpSSu6QRZIk6ecG0ujbg928A7gaOLuUsnkPV78E2AycFxHPmPM71gJva398/yDmJUnSajeIY92/EvhjmiPdfRl4fUTMv9pNpZSLAEopWyLiNTQN/4sRcTHNIXCfT/PVu0toDosrSZKWaBCfuj+qPR0Dzl/kOl8CLpr9oZRyaUScAbyJ5hC5a4Hrgd8D3jP3uPhZ69dt4OlP+oXOdVN0H3pXIkwBoLfb86F9qEmNBGP9xGAAve6byOMefXRqqJnSPaRjeqZ7MBBA7J6ztG8Scyyx5M15n03vyoX8HHHEozvXnPCEE1NjTSc34pLYhCfGcg9xEd23q95YblvsRW5BepkHkGOO2vt1FjC9a1eqbpgyf9EzycfuSNRFoq19899/wJYHlx5ss+RGX0q5ELgwUfdV4NeXOr4kSVqcefSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFBpFetzKVIPprO5dF2dm5ZjIZDJcIyqOXfG7Wy0R/Acx0r1s/uX9uqMxNi4nUWJkkKQD6M51LskvfT8yxbOg+v6YwM8lcWlvp5bbhfiKfrMwkkyUT93MqTQ7IhnX2E3Nk9wjxfTK2ZrJzzczMVGqsUnLbcC+R0Dk5kXv8yPxtzmTu5+T9NZ979JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVaza9LpCYSqmO9dlEoZKLiCLkkgMy2YZJQPDIJMY1s8Nlqnrl+73cTNYMr0uETnY7+fm2O93X/veWHLtU8uR2/CzK58aK/vHmVr75F9nckEyqXclOVgkUtSyS59ORUzU7EqmG2bWPpVGOaA/FvfoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkilUcagO7EokA0zOJsbKhFP3uhVPTuYCU0k/cMIDIhNrkFqSfrMsYH89u+plQm6nUSBHdn4dPTORu19hYJkgkt5/QyycsdRe5gJRMClQi9wXIBaS0hcOpSdZltt+mLreQmRCofjp5JxFIlrhZ2RCz+dyjlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYktOr4uITcCLgOcCTwYeBewCvgd8BPhIKf8RERQRRwI37uFXfryUct5S57Vraprbbr+7c910Ih2uP5NLQIpEslZJpi31erkcpMk1E93HSo0EY4kpTk5OpsYaZkLW+Hj3NQSYmMjctuElAGbXMCuT8pa5v7J12eXIpjZm1j+blFeGmAw3nYkQJZf0lt2GS+bvLFHSz6YNzjOImNqXAu8HbgeuAG4BDgNeDHwQeE5EvLTsvoV9B7h0gd93zQDmJEmSGEyjvw54PvBP8/bc/wD4OvASmqb/yXl13y6lXDiA8SVJ0iKW/B59KeXyUso/lHmvKZdS7gA+0P545lLHkSRJ3Q1ij35PptrThd74PiIifhPYBNwDfK2U8t1lno8kSavKsjX6iBgHXtH+eNkCV/m19t/cmi8Cryyl3LKPY1y9yEUn7OM0JUmq2nJ+ve4dwJOAT5dSPjPn/G3AnwAnAwe2/86g+SDfmcAXImLDMs5LkqRVY1n26CPi9cAbgR8AL597WSnlLuAP55VcGRHnAF8Bngm8Gnj33sYppZy8yPhXAyd1n7kkSXUZ+B59RLyOpkl/HzirlHLvvtSVUqZpvo4HcPqg5yVJ0mo00EYfEecD76X5LvxZ7Sfvu5g9wo0v3UuSNAADa/QR8fvAO4Fv0zT5uxK/5pT29IZBzUuSpNVsII0+It5C8+G7q4GzSymb93DdkyJit3Ej4mzgDe2PHxvEvCRJWu0Gcaz7VwJ/DMwAXwZev8Dxg28qpVzU/v/PgWMj4irgtva8pwDPav//llLKVUudlyRJGsyn7o9qT8eA8xe5zpeAi9r/f5QmBOcXgOcAE8CdwCeA95VSvjyAOTE9Pc3mzfv0OcCH6O3+YsNejU/klnHt2nWdayaSIS5r1uTqMqE248lwj7FEwMT4eG7te73ci1lTU1N7v9I8Y2O5scbGut+2dGhJIoAkO1bWzEz3sJNsqE3qpmUDUpKhNpmUlPw9lgnQyQbG5OqmE9tH7o4enkH9jS250bfHq7+ww/U/BHxoqeNKkqS9M49ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmqmI1ekqSK2eglSaqYjV6SpIrZ6CVJqpiNXpKkitnoJUmq2CDS61ak8bExDjrggM51ExPd09rGxsY612Trer1cstPkZPfbBZAI80tmT0Evkf6VSTQD2LlzZ6ouM15mmwKYns7dtuHJJsMNM2EvNVQqeW2BeO59kk/Y637jsnPMLGM+eS03x0wi5cxMbu1TSYqJmkGl17lHL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxapNr+v1eqxft7ZzXSrdKZuQ1U8kICXH2rVziEloyaePmbXvZ9OnpqdTdZOTk51r+sk7LSJRl00nyyShJTf8YSbKZfUzk0zfruw6dq+bnso9DpTEjcukyQH0EymFWdnHj8x9PTO8TWo37tFLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JDwimXAAAMiElEQVRUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUMRu9JEkVqzbUppTCrkRwSSYootdLhm0kQjqysR6ZUAqAXnR/LjjVn0qNtWt6Z+eakkmKANavXZeqm0ysRzaIqCQKSzYQJDPHIYaxtJWdK1LhNEBqFZOPA/1+7j7L1G3fuiM1VuaRZ826NamR+snHqplESFgk/1wye8iph6oBpdq4Ry9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsUGkl4XEX8GPAM4DjgY2A7cDFwKvK+Ucs8CNacCbwZOAdYBPwI+DLy3lNI9hmie6ZkZNv/s/s51mSS6Xm+scw3AWCIJLYb83KzX6z7edDa9blf3ZK01k5OpsUrJrePUVCIhqze8RLlhJsOlk7WyaX6Z9LpkMlyJTLJk8m8znW7YvWZifCI11nQiGW7nrtzjQCLUE4BI3Ge97NoP6c9lQOF1A+sabwA2AJ8D3g38LTANXAh8NyIeM/fKEfEC4ErgdOBTwPuASeCdwMUDmpMkSaveoPLoN5ZSdtsdi4i3A38A/G/gt9vzNgJ/DcwAZ5ZSvtGe/xbgcuDciDivlGLDlyRpiQayR79Qk299oj09ds555wKHABfPNvk5v+PN7Y+vHcS8JEla7Zb7Dd/ntaffnXPes9rTyxa4/pXANuDUiFiznBOTJGk1GNRL9wBExAXAfsABNB/O+2WaJv+OOVc7vj29bn59KWU6Im4ETgSOBq7dy3hXL3LRCd1mLklSnQba6IELgMPm/HwZ8N9KKXfPOe+A9nSxj8TPnv+IAc9NkqRVZ6CNvpRyOEBEHAacSrMn/62I+M+llG8Ocqx2vJMXOr/d0z9p0ONJkvRwsyzv0ZdS7iylfAo4B9gE/M2ci2f32A/YrfCh59+3HHOTJGk1WdYP45VSbga+D5wYEQe3Z/+wPT1u/vUjYhw4iuY7+Dcs59wkSVoNhnGYtSPa09lDK13enj57geueDqwHriql7FzuiUmSVLslN/qIOC4idnsZPiJ67QFzDqVp3D9rL7oE2AycFxHPmHP9tcDb2h/fv9R5SZKkwXwY79eBP42IrwA3AvfQfPL+DJqvyN0BvGb2yqWULRHxGpqG/8WIuBi4F3g+zVfvLgE+PoB5SZK06g2i0X8eeDzNd+afTvO1uK0035P/KPCeUsq9cwtKKZdGxBnAm4CXAGuB64Hfa6+/5GP5T03PcOfme/d+xXlmZrqHN5RMugQQiWCVHrnEh+ySZkJtsmONj3cf69CDN6XG2sb2VN2O7d2Dd/rJ7SMTyFKSIS4ZM9mxktvHzEz38TJ/zwAkwq3GJ3KBMVmZv7NIxqRkAmp27ppOjUUyJGxisvv6TyYe3yAXhjMT3Ysy2/xCltzoSynXAK9L1H2V5tUASZK0TMyjlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWI2ekmSKmajlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWIxgPyYFSci7ole76C16zZ0L06sR3YFc/E09YpIBImM5wIweomxGpm63BaSqqrw73lW5rEqvRyJuzmz/T5clH73hexnFz+5jJn1z4aEZWRWY+uDD9Dvz9xbSsmld7UGkV63Em0p/T7btz5w0wKXndCe/mCI81nJXI+Hcj0eyvV4KNfjoVyPhxr0ehwJbFnqL6lyj35PIuJqgFLKyaOey0rgejyU6/FQrsdDuR4P5Xo81EpdD9+jlySpYjZ6SZIqZqOXJKliNnpJkipmo5ckqWKr7lP3kiStJu7RS5JUMRu9JEkVs9FLklQxG70kSRWz0UuSVDEbvSRJFbPRS5JUsVXT6CPi0RHx4Yj4aUTsjIibIuJdEXHgqOc2bO1tL4v8u2PU81sOEXFuRLw3Ir4cEVva2/qxvdScGhGfjoh7I2J7RHw3Is6PiLFhzXu5dFmPiDhyD9tLiYiLhz3/QYqITRHx6oj4VERc397X90fEVyLiNyJiwcfJWrePrutR+/YBEBF/FhFfiIhb2/W4NyK+FRF/FBELZsWvpO2j1jz6h4iIY4CrgEOBv6fJCv5F4HeBZ0fEaaWUe0Y4xVG4H3jXAuc/OOyJDMmbgafS3L7b+I/c6AVFxAuATwI7gI8D9wLPA94JnAa8dDknOwSd1qP1HeDSBc6/ZoDzGoWXAu8HbgeuAG4BDgNeDHwQeE5EvLTMObpY5dtH5/Vo1bp9ALwB+CbwOeAuYANwCnAh8N8j4pRSyq2zV15x20cppfp/wGeAAvyPeef/eXv+B0Y9xyGvx03ATaOex5Bv81nAsUAAZ7b3+8cWue5Gmj/mncAz5py/luYJYwHOG/VtGuJ6HNleftGo571Ma/Esmgfh3rzzD6dpcgV4yWrZPhLrUfX2MXvfLnL+29vb/pcrefuo/qX7dm/+HJrm9hfzLv4jYCvw8ojYMOSpaYhKKVeUUn5U2r+4vTgXOAS4uJTyjTm/YwfNnjDAa5dhmkPTcT2qVkq5vJTyD6WU/rzz7wA+0P545pyLqt4+EutRvfa+Xcgn2tNj55y34raP1fDS/Vnt6WcX2HAfiIiv0jwROAX4wrAnN0JrIuJlwGNpnux8F7iylDIz2mmtCM9qTy9b4LIrgW3AqRGxppSyc3jTGrkjIuI3gU3APcDXSinfHfGclttUezo957zVvH0stB6zVuP28bz2dO7tXHHbx2po9Me3p9ctcvmPaBr9cayuRn848NF5590YEa8qpXxpFBNaQRbdZkop0xFxI3AicDRw7TAnNmK/1v77uYj4IvDKUsotI5nRMoqIceAV7Y9zH7RX5faxh/WYVf32EREXAPsBBwDPAH6Zpsm/Y87VVtz2Uf1L9zR3CDQfPlvI7PmPGMJcVoqPAGfTNPsNwJOBv6J5r+2fI+Kpo5vaiuA281DbgD8BTgYObP+dQfNBrTOBL1T61tc7gCcBny6lfGbO+at1+1hsPVbT9nEBzVu+59M0+cuAc0opd8+5zorbPlZDo9c8pZS3tu/D3VlK2VZKuaaU8ls0H05cR/NJUgmAUspdpZQ/LKV8s5RyX/vvSppXwv4VeDzw6tHOcrAi4vXAG2m+ofPyEU9n5Pa0Hqtp+yilHF5KCZqdpBfT7JV/KyJOGu3M9mw1NPrZZ08HLHL57Pn3DWEuK93sB21OH+ksRs9tZh+UUqZpvm4FFW0zEfE64N3A94GzSin3zrvKqto+9mE9FlTr9gHQ7iR9iubJzCbgb+ZcvOK2j9XQ6H/Ynh63yOWzn5Zc7D381WT25adaXmbLWnSbad+nPIrmw0g3DHNSK1RV20xEnA+8l+a732e1nzSfb9VsH/u4HntS1fYxXynlZponQCdGxMHt2Stu+1gNjf6K9vScBY7otD/NwQu2Af8y7ImtQKe0pw/7B6glurw9ffYCl50OrAeuqvAT1RnVbDMR8fs0BzT5Nk1Tu2uRq66K7aPDeuxJNdvHHhzRns5+Y2nFbR/VN/pSyo+Bz9J80Ox35l38Vppnmh8tpWwd8tRGIiKesNAHYyLiSOB97Y97PDTsKnAJsBk4LyKeMXtmRKwF3tb++P5RTGwUIuKkhQ4DGxFn0xwxDB7m20xEvIXmw2ZXA2eXUjbv4erVbx9d1qP27SMijouI3V6Gj4heRLyd5oirV5VSftZetOK2j1gNx8tY4BC41wLPpPmO/XXAqWWVHAI3Ii6k+VDNlcDNwAPAMcBzaY7c9GngRaWUXaOa43KIiBcCL2x/PBz4TzR7GV9uz9tcSrlg3vUvoTmE5cU0h7B8Ps1XZy4B/svD+WAzXdaj/YrUsTR/Q7e1lz+F//i+8FtKKbMPYA87EfFK4CKaPbL3svCnpW8qpVw0p6ba7aPreqyC7eN84E+BrwA30hwj4DCabxYcDdxB82To+3NqVtb2MczD8I3yH/AYmq+V3Q7somly7wIOHPXchrwOZwB/R/Pp2ftoDoBxN80xnF9B++Svtn803yQoe/h30wI1p9E88fkZsB34Hs0eytiob88w1wP4DeAfaY4u+SDNoT1voTmG96+M+rYMYS0K8MXVsn10XY9VsH08iebVzm/T7KlP0zz5+bd2rQ5apG7FbB+rYo9ekqTVqvr36CVJWs1s9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLFbPSSJFXMRi9JUsVs9JIkVcxGL0lSxWz0kiRVzEYvSVLF/j8JAh9TxU932QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 253
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "# Explore the dataset\n",
    "batch_id = 3\n",
    "sample_id = 7000\n",
    "display_stats(cifar10_dataset_folder_path, batch_id, sample_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(x):\n",
    "\n",
    "    min_val = np.min(x)\n",
    "    max_val = np.max(x)\n",
    "    x = (x-min_val) / (max_val-min_val)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(x):\n",
    "\n",
    "    encoded = np.zeros((len(x), 10))\n",
    "    \n",
    "    for idx, val in enumerate(x):\n",
    "        encoded[idx][val] = 1\n",
    "    \n",
    "    return encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _preprocess_and_save(normalize, one_hot_encode, features, labels, filename):\n",
    "    features = normalize(features)\n",
    "    # one hot encoding only for categorical_crossentropy and categorical_accuracy\n",
    "    #labels = one_hot_encode(labels)\n",
    "\n",
    "    pickle.dump((features, labels), open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode):\n",
    "    n_batches = 5\n",
    "    valid_features = []\n",
    "    valid_labels = []\n",
    "    all_features = []\n",
    "    all_labels = []\n",
    "\n",
    "    for batch_i in range(1, n_batches + 1):\n",
    "        features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_i)\n",
    "        \n",
    "        # find index to be the point as validation data in the whole dataset of the batch (10%)\n",
    "        index_of_validation = int(len(features) * 0.1)\n",
    "\n",
    "        valid_features.extend(features[-index_of_validation:])\n",
    "        valid_labels.extend(labels[-index_of_validation:])\n",
    "        all_features.extend(features[:-index_of_validation])\n",
    "        all_labels.extend(labels[:-index_of_validation])\n",
    "\n",
    "    # preprocess the all stacked validation dataset\n",
    "    _preprocess_and_save(normalize, one_hot_encode,\n",
    "                         np.array(valid_features), np.array(valid_labels),\n",
    "                         'datasets/preprocess_validation.p')\n",
    "                                 \n",
    "    _preprocess_and_save(normalize, one_hot_encode,\n",
    "                         np.array(all_features), np.array(all_labels),\n",
    "                         'datasets/preprocess_all.p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "float64\n",
      "(5000, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "valid_features, valid_labels = pickle.load(open('datasets/preprocess_validation.p', mode='rb'))\n",
    "print(valid_features.dtype)\n",
    "print(valid_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "float64\n",
      "(45000, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "train_features, train_labels = pickle.load(open('datasets/preprocess_all.p', mode='rb'))\n",
    "print(train_features.dtype)\n",
    "print(train_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(target_size):\n",
    "    input_tensor = tf.keras.layers.Input(shape=(target_size, target_size, 3))\n",
    "    base_model = tf.keras.applications.MobileNetV2(\n",
    "        include_top=True,\n",
    "        alpha=1,\n",
    "        weights=None,\n",
    "        input_tensor=input_tensor,\n",
    "        input_shape=(target_size, target_size, 3),\n",
    "        pooling='avg')\n",
    "        \n",
    "    output0 = tf.keras.layers.Dense(256, activation='relu')(base_model.output)\n",
    "    output1 = tf.keras.layers.Dropout(.1)(output0)\n",
    "\n",
    "    output_tensor = tf.keras.layers.Dense(10, activation='softmax')(output1)\n",
    "\n",
    "    model = tf.keras.Model(inputs=input_tensor, outputs=output_tensor)\n",
    "\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/nicolasmaquaire/Dropbox/KLoop/Dev/190209_MLEnvs/env/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/nicolasmaquaire/Dropbox/KLoop/Dev/190209_MLEnvs/env/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "model = build_model(32)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['sparse_categorical_accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_pad (ZeroPadding2D)       (None, 33, 33, 3)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Conv1 (Conv2D)                  (None, 16, 16, 32)   864         Conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bn_Conv1 (BatchNormalizationV1) (None, 16, 16, 32)   128         Conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_relu (ReLU)               (None, 16, 16, 32)   0           bn_Conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise (Depthw (None, 16, 16, 32)   288         Conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_BN (Bat (None, 16, 16, 32)   128         expanded_conv_depthwise[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_relu (R (None, 16, 16, 32)   0           expanded_conv_depthwise_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project (Conv2D)  (None, 16, 16, 16)   512         expanded_conv_depthwise_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project_BN (Batch (None, 16, 16, 16)   64          expanded_conv_project[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand (Conv2D)         (None, 16, 16, 96)   1536        expanded_conv_project_BN[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_BN (BatchNormali (None, 16, 16, 96)   384         block_1_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_relu (ReLU)      (None, 16, 16, 96)   0           block_1_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_pad (ZeroPadding2D)     (None, 17, 17, 96)   0           block_1_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise (DepthwiseCon (None, 8, 8, 96)     864         block_1_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_BN (BatchNorm (None, 8, 8, 96)     384         block_1_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_relu (ReLU)   (None, 8, 8, 96)     0           block_1_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project (Conv2D)        (None, 8, 8, 24)     2304        block_1_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project_BN (BatchNormal (None, 8, 8, 24)     96          block_1_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand (Conv2D)         (None, 8, 8, 144)    3456        block_1_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_BN (BatchNormali (None, 8, 8, 144)    576         block_2_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_relu (ReLU)      (None, 8, 8, 144)    0           block_2_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise (DepthwiseCon (None, 8, 8, 144)    1296        block_2_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_BN (BatchNorm (None, 8, 8, 144)    576         block_2_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_relu (ReLU)   (None, 8, 8, 144)    0           block_2_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project (Conv2D)        (None, 8, 8, 24)     3456        block_2_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project_BN (BatchNormal (None, 8, 8, 24)     96          block_2_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_add (Add)               (None, 8, 8, 24)     0           block_1_project_BN[0][0]         \n",
      "                                                                 block_2_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand (Conv2D)         (None, 8, 8, 144)    3456        block_2_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_BN (BatchNormali (None, 8, 8, 144)    576         block_3_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_relu (ReLU)      (None, 8, 8, 144)    0           block_3_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_pad (ZeroPadding2D)     (None, 9, 9, 144)    0           block_3_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise (DepthwiseCon (None, 4, 4, 144)    1296        block_3_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_BN (BatchNorm (None, 4, 4, 144)    576         block_3_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_relu (ReLU)   (None, 4, 4, 144)    0           block_3_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project (Conv2D)        (None, 4, 4, 32)     4608        block_3_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project_BN (BatchNormal (None, 4, 4, 32)     128         block_3_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand (Conv2D)         (None, 4, 4, 192)    6144        block_3_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_BN (BatchNormali (None, 4, 4, 192)    768         block_4_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_relu (ReLU)      (None, 4, 4, 192)    0           block_4_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise (DepthwiseCon (None, 4, 4, 192)    1728        block_4_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_BN (BatchNorm (None, 4, 4, 192)    768         block_4_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_relu (ReLU)   (None, 4, 4, 192)    0           block_4_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project (Conv2D)        (None, 4, 4, 32)     6144        block_4_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project_BN (BatchNormal (None, 4, 4, 32)     128         block_4_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_add (Add)               (None, 4, 4, 32)     0           block_3_project_BN[0][0]         \n",
      "                                                                 block_4_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand (Conv2D)         (None, 4, 4, 192)    6144        block_4_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_BN (BatchNormali (None, 4, 4, 192)    768         block_5_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_relu (ReLU)      (None, 4, 4, 192)    0           block_5_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise (DepthwiseCon (None, 4, 4, 192)    1728        block_5_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_BN (BatchNorm (None, 4, 4, 192)    768         block_5_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_relu (ReLU)   (None, 4, 4, 192)    0           block_5_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project (Conv2D)        (None, 4, 4, 32)     6144        block_5_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project_BN (BatchNormal (None, 4, 4, 32)     128         block_5_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_5_add (Add)               (None, 4, 4, 32)     0           block_4_add[0][0]                \n",
      "                                                                 block_5_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand (Conv2D)         (None, 4, 4, 192)    6144        block_5_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_BN (BatchNormali (None, 4, 4, 192)    768         block_6_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_relu (ReLU)      (None, 4, 4, 192)    0           block_6_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_pad (ZeroPadding2D)     (None, 5, 5, 192)    0           block_6_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise (DepthwiseCon (None, 2, 2, 192)    1728        block_6_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_BN (BatchNorm (None, 2, 2, 192)    768         block_6_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_relu (ReLU)   (None, 2, 2, 192)    0           block_6_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project (Conv2D)        (None, 2, 2, 64)     12288       block_6_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project_BN (BatchNormal (None, 2, 2, 64)     256         block_6_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand (Conv2D)         (None, 2, 2, 384)    24576       block_6_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_BN (BatchNormali (None, 2, 2, 384)    1536        block_7_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_relu (ReLU)      (None, 2, 2, 384)    0           block_7_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise (DepthwiseCon (None, 2, 2, 384)    3456        block_7_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_BN (BatchNorm (None, 2, 2, 384)    1536        block_7_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_relu (ReLU)   (None, 2, 2, 384)    0           block_7_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project (Conv2D)        (None, 2, 2, 64)     24576       block_7_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project_BN (BatchNormal (None, 2, 2, 64)     256         block_7_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_add (Add)               (None, 2, 2, 64)     0           block_6_project_BN[0][0]         \n",
      "                                                                 block_7_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand (Conv2D)         (None, 2, 2, 384)    24576       block_7_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_BN (BatchNormali (None, 2, 2, 384)    1536        block_8_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_relu (ReLU)      (None, 2, 2, 384)    0           block_8_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise (DepthwiseCon (None, 2, 2, 384)    3456        block_8_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_BN (BatchNorm (None, 2, 2, 384)    1536        block_8_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_relu (ReLU)   (None, 2, 2, 384)    0           block_8_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project (Conv2D)        (None, 2, 2, 64)     24576       block_8_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project_BN (BatchNormal (None, 2, 2, 64)     256         block_8_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_8_add (Add)               (None, 2, 2, 64)     0           block_7_add[0][0]                \n",
      "                                                                 block_8_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand (Conv2D)         (None, 2, 2, 384)    24576       block_8_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_BN (BatchNormali (None, 2, 2, 384)    1536        block_9_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_relu (ReLU)      (None, 2, 2, 384)    0           block_9_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise (DepthwiseCon (None, 2, 2, 384)    3456        block_9_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_BN (BatchNorm (None, 2, 2, 384)    1536        block_9_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_relu (ReLU)   (None, 2, 2, 384)    0           block_9_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project (Conv2D)        (None, 2, 2, 64)     24576       block_9_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project_BN (BatchNormal (None, 2, 2, 64)     256         block_9_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_9_add (Add)               (None, 2, 2, 64)     0           block_8_add[0][0]                \n",
      "                                                                 block_9_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand (Conv2D)        (None, 2, 2, 384)    24576       block_9_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_BN (BatchNormal (None, 2, 2, 384)    1536        block_10_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_relu (ReLU)     (None, 2, 2, 384)    0           block_10_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise (DepthwiseCo (None, 2, 2, 384)    3456        block_10_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_BN (BatchNor (None, 2, 2, 384)    1536        block_10_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_relu (ReLU)  (None, 2, 2, 384)    0           block_10_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project (Conv2D)       (None, 2, 2, 96)     36864       block_10_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project_BN (BatchNorma (None, 2, 2, 96)     384         block_10_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand (Conv2D)        (None, 2, 2, 576)    55296       block_10_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_BN (BatchNormal (None, 2, 2, 576)    2304        block_11_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_relu (ReLU)     (None, 2, 2, 576)    0           block_11_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise (DepthwiseCo (None, 2, 2, 576)    5184        block_11_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_BN (BatchNor (None, 2, 2, 576)    2304        block_11_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_relu (ReLU)  (None, 2, 2, 576)    0           block_11_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project (Conv2D)       (None, 2, 2, 96)     55296       block_11_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project_BN (BatchNorma (None, 2, 2, 96)     384         block_11_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_add (Add)              (None, 2, 2, 96)     0           block_10_project_BN[0][0]        \n",
      "                                                                 block_11_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand (Conv2D)        (None, 2, 2, 576)    55296       block_11_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_BN (BatchNormal (None, 2, 2, 576)    2304        block_12_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_relu (ReLU)     (None, 2, 2, 576)    0           block_12_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise (DepthwiseCo (None, 2, 2, 576)    5184        block_12_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_BN (BatchNor (None, 2, 2, 576)    2304        block_12_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_relu (ReLU)  (None, 2, 2, 576)    0           block_12_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project (Conv2D)       (None, 2, 2, 96)     55296       block_12_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project_BN (BatchNorma (None, 2, 2, 96)     384         block_12_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_12_add (Add)              (None, 2, 2, 96)     0           block_11_add[0][0]               \n",
      "                                                                 block_12_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand (Conv2D)        (None, 2, 2, 576)    55296       block_12_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_BN (BatchNormal (None, 2, 2, 576)    2304        block_13_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_relu (ReLU)     (None, 2, 2, 576)    0           block_13_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_pad (ZeroPadding2D)    (None, 3, 3, 576)    0           block_13_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise (DepthwiseCo (None, 1, 1, 576)    5184        block_13_pad[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_BN (BatchNor (None, 1, 1, 576)    2304        block_13_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_relu (ReLU)  (None, 1, 1, 576)    0           block_13_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project (Conv2D)       (None, 1, 1, 160)    92160       block_13_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project_BN (BatchNorma (None, 1, 1, 160)    640         block_13_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand (Conv2D)        (None, 1, 1, 960)    153600      block_13_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_BN (BatchNormal (None, 1, 1, 960)    3840        block_14_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_relu (ReLU)     (None, 1, 1, 960)    0           block_14_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise (DepthwiseCo (None, 1, 1, 960)    8640        block_14_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_BN (BatchNor (None, 1, 1, 960)    3840        block_14_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_relu (ReLU)  (None, 1, 1, 960)    0           block_14_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project (Conv2D)       (None, 1, 1, 160)    153600      block_14_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project_BN (BatchNorma (None, 1, 1, 160)    640         block_14_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_add (Add)              (None, 1, 1, 160)    0           block_13_project_BN[0][0]        \n",
      "                                                                 block_14_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand (Conv2D)        (None, 1, 1, 960)    153600      block_14_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_BN (BatchNormal (None, 1, 1, 960)    3840        block_15_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_relu (ReLU)     (None, 1, 1, 960)    0           block_15_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise (DepthwiseCo (None, 1, 1, 960)    8640        block_15_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_BN (BatchNor (None, 1, 1, 960)    3840        block_15_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_relu (ReLU)  (None, 1, 1, 960)    0           block_15_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project (Conv2D)       (None, 1, 1, 160)    153600      block_15_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project_BN (BatchNorma (None, 1, 1, 160)    640         block_15_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_15_add (Add)              (None, 1, 1, 160)    0           block_14_add[0][0]               \n",
      "                                                                 block_15_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand (Conv2D)        (None, 1, 1, 960)    153600      block_15_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_BN (BatchNormal (None, 1, 1, 960)    3840        block_16_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_relu (ReLU)     (None, 1, 1, 960)    0           block_16_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise (DepthwiseCo (None, 1, 1, 960)    8640        block_16_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_BN (BatchNor (None, 1, 1, 960)    3840        block_16_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_relu (ReLU)  (None, 1, 1, 960)    0           block_16_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project (Conv2D)       (None, 1, 1, 320)    307200      block_16_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project_BN (BatchNorma (None, 1, 1, 320)    1280        block_16_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1 (Conv2D)                 (None, 1, 1, 1280)   409600      block_16_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1_bn (BatchNormalizationV1 (None, 1, 1, 1280)   5120        Conv_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "out_relu (ReLU)                 (None, 1, 1, 1280)   0           Conv_1_bn[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 1280)         0           out_relu[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "Logits (Dense)                  (None, 1000)         1281000     global_average_pooling2d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256)          256256      Logits[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 256)          0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           2570        dropout[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 3,797,810\n",
      "Trainable params: 3,763,698\n",
      "Non-trainable params: 34,112\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainableLayers(model):\n",
    "    \n",
    "        for layer in model.layers:\n",
    "            layer.trainable = True  \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainableLayers(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = tf.keras.callbacks.TensorBoard(log_dir=logdir, histogram_freq=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_all_batches(xEpochs):\n",
    "        \n",
    "        model.fit(\n",
    "            train_features, train_labels,\n",
    "            epochs=xEpochs,\n",
    "            batch_size=32,\n",
    "            validation_data=(valid_features, valid_labels),\n",
    "            callbacks=[tensorboard] #, earlystopping]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 45000 samples, validate on 5000 samples\n",
      "WARNING:tensorflow:From /Users/nicolasmaquaire/Dropbox/KLoop/Dev/190209_MLEnvs/env/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/20\n",
      "45000/45000 [==============================] - 294s 7ms/sample - loss: 2.1632 - sparse_categorical_accuracy: 0.1638 - val_loss: 2.3215 - val_sparse_categorical_accuracy: 0.0998\n",
      "Epoch 2/20\n",
      "45000/45000 [==============================] - 366s 8ms/sample - loss: 2.0305 - sparse_categorical_accuracy: 0.2001 - val_loss: 2.8447 - val_sparse_categorical_accuracy: 0.1030\n",
      "Epoch 3/20\n",
      "45000/45000 [==============================] - 424s 9ms/sample - loss: 1.9391 - sparse_categorical_accuracy: 0.2265 - val_loss: 2.4241 - val_sparse_categorical_accuracy: 0.1206\n",
      "Epoch 4/20\n",
      "45000/45000 [==============================] - 415s 9ms/sample - loss: 1.8920 - sparse_categorical_accuracy: 0.2545 - val_loss: 2.1354 - val_sparse_categorical_accuracy: 0.1704\n",
      "Epoch 5/20\n",
      "45000/45000 [==============================] - 416s 9ms/sample - loss: 1.8563 - sparse_categorical_accuracy: 0.2748 - val_loss: 3.9259 - val_sparse_categorical_accuracy: 0.1178\n",
      "Epoch 6/20\n",
      "45000/45000 [==============================] - 416s 9ms/sample - loss: 1.8943 - sparse_categorical_accuracy: 0.2635 - val_loss: 2.2303 - val_sparse_categorical_accuracy: 0.1678\n",
      "Epoch 7/20\n",
      "45000/45000 [==============================] - 421s 9ms/sample - loss: 1.7787 - sparse_categorical_accuracy: 0.3129 - val_loss: 3.3465 - val_sparse_categorical_accuracy: 0.1764\n",
      "Epoch 8/20\n",
      "45000/45000 [==============================] - 417s 9ms/sample - loss: 1.7913 - sparse_categorical_accuracy: 0.3152 - val_loss: 2.0247 - val_sparse_categorical_accuracy: 0.2660\n",
      "Epoch 9/20\n",
      "45000/45000 [==============================] - 422s 9ms/sample - loss: 1.6433 - sparse_categorical_accuracy: 0.3753 - val_loss: 1.9637 - val_sparse_categorical_accuracy: 0.3016\n",
      "Epoch 10/20\n",
      "45000/45000 [==============================] - 413s 9ms/sample - loss: 1.5800 - sparse_categorical_accuracy: 0.4061 - val_loss: 2.0284 - val_sparse_categorical_accuracy: 0.2658\n",
      "Epoch 11/20\n",
      "45000/45000 [==============================] - 415s 9ms/sample - loss: 1.5731 - sparse_categorical_accuracy: 0.4040 - val_loss: 1.6217 - val_sparse_categorical_accuracy: 0.3820\n",
      "Epoch 12/20\n",
      "45000/45000 [==============================] - 416s 9ms/sample - loss: 1.4952 - sparse_categorical_accuracy: 0.4367 - val_loss: 1.6086 - val_sparse_categorical_accuracy: 0.3916\n",
      "Epoch 13/20\n",
      "45000/45000 [==============================] - 422s 9ms/sample - loss: 1.5139 - sparse_categorical_accuracy: 0.4375 - val_loss: 2.4662 - val_sparse_categorical_accuracy: 0.2162\n",
      "Epoch 14/20\n",
      "45000/45000 [==============================] - 425s 9ms/sample - loss: 1.5168 - sparse_categorical_accuracy: 0.4420 - val_loss: 1.6903 - val_sparse_categorical_accuracy: 0.4066\n",
      "Epoch 15/20\n",
      "45000/45000 [==============================] - 421s 9ms/sample - loss: 1.4852 - sparse_categorical_accuracy: 0.4572 - val_loss: 1.4609 - val_sparse_categorical_accuracy: 0.4600\n",
      "Epoch 16/20\n",
      "45000/45000 [==============================] - 421s 9ms/sample - loss: 1.4388 - sparse_categorical_accuracy: 0.4727 - val_loss: 2.1485 - val_sparse_categorical_accuracy: 0.2824\n",
      "Epoch 17/20\n",
      "45000/45000 [==============================] - 420s 9ms/sample - loss: 1.4261 - sparse_categorical_accuracy: 0.4762 - val_loss: 1.9916 - val_sparse_categorical_accuracy: 0.3000\n",
      "Epoch 18/20\n",
      "45000/45000 [==============================] - 417s 9ms/sample - loss: 1.4006 - sparse_categorical_accuracy: 0.4883 - val_loss: 1.6109 - val_sparse_categorical_accuracy: 0.4416\n",
      "Epoch 19/20\n",
      "45000/45000 [==============================] - 417s 9ms/sample - loss: 1.3531 - sparse_categorical_accuracy: 0.5078 - val_loss: 1.9634 - val_sparse_categorical_accuracy: 0.3752\n",
      "Epoch 20/20\n",
      "45000/45000 [==============================] - 417s 9ms/sample - loss: 1.3629 - sparse_categorical_accuracy: 0.5047 - val_loss: 1.9313 - val_sparse_categorical_accuracy: 0.3264\n"
     ]
    }
   ],
   "source": [
    "# For a better accuracy, increase the number of Epochs\n",
    "fit_all_batches(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_image = valid_features[3:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 32, 32, 3)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.0930685e-03, 8.0232584e-01, 1.0923524e-03, 1.1233806e-03,\n",
       "        4.8263199e-04, 6.5392966e-04, 1.8187360e-03, 3.9170854e-04,\n",
       "        2.2441681e-02, 1.6057664e-01]], dtype=float32)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict(sample_image)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.0930685e-03, 8.0232584e-01, 1.0923524e-03, 1.1233806e-03,\n",
       "       4.8263199e-04, 6.5392966e-04, 1.8187360e-03, 3.9170854e-04,\n",
       "       2.2441681e-02, 1.6057664e-01], dtype=float32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_flat = y_pred.flatten()\n",
    "y_pred_flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['airplane',\n",
       " 'automobile',\n",
       " 'bird',\n",
       " 'cat',\n",
       " 'deer',\n",
       " 'dog',\n",
       " 'frog',\n",
       " 'horse',\n",
       " 'ship',\n",
       " 'truck']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions():\n",
    "    \n",
    "    sample_image = valid_features[107:108]\n",
    "    print(sample_image.shape)\n",
    "    y_pred = model.predict(sample_image)\n",
    "    y_pred_flat = y_pred.flatten()\n",
    "    \n",
    "    results = np.column_stack((label_names, y_pred_flat))\n",
    "    print(results)\n",
    "    plt.imshow(valid_features[107])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 32, 3)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_features[107].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 32, 32, 3)\n",
      "[['airplane' '0.023745026']\n",
      " ['automobile' '0.005537284']\n",
      " ['bird' '0.05996335']\n",
      " ['cat' '0.43692192']\n",
      " ['deer' '0.00812802']\n",
      " ['dog' '0.3433302']\n",
      " ['frog' '0.02444002']\n",
      " ['horse' '0.020411942']\n",
      " ['ship' '0.0677372']\n",
      " ['truck' '0.009784946']]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAH0CAYAAADVH+85AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3WmQZFd55vHnzaUqa996V0t0S6gldpBkkCWP1kADxuzSjD4AGodhbGwGC6MJz7DYwoYwnpgwi7CFwywKC4cFIwJ5PMYsRgIBwsYIhCRrBXW3et+qa1+z8syHewtXl6q6+7yVVVk69f9FdGTXzXzznLp1M9+8mTfvYyEEAQCANBUaPQEAALB8aPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJKzU6AksBzPbKalT0q4GTwUAAK9tkoZCCNuXcicNbfRmtlXSH0l6laQ+SQck3SXpQyGE40u4685yual30+atvdGVIUSXTE1NRddI0ujYSHTN+Pi4aywp/veSJJM5x1sZpVLZWVd01c3M1KJrarUZ11hm8eu+0lJxjdXR1hZd09zsGys4HmOSND0Zv+0fHxhwjTVZjf87rzjPalzBh/NKP3cE53Och+c388xuemrC/XiZq2GN3szOkXSfpA2S/k7SY5JeLul3Jb3KzC4NIRxz3v2uTZu39r7v/f8rurA6E/+kvHv3zugaSfrhv34/uuaRf3vINVYIvieuYtGxSQffA9wcnyRtWL/BNVZPT5+rbnhkOL5mNP4FnSSVy/EvYl74/B2usS6/+OXRNTt2PM811vhk1VV3YOdPo2u+9JX/6xrrqUPxf7Oi+T4JrXmfyB11nhePeWF0SaGwsp8Mh5rjOc58z4vm+N08L0T27HxcU5Pju6IL52nkZ/R/oazJvzuE8IYQwv8IIVwl6WOSzpP0kQbODQCAJDSk0ed789co+wz9z+dd/YeSRiW91czi308EAAC/0Kg9+ivzy2+Eee8phxCGJX1fUquki1d6YgAApKRRjf68/PKJRa5/Mr/0feAIAAAkNe5gvK78cnCR62eXd5/sTszs/kWuOt8zKQAAUsMJcwAASFij9uhn99i7Frl+dvlJvwQbQrhwoeX5nv4FvqkBAJCORu3RP55fLvYZ/Ln55WKf4QMAgNPQqEZ/T355jdmJZ5kwsw5Jl0oak/TPKz0xAABS0pBGH0L4uaRvKDuP7+/Mu/pDktok3R5CGF3hqQEAkJRGnuv+t5WdAveTZna1pEclvULZd+yfkPT+Bs4NAIAkNOyo+3yv/iJJtylr8O+VdI6kT0i6eAnnuQcAALmGpteFEPZI+vXluO+x0WHd/8PvRNdNTsaH2kxNTkTXSNL63pOeJmBBXV2drrGGR3yfggRHeIP31WOpGL85Do/7fi9vnSe2qlD0JexNVONTEZ/eu8811rqe+KDHrs7FvjRzclP9vmDKUGyKrtnYF/8Yk6SdR+NDbazk2/ILzlAbT6qZN2jGM5Y3Tc6b1hbMsT7Ml2JppfjnKnPMzwr1SQDke/QAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCGhpqs5yq01M6vG9ndN3o6Fh0TXNzfNiGJKkYH6iwfesm11BDQ/EhHZI0U4sPYqjO+EIphsbHo2tq1fgQIkkqO0IpJEmO4JKinMEZFj9We0uLa6zurvjwl0qTb7tvcgZ1hGotumZDry94p7V8KLqm6t1vKjpDXGrx69GcIS5B8eu+VvM9Ns2Z41JwPJ8WHI8xSa64Hl9WD6E2AADgFGj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwpJNr5upVnX8+NHouqojIatYbIuukaTpqfg4I0+anCSVnQlZHZX4TaS51bc+1lU7o2smHIl3kjQ86qubmI5fj+Vy2TVWrRa/LXa3t7rGanY8ExTkSycr1KquulIhfpJtFV+a36au+PV4fGzSNdbYpK+uUIjfrgqOdShJVnImdK5yMzPObdgRsVcoxu9XmzfKb/7YdbkXAACwKtHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASFiyoTbT1WkdOnwgum5sbCq6ptLiC85oaq5E1wT5Qg6mpnzBGeWm+OCM1sqwa6yutvgwnPaKL2yjXPKFvwyPxodgVFV0jVWdmIiu2dDX5xqrXIyfY3XSFwxkYdpVV3Rs+hXHY0ySWorxf+daqy+8aHzM93iZmoxfj6UmX7hVW2t3/FglX3uZmYkPc5KkmiOgplz2PX94wmY8eWSE2gAAgFOi0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAlrWHqdme2S9JxFrj4UQti0lPuvzlTVPzAQXTc9E58WVI2uyLQXHavfFz6latWXGFarxSdCTU/4Us1Ghwaja0qO1DVJampu9tVV4pMKB4ZGXWO1OOb4wudud401MzUWXTM+5ksZGxw45qqbnoxfj6Wy7ymuFuIf1eWSL8WysyM+GU6ShkYcfzPnY7M4Ev846+7pdY1lRV+inCck0psOVwvxT8Q1R3ydOdNK52t0TO2gpI8vsHxkpScCAECKGt3oB0IINzd4DgAAJIvP6AEASFij9+ibzewtks6SNCrpQUn3hhDiPxgGAADP0OhGv0nS7fOW7TSzXw8hfOdUxWZ2/yJXnb/kmQEAkIBGvnX/eUlXK2v2bZJeJOkvJW2T9I9m9pLGTQ0AgDQ0bI8+hPCheYselvRbZjYi6b2Sbpb0xlPcx4ULLc/39C+owzQBAHhWW40H4306v7ysobMAACABq7HRH8kv2xo6CwAAErAaG/3F+eVTDZ0FAAAJaEijN7Pnmdkz9tjNbJukT+U/fmEl5wQAQIoadTDef5b0XjO7V9JuScOSzpH0GkkVSV+V9L8bNDcAAJLRqEZ/j6TzJL1M0qXKPo8fkPQ9Zd+rvz0ER2oAAAA4QUMafX4ynFOeEGdpY0jTjuS1YrkSXdPS5jtu0JOQFWZ8Jw3sbPcla9Vm4lPvxkZ9CVnTFv9J0vi4b6yZwfikPEkqlcvRNa3Odb+pLz79q7nkS5SbmozPkSoUnK/FHduU5Esq9G4fXd1d0TX7jwy7xjLzJTC2t7VH11Rn4hM9Jan/yMHomkLBl7zW3bfFVeeK9nTuT1rB8am34/mt4Blnofupy70AAIBViUYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwhqVXrf8TJLFhyr09sQHifQ6AjAkaWQkPlhlYDA+fESS+no6XXXFWnyIS1MxvkaSCsX4zbGpqdk3ljNwo2jxIRibNqxzjVVwbL979u9zjbX1OVuja4IjNEqS2lvjg6Mk6cknjkTX7NmzxzVWhyOoanTnftdY1uR7bAbH9tHkCGWSpI621uia2qQvUKjgCPuSpKbWjvgi39OACo6AmjAT/9xhjnEWwh49AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJSza9rlgoqt2RZtTeEp/StKHXl15XKsSnGQ0ND7vGmpmuueoqlZbomvXrN7rGMkca1+Ytm11jDQ8Pueo0NRFd0tnmS2ubnIwf6/BBX3rdk48/Hl2zrm+9a6xazbctHj58KLpm/8H4GkkqNxejaybGR11jdbT2uOpKjuTG6ekp11jN5fg0v/YWX7JkbWrMVVdqj08B7Kj45ths8Ql7lUr8fvWPS854vXnYowcAIGE0egAAEkajBwAgYTR6AAASRqMHACBhNHoAABJGowcAIGE0egAAEkajBwAgYTR6AAASRqMHACBhNHoAABKWcKhNQZ2d8SEHk5OT0TXlcjm6RpKaS/F1zztnh2usloovWKU2MxNds2OHb46eUJug+GAgSRoYOO6qM0cgy/Hj/a6xqtX44IxiMT6MRZL6+49F14yPxYfuSNLEhK9uaDQ+0Gl4wheQMjYwHl3T0e4Lt2pxBOhI0rTiH5tFR5CWJKk6HV1SCL79yNayry2tb4kP7Dlrky9QqKOjPbqmu68vuuYbd39Tx3xPHydgjx4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITVJb3OzK6VdLmkl0p6iaQOSX8TQnjLSWoukfQBSRdLapH0pKTPSbolhBAfyzRPkC/9a3oqfmgr+NKnNm3YFF2zZcsZrrGmp+PTpyRpeDg+May1tc01VlNTU3TN/gP7XWONjY646iql+L91qeB7Pd3W1hJdUyjEJwBKvsfKzEx8jSQNDAy66iYcyZKtba2usaZm4lMKqyG+RpL6jx9x1RVK8U/f1an4dShJ3e3x6/HMzRtcY63r6XbVbVi3MbqmvXuda6zmrvjfrccxv2LZlzo6X71iaj+grMGPSNor6fyT3djMXi/py5ImJH1RUr+k10r6mKRLJV1Xp3kBALCm1eut+/dI2iGpU9I7T3ZDM+uU9FeSZiRdEUL4jRDCf1f2bsAPJF1rZtfXaV4AAKxpdWn0IYR7QghPhhDCadz8WknrJd0RQvjRnPuYUPbOgHSKFwsAAOD0NOJgvKvyy68tcN29ksYkXWJmzSs3JQAA0lSvz+hjnJdfPjH/ihBC1cx2SnqBpLMlPXqyOzKz+xe56qTHCAAAsFY0Yo++K79c7NDb2eW+Qy8BAMAvNGKPvm5CCBcutDzf079ghacDAMCq04g9+tk99q5Frp9dPrACcwEAIGmNaPSP55c75l9hZiVJ2yVVJT21kpMCACBFjWj0d+eXr1rgussktUq6L4TgO4UTAAD4hUY0+jslHZV0vZldNLvQzCqSPpz/eGsD5gUAQHLqda77N0h6Q/7j7Ancf9nMbsv/fzSEcJMkhRCGzOwdyhr+t83sDmWnwH2dsq/e3anstLgAAGCJ6nXU/Usl3TBv2dn5P0naLemm2StCCHeZ2eWS3i/pzZIqkn4m6fckffI0z7AHAABOoS6NPoRws6SbI2u+L+lX6zH+IgMo1OJfL4xPOA4NCL7EsE2bNkfXtLd3uMY6fvy4q66np9dR5Vsfnpd33iS04bEpV92gI6GsOjnqGuucM7ZE1xQLvod0sRhfNzLiSwA8etSX1jbhSF5rqcQnAErS4GD871ar+UI3B/r7XXV9PT3RNS1F32Pz7C3xz1Ub1q13jdXd66vr7YtPh+t0pte1dsc/L1Y6OqNrikVfMup85NEDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJq1d63apTLBbV1dUVXTcxOR1dU6vFB51I0tatZ0TXDA4Ou8Y6evSoq+6cc84+9Y3mmZ6uusYaHXWEvzhzDs1ZODgwEF0zNuYLtVnf3R1dc87Zz3GNNTY+Fl0zOeELBpqa8tV5to/Jmm9bPHvbWdE1hw4fdo3VvNEXrCJHaJcrOUpSv2O7X9fr+73aO9pddS0trdE1GzfGh/VIUnNbfG8pNTdH1xQLhNoAAIBToNEDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJSza9rlwu64yN66Pr9j79dHTNUWdqVVM5fvUPDva7xjp4YJ+r7rxzt0fXVOVLJxsaik/Imp7xpZM950xfatWLzotP8+vp7HSNNTUWnyi39YxtrrEe+9kT0TVW9CVrTc/40h43bor/mx065ktt3LJpY3TNsSNHXGOVyvGpZpI06Hi8lMtl11hWjN8n7Ojxbffr+npddU3NLdE1XT3xCZGS1NQU/zfzPFd5UzbnY48eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIWLKhNiapVIh/HRMcwQO1mZnoGkk6sH9/dM3BA/E1ktTa4gvOKJXi1+HI0UHXWJ6QjqGRcddYba3xgUeSdN6O86NrRgaPu8bacc450TUDg6OusbZs2RZdc/iwb1sslnxPO8/Zti26Zufeva6xjhyOD6ipNFVcY7U5Q4+aHY/piiOMRZI2b4wP+TnrzDNdY7W1tbrqOjp7omsmJuKDoyQpKP45/0cPPRxdMzbum9987NEDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJCwuqTXmdm1ki6X9FJJL5HUIelvQghvWeC22yTtPMndfTGEcP1S51Sr1TQ+Hp9sZmbxY4VadI0kHT8en2o2NDTkGqurs8tVt9+RsDc9Pekaa3AwPr1ubMKXHNh/LH4sSWpri08aO3xgn2+s3vi/2bGho66x1m/YHF0zPORL5du27TmuuuGh4eia4Hxsjo7FpwA2NTW5xuro6HDVVWvxSZtdzqS87u7u6Jpyuewaq7XVl17X5Ejma2r2/c1+8tMHo2u++6N/i64ZGfWlc85Xr5jaDyhr8COS9ko6nSzPn0q6a4Hl8Vl+AABgQfVq9O9R1uB/pmzP/p7TqHkghHBzncYHAAALqEujDyH8orF73voGAADLo1579B5bzOw3JfVJOibpByGE+A8+AADAohrZ6F+Z//sFM/u2pBtCCE+fzh2Y2f2LXHU6xwgAAJC8Rny9bkzSH0u6UFJP/m/2c/0rJH3LzNoaMC8AAJKz4nv0IYTDkv5g3uJ7zewaSd+T9ApJb5f0idO4rwsXWp7v6V+wxKkCAPCst2pOmBNCqEr6TP7jZY2cCwAAqVg1jT53JL/krXsAAOpgtTX6i/PLpxo6CwAAErHijd7MLjCzZ4xrZlcrO/GOJH1hZWcFAECa6nWu+zdIekP+46b88pfN7Lb8/0dDCDfl//8zSeea2X3KzqYnSS+WdFX+/w+GEO6rx7wAAFjr6nXU/Usl3TBv2dn5P0naLWm20d8u6Y2SfknSqyWVJR2S9CVJnwohfLdOc5IpPtCivT3+8ICaI1xCko71xweQjI6OucZqa/Md9vDgQ/HRA2ds2eIaq6sjPnDDwohrrKai9wyOIbrCG9IxeDQ+NKaryxeQMjpyLLpm+5lnuMbavXePq+7RJ5+IrmmtVFxjjY1PRNeUy76xDh+LX/eSVGmOD42ZmvQFTtVq8eFRx5y/V09Pr6uuPB0fADNy2Pf8cd+PfxJdM23xzwNB9TnTbL1OgXuzpJtP87aflfTZeowLAABObrUdjAcAAOqIRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDC6pVet0rFJ42Vy45VYvHjSNLYWHwSXaXS7BprZiY+fUqS+vrWRde0O1LoJF/KW7Fw0DWWOVOhRkdHo2sK5htrZGgouua55253jfXAA/FpXOvP2eAaq7evx1V31vjW6JoN6/pcY+3fty+6ZmzSl2K5ceNGV93QYHy6Yanke8pvaopPyhsejt9+JWlwcNBVF2rxaaWP/tyXpDhTbo+u6WiJrykUi9E1C95PXe4FAACsSjR6AAASRqMHACBhNHoAABJGowcAIGE0egAAEkajBwAgYTR6AAASRqMHACBhNHoAABJGowcAIGE0egAAEkajBwAgYQmn1wXNVOMT2wYH4xOXzJlet64nPlnLm3TV3tbmqmt11JXL8UlXknR8YCC6plbzrfupyQlX3Z698WlXWzb4EtRGx8eja1qc637rhvgkOk/6oiRt2+ZL2JuanI6uqU7H10jSpg3xj7O9B4+4xhoeHXbVyZGK2NPrSw4sleJT1CYmnI+xPb5EuWJTU3TNkwd823Bba3d0Tc2xW+0MvnwG9ugBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICEJRtqUygU1dHWEV233hHusa7PF1rS3R0fMDEzEx/UI0nVatVVNzM1FV2zZ+9e11iHjhyNrulqa3eNVSr7Nv2n98X/bt29Xa6xOrvigzNGR0ddY9UciRudXb6gpFrVFzRTDfEJH8XmVtdYXe3xdVO1mmusgw8ddNXtOO/86JrR4fjgKEkaG4/friYm4587JGlgeMRVNzgSH1DTuiF+HUpSsRj/eAnm2z7qgT16AAASRqMHACBhNHoAABJGowcAIGE0egAAEkajBwAgYTR6AAASRqMHACBhNHoAABJGowcAIGE0egAAEkajBwAgYTR6AAAStuT0OjPrk/RGSa+R9CJJZ0iakvSQpM9L+nwI4RmxPWZ2iaQPSLpYUoukJyV9TtItIQRfRNscxWJB7V2d0XUvffFLomu6OuLHkaTxsfhEqMnJCddYwXyv6YZGx6Nrpqd9f76Zanzd2JRvfXQ1+VLvJh2JXKVi0TXW5o290TVPH4xPAJSku//pO9E1F77s+a6xNm/d6qqzZz6NnFKxEJ94J0nr18cnSxbMl8rX0/MfXHXdvfFJm7t3/sw1VlHx6ZdjY4ddY3mTNs3i/9blUtk3ViH++dQReOf6nRZSj5ja6yTdKumApHskPS1po6Q3SfqMpFeb2XUhhDBbYGavl/RlSROSviipX9JrJX1M0qX5fQIAgCWqR6N/QtLrJP3D3D13M3ufpB9KerOypv/lfHmnpL+SNCPpihDCj/LlH5R0t6Rrzez6EMIddZgbAABr2pI/ow8h3B1C+Pv5b8+HEA5K+nT+4xVzrrpW0npJd8w2+fz2E8reypekdy51XgAAYPkPxpv90Gruhy5X5ZdfW+D290oak3SJmTUv58QAAFgL6vHW/YLMrCTpbfmPc5v6efnlE/NrQghVM9sp6QWSzpb06CnGuH+Rq86Pmy0AAGlazj36j0p6oaSvhhC+Pmd5V345uEjd7PLu5ZoYAABrxbLs0ZvZuyW9V9Jjkt66HGNIUgjhwkXGv1/SBcs1LgAAzxZ136M3s3dJ+oSkRyRdGULon3eT2T32Li1sdvlAvecGAMBaU9dGb2Y3SrpF0sPKmvzBBW72eH65Y4H6kqTtyg7ee6qecwMAYC2qW6M3s99XdsKbB5Q1+cVOi3R3fvmqBa67TFKrpPtCCJP1mhsAAGtVXRp9frKbj0q6X9LVIYSTnYfzTklHJV1vZhfNuY+KpA/nP95aj3kBALDW1eNc9zdI+iNlZ7r7rqR3L3B+3l0hhNskKYQwZGbvUNbwv21mdyg7Be7rlH317k5lp8UFAABLVI+j7rfnl0VJNy5ym+9Ium32hxDCXWZ2uaT3KztFbkXSzyT9nqRPzj0vvtd0taojx45E102Mx4eWaMOm+BpJW7bE183UfIEx5eYWV93YRHxQR7mp4horPrJEGhjyHbM5MjLiqmtvbY2uCdO+sJOhYwsd4nJyu57a7RrLs/YHj88/zvb0dPXEB8ZIUktTfABJT7cvvGhzb190zeG9e1xjtTgfmxPjY9E1vb3xQUmSdOjA/uiaiUnfc9XUjC/UplB0BNQ4Q2N8oTaNC4tdcqMPIdws6WZH3fcl/epSxwcAAIsjjx4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEhYPdLrVqVioaD2SnyK2vjIcHTNxKgvQa2k+CSp5ibfa7OZmfikK0nq7miLrmltbXaNNTw+Gl1TKBZdY2nakVIoqbMtPmmso+J7mPX3H4+u2ehMJ+s/ejS6prOjyTVWe9mXGDY6E5+wt3/fXtdYk8Px676r4lsfo1VfWlt7uyMlstWXLPnIo49F1+zce8A1VqXie/5o7eiOruluiU+jlKRiKT4pr+bIZDWrz744e/QAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACQs2fS6SnOznvfcs6Pr1nXEpxmNT4xH10jS4cMHo2tanWlLY2Mjrrqedeuja4qVdtdYRcWnk01PTbrG6ulZ56rTjGO8GUdslSRZfELW+Eh8AqAkTY0NRteUan2usUo13+NlaCA+JfIb9/6La6zXvPLS6JoXv/hi11j7j/qSJUMxfvsYcKxDSVq/Lv7x8uSuPa6xhgaHXHWVns3RNc3O51OF+Me0L70uvmYh7NEDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJSzbUxkwql+J/vS0b44MRxqd9wSr9A/FBM8MjE66xavF5MZKkYiF+HXa2+UJtWpvjAzd29h93jTU1NeOqa2kuRtc8ufuAa6xtmzuja9oqvhSM5ua26JqpWnyoiiTtO3DYVbdrT/x67Onb4Bqr0Bof2PPgE/tcYw2NOkN+hh1BRKUm11jFpubomqYm31ihNu2qa+mMD95pcvxeklSdnnJUeR6b9Um1YY8eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICELTm9zsz6JL1R0mskvUjSGZKmJD0k6fOSPh9CqM25/TZJO09yl18MIVy/9HkVVGpqia5r6XCkNM34ouH2HX4iuqYafKlrfX3xyU6SNDw6Fl3T1ed7/dje0RFd01LxpU8VVXXVlcrxiVy7D/rS684/Oz5J8dwEbTC5AAASaklEQVTnvsQ1VknxSXTTE54EL+mx3ftddZ4krwtfdJ5rpKbm+PXR1t3jGmvafClv5Ur846VU9iUOPvDwQ9E1+w8ddI21fsMmV11P38bomuay7/mjVIhPsQzF+OfFQqE+++L1iKm9TtKtkg5IukfS05I2SnqTpM9IerWZXRdCCPPqfirprgXu7+E6zAkAAKg+jf4JSa+T9A/z9tzfJ+mHkt6srOl/eV7dAyGEm+swPgAAWMSS3xcIIdwdQvj7uU0+X35Q0qfzH69Y6jgAACBePfboT2Y6v1zoA9EtZvabkvokHZP0gxDCg8s8HwAA1pRla/RmVpL0tvzHry1wk1fm/+bWfFvSDSGEp09zjPsXuer805wmAABJW86v131U0gslfTWE8PU5y8ck/bGkCyX15P8uV3Yg3xWSvmVmbcs4LwAA1oxl2aM3s3dLeq+kxyS9de51IYTDkv5gXsm9ZnaNpO9JeoWkt0v6xKnGCSFcuMj490u6IH7mAACkpe579Gb2LmVN+hFJV4YQ+k+nLoRQVfZ1PEm6rN7zAgBgLaprozezGyXdouy78FfmR97HOJJf8tY9AAB1ULdGb2a/L+ljkh5Q1uQPO+7m4vzyqXrNCwCAtawujd7MPqjs4Lv7JV0dQjh6ktteYGbPGNfMrpb0nvzHL9RjXgAArHX1ONf9DZL+SNKMpO9KerfZM85JvSuEcFv+/z+TdK6Z3Sdpb77sxZKuyv//wRDCfUudFwAAqM9R99vzy6KkGxe5zXck3Zb//3ZlITi/JOnVksqSDkn6kqRPhRC+W4c5qVAoqrOjM7rOivFhBQP9g9E1kjQxHR9Q48zP0eikL4BkYmwiuqZtYNg1VqVSia7p6Y4P9pCkvp74bUOSJifj/2YHj5zW8ajPcGBkfjzEqZ1d8QWrtLXFr4/Bob2nvtECztq8wVVXbop/bHa2xwdbSZKq8dv9vn37XEPt2b/oG6An1ewIqDl46JBrrANH4uvaHM+/ktS17ixX3bq++O2qUIgPSpKkajX+sdncHB9eZM75zbfkRp+fr/7miNt/VtJnlzouAAA4NfLoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNHoAQBIGI0eAICE0egBAEgYjR4AgITR6AEASBiNHgCAhNUjvW5VKhQKamlrj64rNcUnqD2192B0jSTV4gOQVKnE/06SNDQ07qobHh6IrimV41OaJGnDho3RNU1Nza6xiqX45C9Jmhgci67p7PSleA1Pxqe1jVbjt19Jmu7dGl2zYdOZrrG2rvMlDv7bQz+JrukfOO4aq9zaFl2z96gvtXFoJH6bkiSbmY6u+fmuna6xxqrxNWee/TzXWDue/1JXXakU387GJ3zrfmoqfoWUKo4kRUePWAh79AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJCzh9Lqimlvjk94OH4tPaxsd9SXDlcvxCWpW8P3JBodGXHUTExPRNeUmX3rd4GB8+ldHZ49rrDHH7yVJvevXx4817htrz+490TWTL/dtH+c876LomuNH9rnG2nVor6vuqONxNjTmSydrtfgUwI4237ZYrfn+ZkPDQ9E1Hb2bXWNt7jsjumb7uc93jdXV7VuP09PxaX5mvn3dSiV++ygW4tMoJXPUPBN79AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMKSDbWZmJrSkzt3R9c98vjPo2uOHj0SXSNJ7e1t0TVBvgCdkdFRV93GDX3RNd5Qm/6jx6NrNm3a4BprctK3HqvVmeiavt74dShJg3sORNf87Rc/5xrrogtfEV0zMekL63n0sYdcdZOT8QE1vT29rrF6SvH7QObJLJE0XYwP35Kk1r748JcXbPcFxnT0rIuuqbS1usby7n2WSo52FoJrrKJjrIKjxuqTacMePQAAKaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJAwGj0AAAmj0QMAkDAaPQAACatLep2Z/amkiyTtkLRO0rik3ZLukvSpEMKxBWoukfQBSRdLapH0pKTPSbolhBAfETZP//Hj+sKX/k90XVO5Obqm0hxfI0mFYvzrrIkJX+qaFX3RWpOT8X+Ko0f6XWOVS+Xomr174xPeJKmzMz45UJJGx6eia0qladdY6xzJaw8+/oRrrK9+8x+jazZu2eYaq7nDlzi4Zfv66JqOtg7XWAXH48X7pNXpTHtsaalE1zQ5xzLHc1Wx7GsvtWnnmnQkvZkzHs6zHkMpfpuyQn3i6+q1R/8eSW2SvinpE5L+RlJV0s2SHjSzM+fe2MxeL+leSZdJ+oqkT0lqkvQxSXfUaU4AAKx59cqj7wwhPCOc2sw+Iul9kv6npN/Ol3VK+itlL4CvCCH8KF/+QUl3S7rWzK4PIdDwAQBYorrs0S/U5HNfyi/PnbPsWknrJd0x2+Tn3McH8h/fWY95AQCw1i33wXivzS8fnLPsqvzyawvc/l5JY5IuMTPfB98AAOAX6vXWvSTJzG6S1C6pS9nBeb+irMl/dM7Nzssvn3HUUAihamY7Jb1A0tmSHj3FePcvctX5cTMHACBNdW30km6StHHOz1+T9F9CCEfmLOvKLwcXuY/Z5d11nhsAAGtOXRt9CGGTJJnZRkmXKNuT/4mZ/VoI4cf1HCsf78KFlud7+hfUezwAAJ5tluUz+hDCoRDCVyRdI6lP0l/PuXp2j73rGYUnLh9YjrkBALCWLOvBeCGE3ZIekfQCM1uXL348v9wx//ZmVpK0Xdl38J9azrkBALAWrMQpcLfkl7OnO7o7v3zVAre9TFKrpPtCCJPLPTEAAFK35EZvZjvM7Blvw5tZIT9hzgZljft4ftWdko5Kut7MLppz+4qkD+c/3rrUeQEAgPocjPerkv7EzL4naaekY8qOvL9c2VfkDkp6x+yNQwhDZvYOZQ3/22Z2h6R+Sa9T9tW7OyV9sQ7zAgBgzatHo/8nSc9V9p35lyn7Wtyosu/J3y7pkyGEE1JOQgh3mdnlkt4v6c2SKpJ+Jun38tuHpU7KTCqX4t+w6O1Z7BjBxW09c2t0jSRNTMR/OnHg4EHXWC3N8YExklStxZ+3KFh82IYkjToCewaHRl1jNbX4Qm2sqTW6Zjz4zv3U2tEeXfPKX3uZa6ya4yFXafL9nb3BKnIEkHjCWCRJNcdTkPNpyxxhJ5Ikx+/mfmqdqTlKfGFOtVr8WJJUKMSvj0KzswV6wnBqjrCeJXfCzJIbfQjhYUnvctR9X9m7AQAAYJmQRw8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCrA75MauOmR0zs97m5vgwkVIx/vT/ZWdIR3CEN0xMxgfhSFLBE8IgqeQI3Cg6wiUkXybIzIwjKEJSuewL+ZHF/27mXB+FQvy6Lzl/L8+zQMGxLiTJnNuibzBn3Uo+La7g+ljBNf/ssJLbosOhA3s1PT3VH0LoW8r91CO9bjUaCiFoYmJi1wLXnZ9fPla30XwBaqtF/dfHsxvr40SsjxOxPk7E+jhRvdfHNklDS72TJPfoT8bM7pekEMKFjZ7LasD6OBHr40SsjxOxPk7E+jjRal0ffEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwtbcUfcAAKwl7NEDAJAwGj0AAAmj0QMAkDAaPQAACaPRAwCQMBo9AAAJo9EDAJCwNdPozWyrmX3OzPab2aSZ7TKzj5tZT6PnttLy3z0s8u9go+e3HMzsWjO7xcy+a2ZD+e/6hVPUXGJmXzWzfjMbN7MHzexGM4sPil9lYtaHmW07yfYSzOyOlZ5/PZlZn5m93cy+YmY/y//Wg2b2PTP7DTNb8Hky1e0jdn2kvn1Ikpn9qZl9y8z25Ouj38x+YmZ/aGYLZsWvpu0j1Tz6E5jZOZLuk7RB0t8pywp+uaTflfQqM7s0hHCsgVNshEFJH19g+chKT2SFfEDSS5T9fnv177nRCzKz10v6sqQJSV+U1C/ptZI+JulSSdct52RXQNT6yP1U0l0LLH+4jvNqhOsk3SrpgKR7JD0taaOkN0n6jKRXm9l1Yc7ZxRLfPqLXRy7V7UOS3iPpx5K+KemwpDZJF0u6WdJ/NbOLQwh7Zm+86raPEELy/yR9XVKQ9N/mLf+zfPmnGz3HFV4fuyTtavQ8Vvh3vlLSuZJM0hX53/0Li9y2U9mDeVLSRXOWV5S9YAySrm/077SC62Nbfv1tjZ73Mq2Lq5Q9CRfmLd+krMkFSW9eK9uHY30kvX3M/m0XWf6R/Hf/i9W8fST/1n2+N3+Nsub25/Ou/kNJo5LeamZtKzw1rKAQwj0hhCdD/og7hWslrZd0RwjhR3PuY0LZnrAkvXMZprliItdH0kIId4cQ/j6EUJu3/KCkT+c/XjHnqqS3D8f6SF7+t13Il/LLc+csW3Xbx1p46/7K/PIbC2y4w2b2fWUvBC6W9K2VnlwDNZvZWySdpezFzoOS7g0hzDR2WqvCVfnl1xa47l5JY5IuMbPmEMLkyk2r4baY2W9K6pN0TNIPQggPNnhOy206v6zOWbaWt4+F1sestbh9vDa/nPt7rrrtYy00+vPyyycWuf5JZY1+h9ZWo98k6fZ5y3aa2a+HEL7TiAmtIotuMyGEqpntlPQCSWdLenQlJ9Zgr8z//YKZfVvSDSGEpxsyo2VkZiVJb8t/nPukvSa3j5Osj1nJbx9mdpOkdkldki6S9CvKmvxH59xs1W0fyb91r+wPImUHny1kdnn3Csxltfi8pKuVNfs2SS+S9JfKPmv7RzN7SeOmtiqwzZxoTNIfS7pQUk/+73JlB2pdIelbiX709VFJL5T01RDC1+csX6vbx2LrYy1tHzcp+8j3RmVN/muSrgkhHJlzm1W3fayFRo95Qggfyj+HOxRCGAshPBxC+C1lBye2KDuSFJAkhRAOhxD+IITw4xDCQP7vXmXvhP2LpOdKentjZ1lfZvZuSe9V9g2dtzZ4Og13svWxlraPEMKmEIIp20l6k7K98p+Y2QWNndnJrYVGP/vqqWuR62eXD6zAXFa72QNtLmvoLBqPbeY0hBCqyr5uJSW0zZjZuyR9QtIjkq4MIfTPu8ma2j5OY30sKNXtQ5LynaSvKHsx0yfpr+dcveq2j7XQ6B/PL3cscv3s0ZKLfYa/lsy+/ZTK22xei24z+eeU25UdjPTUSk5qlUpqmzGzGyXdouy731fmR5rPt2a2j9NcHyeT1PYxXwhht7IXQC8ws3X54lW3fayFRn9PfnnNAmd06lB28oIxSf+80hNbhS7OL5/1T1BLdHd++aoFrrtMUquk+xI8otojmW3GzH5f2QlNHlDW1A4vctM1sX1ErI+TSWb7OIkt+eXsN5ZW3faRfKMPIfxc0jeUHWj2O/Ou/pCyV5q3hxBGV3hqDWFmz1vowBgz2ybpU/mPJz017Bpwp6Sjkq43s4tmF5pZRdKH8x9vbcTEGsHMLljoNLBmdrWyM4ZJz/Jtxsw+qOxgs/slXR1COHqSmye/fcSsj9S3DzPbYWbPeBvezApm9hFlZ1y9L4RwPL9q1W0fthbOl7HAKXAflfQKZd+xf0LSJWGNnALXzG5WdlDNvZJ2SxqWdI6k1yg7c9NXJb0xhDDVqDkuBzN7g6Q35D9ukvQfle1lfDdfdjSEcNO829+p7BSWdyg7heXrlH115k5J/+nZfLKZmPWRf0XqXGWPob359S/Wv39f+IMhhNknsGcdM7tB0m3K9shu0cJHS+8KIdw2pybZ7SN2fayB7eNGSX8i6XuSdio7R8BGZd8sOFvSQWUvhh6ZU7O6to+VPA1fI/9JOlPZ18oOSJpS1uQ+Lqmn0XNb4fVwuaS/VXb07ICyE2AcUXYO57cpf/GX2j9l3yQIJ/m3a4GaS5W98DkuaVzSQ8r2UIqN/n1Wcn1I+g1J/0/Z2SVHlJ3a82ll5/D+D43+XVZgXQRJ314r20fs+lgD28cLlb3b+YCyPfWqshc//5qvq95F6lbN9rEm9ugBAFirkv+MHgCAtYxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACSMRg8AQMJo9AAAJIxGDwBAwmj0AAAkjEYPAEDCaPQAACTs/wO3+3lxG1uFQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 253
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nottrainableLayers(model):\n",
    "    \n",
    "        for layer in model.layers:\n",
    "            layer.trainable = False  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "nottrainableLayers(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_1 False\n",
      "Conv1_pad False\n",
      "Conv1 False\n",
      "bn_Conv1 False\n",
      "Conv1_relu False\n",
      "expanded_conv_depthwise False\n",
      "expanded_conv_depthwise_BN False\n",
      "expanded_conv_depthwise_relu False\n",
      "expanded_conv_project False\n",
      "expanded_conv_project_BN False\n",
      "block_1_expand False\n",
      "block_1_expand_BN False\n",
      "block_1_expand_relu False\n",
      "block_1_pad False\n",
      "block_1_depthwise False\n",
      "block_1_depthwise_BN False\n",
      "block_1_depthwise_relu False\n",
      "block_1_project False\n",
      "block_1_project_BN False\n",
      "block_2_expand False\n",
      "block_2_expand_BN False\n",
      "block_2_expand_relu False\n",
      "block_2_depthwise False\n",
      "block_2_depthwise_BN False\n",
      "block_2_depthwise_relu False\n",
      "block_2_project False\n",
      "block_2_project_BN False\n",
      "block_2_add False\n",
      "block_3_expand False\n",
      "block_3_expand_BN False\n",
      "block_3_expand_relu False\n",
      "block_3_pad False\n",
      "block_3_depthwise False\n",
      "block_3_depthwise_BN False\n",
      "block_3_depthwise_relu False\n",
      "block_3_project False\n",
      "block_3_project_BN False\n",
      "block_4_expand False\n",
      "block_4_expand_BN False\n",
      "block_4_expand_relu False\n",
      "block_4_depthwise False\n",
      "block_4_depthwise_BN False\n",
      "block_4_depthwise_relu False\n",
      "block_4_project False\n",
      "block_4_project_BN False\n",
      "block_4_add False\n",
      "block_5_expand False\n",
      "block_5_expand_BN False\n",
      "block_5_expand_relu False\n",
      "block_5_depthwise False\n",
      "block_5_depthwise_BN False\n",
      "block_5_depthwise_relu False\n",
      "block_5_project False\n",
      "block_5_project_BN False\n",
      "block_5_add False\n",
      "block_6_expand False\n",
      "block_6_expand_BN False\n",
      "block_6_expand_relu False\n",
      "block_6_pad False\n",
      "block_6_depthwise False\n",
      "block_6_depthwise_BN False\n",
      "block_6_depthwise_relu False\n",
      "block_6_project False\n",
      "block_6_project_BN False\n",
      "block_7_expand False\n",
      "block_7_expand_BN False\n",
      "block_7_expand_relu False\n",
      "block_7_depthwise False\n",
      "block_7_depthwise_BN False\n",
      "block_7_depthwise_relu False\n",
      "block_7_project False\n",
      "block_7_project_BN False\n",
      "block_7_add False\n",
      "block_8_expand False\n",
      "block_8_expand_BN False\n",
      "block_8_expand_relu False\n",
      "block_8_depthwise False\n",
      "block_8_depthwise_BN False\n",
      "block_8_depthwise_relu False\n",
      "block_8_project False\n",
      "block_8_project_BN False\n",
      "block_8_add False\n",
      "block_9_expand False\n",
      "block_9_expand_BN False\n",
      "block_9_expand_relu False\n",
      "block_9_depthwise False\n",
      "block_9_depthwise_BN False\n",
      "block_9_depthwise_relu False\n",
      "block_9_project False\n",
      "block_9_project_BN False\n",
      "block_9_add False\n",
      "block_10_expand False\n",
      "block_10_expand_BN False\n",
      "block_10_expand_relu False\n",
      "block_10_depthwise False\n",
      "block_10_depthwise_BN False\n",
      "block_10_depthwise_relu False\n",
      "block_10_project False\n",
      "block_10_project_BN False\n",
      "block_11_expand False\n",
      "block_11_expand_BN False\n",
      "block_11_expand_relu False\n",
      "block_11_depthwise False\n",
      "block_11_depthwise_BN False\n",
      "block_11_depthwise_relu False\n",
      "block_11_project False\n",
      "block_11_project_BN False\n",
      "block_11_add False\n",
      "block_12_expand False\n",
      "block_12_expand_BN False\n",
      "block_12_expand_relu False\n",
      "block_12_depthwise False\n",
      "block_12_depthwise_BN False\n",
      "block_12_depthwise_relu False\n",
      "block_12_project False\n",
      "block_12_project_BN False\n",
      "block_12_add False\n",
      "block_13_expand False\n",
      "block_13_expand_BN False\n",
      "block_13_expand_relu False\n",
      "block_13_pad False\n",
      "block_13_depthwise False\n",
      "block_13_depthwise_BN False\n",
      "block_13_depthwise_relu False\n",
      "block_13_project False\n",
      "block_13_project_BN False\n",
      "block_14_expand False\n",
      "block_14_expand_BN False\n",
      "block_14_expand_relu False\n",
      "block_14_depthwise False\n",
      "block_14_depthwise_BN False\n",
      "block_14_depthwise_relu False\n",
      "block_14_project False\n",
      "block_14_project_BN False\n",
      "block_14_add False\n",
      "block_15_expand False\n",
      "block_15_expand_BN False\n",
      "block_15_expand_relu False\n",
      "block_15_depthwise False\n",
      "block_15_depthwise_BN False\n",
      "block_15_depthwise_relu False\n",
      "block_15_project False\n",
      "block_15_project_BN False\n",
      "block_15_add False\n",
      "block_16_expand False\n",
      "block_16_expand_BN False\n",
      "block_16_expand_relu False\n",
      "block_16_depthwise False\n",
      "block_16_depthwise_BN False\n",
      "block_16_depthwise_relu False\n",
      "block_16_project False\n",
      "block_16_project_BN False\n",
      "Conv_1 False\n",
      "Conv_1_bn False\n",
      "out_relu False\n",
      "global_average_pooling2d False\n",
      "Logits False\n",
      "dense False\n",
      "dropout False\n",
      "dense_1 False\n"
     ]
    }
   ],
   "source": [
    "for l in model.layers:\n",
    "    print(l.name, l.trainable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras_file = \"models/keras_CIFAR10_MobileNetV2_sparse_model/keras_CIFAR10_MobileNetV2_sparse.h5\"\n",
    "model.save(keras_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n",
      "WARNING:tensorflow:From /Users/nicolasmaquaire/Dropbox/KLoop/Dev/190209_MLEnvs/env/lib/python3.7/site-packages/tensorflow/lite/python/lite.py:591: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.compat.v1.graph_util.convert_variables_to_constants\n",
      "WARNING:tensorflow:From /Users/nicolasmaquaire/Dropbox/KLoop/Dev/190209_MLEnvs/env/lib/python3.7/site-packages/tensorflow/python/framework/graph_util_impl.py:245: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.compat.v1.graph_util.extract_sub_graph\n",
      "INFO:tensorflow:Froze 266 variables.\n",
      "INFO:tensorflow:Converted 266 variables to const ops.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3834984"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert to TensorFlow Lite model.\n",
    "converter = tf.lite.TFLiteConverter.from_keras_model_file(keras_file)\n",
    "converter.post_training_quantize=True\n",
    "tflite_model = converter.convert()\n",
    "open(\"models/keras_CIFAR10_MobileNetV2_sparse_model/keras_CIFAR10_MobileNetV2_sparse.tflite\", \"wb\").write(tflite_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}