4.0-seq2seq-fashion-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/timsainb/tensorflow2-generative-models/blob/master/4.0-seq2seq-fashion-mnist.ipynb)\n",
    "\n",
    "## Seq2Seq Autoencoder (without attention)  \n",
    "\n",
    "Seq2Seq models use recurrent neural network cells (like LSTMs) to better capture sequential organization in data. This implementation uses Convolutional Layers as input to the LSTM cells, and a single Bidirectional LSTM layer. \n",
    "\n",
    "![a seq2seq bidirectional lstm in tensorflow 2.0](imgs/seq2seq.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note: We're treating fashion MNIST like a sequence (on it's x-axis) here. To see the same network on a more-sequential dataset - see the nsynth example! In addition - our base clase is the same autoencoder as 0.0, because we want a one dimensional z-space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Install packages if in colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### install necessary packages if in colab\n",
    "def run_subprocess_command(cmd):\n",
    "    process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)\n",
    "    for line in process.stdout:\n",
    "        print(line.decode().strip())\n",
    "\n",
    "\n",
    "import sys, subprocess\n",
    "\n",
    "IN_COLAB = \"google.colab\" in sys.modules\n",
    "colab_requirements = [\"pip install tf-nightly-gpu-2.0-preview==2.0.0.dev20190513\"]\n",
    "if IN_COLAB:\n",
    "    for i in colab_requirements:\n",
    "        run_subprocess_command(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:55.949441Z",
     "start_time": "2019-05-13T20:41:55.461132Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/cube/tsainbur/conda_envs/tpy3/lib/python3.6/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.autonotebook import tqdm\n",
    "%matplotlib inline\n",
    "from IPython import display\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:55.954492Z",
     "start_time": "2019-05-13T20:41:55.951386Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0-dev20190513\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a fashion-MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:56.046486Z",
     "start_time": "2019-05-13T20:41:55.956193Z"
    }
   },
   "outputs": [],
   "source": [
    "TRAIN_BUF=60000\n",
    "BATCH_SIZE=512\n",
    "TEST_BUF=10000\n",
    "DIMS = (28,28,1)\n",
    "N_TRAIN_BATCHES =int(TRAIN_BUF/BATCH_SIZE)\n",
    "N_TEST_BATCHES = int(TEST_BUF/BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.233179Z",
     "start_time": "2019-05-13T20:41:56.050284Z"
    }
   },
   "outputs": [],
   "source": [
    "# load dataset\n",
    "(train_images, _), (test_images, _) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "\n",
    "# split dataset\n",
    "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype(\n",
    "    \"float32\"\n",
    ") / 255.0\n",
    "test_images = test_images.reshape(test_images.shape[0], 28, 28, 1).astype(\"float32\") / 255.0\n",
    "\n",
    "# batch datasets\n",
    "train_dataset = (\n",
    "    tf.data.Dataset.from_tensor_slices(train_images)\n",
    "    .shuffle(TRAIN_BUF)\n",
    "    .batch(BATCH_SIZE)\n",
    ")\n",
    "test_dataset = (\n",
    "    tf.data.Dataset.from_tensor_slices(test_images)\n",
    "    .shuffle(TEST_BUF)\n",
    "    .batch(BATCH_SIZE)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the network as tf.keras.model object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.247614Z",
     "start_time": "2019-05-13T20:41:59.235143Z"
    }
   },
   "outputs": [],
   "source": [
    "class AE(tf.keras.Model):\n",
    "    \"\"\"a basic autoencoder class for tensorflow\n",
    "    Extends:\n",
    "        tf.keras.Model\n",
    "    \"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AE, self).__init__()\n",
    "        self.__dict__.update(kwargs)\n",
    "         \n",
    "        self.enc = tf.keras.Sequential(self.enc)\n",
    "        self.dec = tf.keras.Sequential(self.dec)\n",
    "\n",
    "    @tf.function\n",
    "    def encode(self, x):\n",
    "        return self.enc(x)\n",
    "\n",
    "    @tf.function\n",
    "    def decode(self, z):\n",
    "        return self.dec(z)\n",
    "    \n",
    "    @tf.function\n",
    "    def compute_loss(self, x):\n",
    "        z = self.encode(x)\n",
    "        _x = self.decode(z)\n",
    "        ae_loss = tf.reduce_mean(tf.square(x - _x))\n",
    "        return ae_loss\n",
    "    \n",
    "    @tf.function\n",
    "    def compute_gradients(self, x):\n",
    "        with tf.GradientTape() as tape:\n",
    "            loss = self.compute_loss(x)\n",
    "        return tape.gradient(loss, self.trainable_variables)\n",
    "\n",
    "    @tf.function\n",
    "    def train(self, train_x):    \n",
    "        gradients = self.compute_gradients(train_x)\n",
    "        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.400383Z",
     "start_time": "2019-05-13T20:41:59.249447Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import (\n",
    "    Conv2D,\n",
    "    Reshape,\n",
    "    Bidirectional,\n",
    "    Dense,\n",
    "    RepeatVector,\n",
    "    TimeDistributed,\n",
    "    LSTM\n",
    ")\n",
    "#from tensorflow.keras.layers import UnifiedLSTM as LSTM\n",
    "\n",
    "N_Z = 128\n",
    "shape_final = (7,7,64) # x channel will be the sequence length\n",
    "\n",
    "enc = [\n",
    "    Conv2D(\n",
    "        filters=32, kernel_size=3, strides=(2, 2), activation=\"relu\", padding=\"same\"\n",
    "    ),\n",
    "    Conv2D(\n",
    "        filters=shape_final[2], kernel_size=3, strides=(2, 2), activation=\"relu\", padding=\"same\"\n",
    "    ),\n",
    "    Reshape(target_shape=(shape_final[0], np.prod(shape_final[1:]))),\n",
    "    Bidirectional(LSTM(units=100, activation=\"relu\")),\n",
    "    Dense(units=512),\n",
    "    Dense(units=N_Z),\n",
    "]\n",
    "\n",
    "dec = [\n",
    "    Dense(units=512),\n",
    "    RepeatVector(shape_final[0]),\n",
    "    Bidirectional(LSTM(units=100, activation=\"relu\", return_sequences=True)),\n",
    "    TimeDistributed(Dense(np.prod(shape_final[1:]))),\n",
    "    Reshape(target_shape=(shape_final[0], shape_final[1], shape_final[2])),\n",
    "    tf.keras.layers.Conv2DTranspose(\n",
    "        filters=64, kernel_size=3, strides=(2, 2), padding=\"SAME\", activation=\"relu\"\n",
    "    ),\n",
    "    tf.keras.layers.Conv2DTranspose(\n",
    "        filters=32, kernel_size=3, strides=(2, 2), padding=\"SAME\", activation=\"relu\"\n",
    "    ),\n",
    "    tf.keras.layers.Conv2DTranspose(\n",
    "        filters=1, kernel_size=3, strides=(1, 1), padding=\"SAME\", activation=\"sigmoid\"\n",
    "    ),\n",
    "    Reshape(target_shape=(28, 28, 1)),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.470980Z",
     "start_time": "2019-05-13T20:41:59.402403Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shape_final[1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-10T18:40:40.306731Z",
     "start_time": "2019-05-10T18:40:40.292930Z"
    }
   },
   "source": [
    "### Create Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.567091Z",
     "start_time": "2019-05-13T20:41:59.472726Z"
    }
   },
   "outputs": [],
   "source": [
    "# optimizers\n",
    "optimizer = tf.keras.optimizers.Adam(1e-3, beta_1=0.5)\n",
    "\n",
    "# model\n",
    "model = AE(\n",
    "    enc = enc,\n",
    "    dec = dec,\n",
    "    optimizer = optimizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.837994Z",
     "start_time": "2019-05-13T20:41:59.569022Z"
    }
   },
   "outputs": [],
   "source": [
    "# exampled data for plotting results\n",
    "example_data = next(iter(test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.847049Z",
     "start_time": "2019-05-13T20:41:59.839958Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_reconstruction(model, example_data, nex=5, zm=2):\n",
    "\n",
    "    example_data_reconstructed = model.decode(model.encode(example_data))\n",
    "    fig, axs = plt.subplots(ncols=nex, nrows=2, figsize=(zm * nex, zm * 2))\n",
    "    for exi in range(nex):\n",
    "        axs[0, exi].matshow(\n",
    "            example_data.numpy()[exi].squeeze(), cmap=plt.cm.Greys, vmin=0, vmax=1\n",
    "        )\n",
    "        axs[1, exi].matshow(\n",
    "            example_data_reconstructed.numpy()[exi].squeeze(),\n",
    "            cmap=plt.cm.Greys,\n",
    "            vmin=0,\n",
    "            vmax=1,\n",
    "        )\n",
    "    for ax in axs.flatten():\n",
    "        ax.axis(\"off\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:41:59.983741Z",
     "start_time": "2019-05-13T20:41:59.850984Z"
    }
   },
   "outputs": [],
   "source": [
    "# a pandas dataframe to save the loss information to\n",
    "losses = pd.DataFrame(columns = ['MSE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:53:25.359774Z",
     "start_time": "2019-05-13T20:42:08.093532Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 | MSE: 0.007553647272288799\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAD2CAYAAAAZOLmfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XewXVX5//EHAZEQ0kNCSAKEAImhGKqCbQTBhih2Z+y9go4oKGP7KoojKooyVvQPwD6iFBtFigUMJCaEEBOTkN4DhCrK74/fuPisD9krZ93ce+6957xffz1n1r77nJxdzs56nrXWTo899lgAAACgdU/q7w8AAAAw2PAABQAAUIkHKAAAgEo8QAEAAFTiAQoAAKASD1AAAACVeIACAACoxAMUAABAJR6gAAAAKvEABQAAUIkHKAAAgEo8QAEAAFTiAQoAAKASD1AAAACVeIACAACoxAMUAABAJR6gAAAAKvEABQAAUIkHKAAAgEo8QAEAAFTiAQoAAKASD1AAAACVeIACAACotEub3++xNr9fo7vvvjvFF110UdZ2zjnnpHiPPfZo3Md9992X4m9/+9tZ2+te97oU77PPPj3+nH1gp17az4A5lurss8/OXp955pkpHjVqVOPf3XTTTduMIyLOOuusXvp0va63jmXEAD2e7kc/+lGKN27cmOIPf/jD/fFxetuguDYfe6x59zvt1Po/4aijjkrxy172shTvu+++2XazZ89O8erVq7O2Sy+9tOX3U/pvqPnMFQbttfnggw+m+MlPfnLWtvPOO7fzowwk2zye9EABAABUancP1IBx6qmnpnjVqlVZ25IlS1L8lKc8JcX/+c9/su2GDh2a4vnz52dtCxcuTPF3vvOdHfuwaNmJJ56YvV6+fHmKSz1Qu+66a4rf+MY39v4HQ494b8fPf/7zFN91110p1msxIuKd73xn336wLtPTHhu9D77gBS/I2rZs2ZLiL37xiy3t/7///W/2esyYMSm++uqrs7ajjz66cT991OvUEfRYe4/Tl7/85RRfddVVWdvIkSNTvHbt2hTfc889je/l+7/wwgtTrFmiiLy38bLLLsva9txzz8b36Ev0QAEAAFTiAQoAAKASD1AAAACVurYGavr06SnWXHxEXs80YsSIFGudjBsyZEj2WkfooX02b96cvd5///1b+jutlbrjjjuytre+9a07/sHQIzfeeGP2WuueHn300RT//ve/z7Yr1UC1YQTWoOe1Z6Xv6be//W2Kv/71r2dtCxYsaPy74cOHp1hHe+2+++7Zdps2bUqx10DtssvjP2GveMUrsrb99tsvxTNnzszaLrjggsbP1e3nh9b9uuuvvz7FOooyIuL4449P8RVXXJFir0/UWqlf/vKXWdsHP/jBFF9yySVZm96XtU45IuKwww5r/Mx9iR4oAACASjxAAQAAVOraFJ526R544IFZ2+GHH57iUlfkm9/85hR7GvDhhx/ujY+JSnPnzs1e63GZMmVK49997GMfS/Hb3/723v9gyLSaJjnhhBOy18997nNTrKmGWbNmZdvpdfuSl7ykR+/dzUopPB1qHhFx7rnnpnjy5MlZ27Rp01KsEzRG5MPU9X557733Ztv9+9//TrGm5SLySYq9bOKBBx5I8fe+972sberUqSn+wAc+kLV1+znxpCc196usWLEixT51wEMPPZTiGTNmpFi/64iIP/zhDyk+4ogjsjbd5+mnn561acr+pJNOavyM7UQPFAAAQCUeoAAAACrxAAUAAFCpa2ugdCjl2LFjszato9ElX9avX59tp7ncl7/85Vmb10uhPTw3/sxnPjPFOsxZj39ExMknn5ziefPm9dGnQyseeeSRFOsisxERhxxySIo3bNiQ4qVLl2bb6bQUaI3WPZXqYL7//e9nr/Va8vohrWfSWqaIvJ5J25YtW5Ztp5/Fr1s9zqWanGHDhmVtP/7xj1PsNVCqZjqHTrR169bstS5n5r9xo0ePTrF+376E1k9/+tMUe43jaaedluKLL744a9OaOT22/YkeKAAAgEo8QAEAAFTq2hTezTffnOLXv/71Wdv48eNTvO+++6bYh+iWZsjVruUJEybs2IdFy7xb+bzzzkvxu9/97hT/4Ac/yLbTGZS/9a1v9dGnQyt0Vuqvfe1rWdvee++d4pUrV6b4Rz/6UbbdM57xjMb9d1sapjdoWtXTozqdhE9VoNejp9F0lYc1a9akWNNEEfmM5X5967Y+lczatWtT7Mf8zjvvTLGng0ozcXcbL1uZOHFiin2FBk3X6vmyePHibLvnP//5KX7BC16Qtenv6Atf+MKsTc87Tef1J3qgAAAAKvEABQAAUIkHKAAAgEpdUwPleXVdxduX/9Ch77vttluKd91118b965QGEXm9lNdHlYYIY8f4sNu//e1vKdYVu33V+LPPPjvFT3/60/vo0+F/Wq1D8mtFpyu46aabUnz55Zdn2z3taU/b4ffuNqUlbn7961+n2GuNtGb0nnvuady/15BqPZMu06Gx82VexowZk2KteYrIp0bwuiZd9uX666/P2rQup9unMVi3bl32+re//W2KtaY0Iq9dVDvvvHPj/r3+bOPGjSlevXp11vbPf/4zxQNlmiB+yQEAACrxAAUAAFCpa1J4ngrQdJwOuYyI+L//+78Uf/SjH23cTlMIe+yxR9amXcak7Nrnuuuuy15fe+21KdbUwO67755tp939ngpA39q8eXP2+o9//GOKJ02alLXpa52q4Morr8y20+kPPL2HbSvdp372s5+l2EsStDzCU17atmTJkqxNrzNNFXkq6P7770+xTncQkZ87pdSffy59rf+2iDyF1+33bk9Z6uzxfpz091GPu6fwtJTC07o6NYlPiaG/qdOmTdveR2+L7j47AAAAeoAHKAAAgEo8QAEAAFTqmhooH96ueVkflvvZz342xTpccs6cOdl255xzzjbjiIh//etfPf+w6DFdoiEir3VYsGBBinWJnoj8fNDatoiIU045pTc/Isxb3vKW7LXWtey1115Z2yc+8YkUL1u2LMXXXHNNtt2sWbNS7EtO6FI9LNvRGj0mugRLRF4T5fVRpVpQXQpLr1OfckbrS30fWh/lf1eiy8ro1Bho3YYNG7LXWs+kNaU+HcEXvvCFFPsSTHpu6bGNiHjggQe2uf/+RA8UAABAJR6gAAAAKnVNCs+7EUeNGpVineE0Ih/6fsIJJ6TYuyw1XeRd1+PGjUuxdzeWZjTHjtFu3oh8GK6m6fbff/9sO00N/PCHP8zazjvvvF78hIjIU+pjx47N2o466qgU33DDDVnbyJEjU6zD4H0Iu6Zdve1FL3pRii+88MKs7alPfep2P3s3uuOOO1Lsw9L12vF7nQ51L6VLdci6l1vo/dL3oa/97zSl5+eAfubFixc3fq5u59M/6Ezw/pun5S56PPWajcjLJ3xqoH322SfFnsLTlK/PYN5f6IECAACoxAMUAABAJR6gAAAAKnVNDZSu2h2RrxruK7drbldX7V6/fn223YQJE1J8wAEHZG26avVpp53Wg0+MntClBiLymgzN53/pS1/KttP8uq9Ajt6nNSlen6K1K8cdd1zW9pWvfCXF8+bNS/HMmTOz7bQ2xpft0bbPf/7zWdsll1yy3c/eDbwuVK8PrR+NiHj44YdTXFrew+uXtJZKj7lPR6Dnh9dY7bbbbikuLTHj9HN6bave570+r9v4933MMcek2JcvK/2dOvjgg1M8f/78rG3q1KmN+x8yZEiK9ZzrT/RAAQAAVOIBCgAAoFLXpPB8eLsOef31r3+dtWnKTf/uF7/4RbbdmWeemWId3hmRdwNrujAiYvjw4a1+bFTy71aHK3/sYx9Lsc5EHBHx8pe/PMWeokDv02HIPq2Hpnp8RfZddnn8lvXsZz87xZ6m05mzN27cmLVp2sfb9HrXlEG3+etf/5q91pSJHoOIJw5FV5qm81nENc1T2k5fl9JynvIppff02PqQ+LvuuivF3Z7C86kE9Lx4xzvekbXpb6DGc+fOzbY79thjUzx79uysrdXZ5P33vL/QAwUAAFCJBygAAIBKPEABAABU6poaKB+qOn369BTr0MyIfOi05m+POOKIbLs1a9ak2PPBOiX9QFk5uhuUlpLQ47xy5cpsOx22rbUTEXmNRGk5CrROl9jxmhrlx1OXhdC6Fq+V0hoJr6sYPXp04/vp8kxHHnlk43adzmtTlB8TPQ5+LPV60WMekU8rorHvQ9/Pa5m0fs5r6fS1nx9N05tEPPHe0M38WGt9qF9Huq3GPrWF1pteddVVWZueI0xjAAAA0IF4gAIAAKjUNSm8m266KXv96U9/OsVz5szJ2q644ooU6+rsPqRzxIgRjfsYP358in1I8Ete8pIWPzVq6YzJEfkwdU3bvvnNb862W7RoUYp91npNB5HC6x2aNvEhyTrztHf/a2pAj8WWLVuy7fTv9NhGREyaNCnFU6ZMydquvPLKFHdzCu/Pf/5z9lrTYZ7W0Wla9txzz6xNp4vxVKruR4+5p/r0vutpOk3N6VQIEfk5VpoawVP2Pjt2N/PfPE3h+fet19wf//jHFPvqEJMnT06xp+J0+gM/1rp/naakP9EDBQAAUIkHKAAAgEpdk8J77Wtfm73WRUN9AVHt0tXuXJ9RfO3atY1tv/nNb1J89NFH9+AToyd+9atfNbZpasDTRto97KMtS4tmomd05nBP2egoLE8XaRpI9+ELQGv6xvehI8z22muvrE2v6W72j3/8I3s9dOjQFHsqTo+DXyt6nfloNx9R9z+lmc79XCnxFJPSFJ6nHS+77LIUf+Yzn2n5/TqRH+umUbAR+bFfsWJFin2Uux7D0mz/fvx0NHSrM5b3NXqgAAAAKvEABQAAUIkHKAAAgEpdUwP17W9/O3v9vOc9L8U+VFqHVmqb599LedhXv/rVKfYcO/qOzgAfkc8Wr3TYdETE/vvvn+KTTjopayvVUqBnlixZkmKtcYnIa2qa6mQi8voln8ZApzjwYeqLFy9OsQ6zj8iv8fvuuy9r66br2GsEtfbFh7Zr7YsOc4/IpxXx+6zWtOg1VqqJc3psS/dnP4+0HsvrtnRVgm5XqoHyqSG01lB/Q32qEOXX2OrVq1O89957N/7d8OHDG9vaiR4oAACASjxAAQAAVOqaFN6CBQuy1zrb9CmnnJK13XjjjSnWWVR9aK/ONu6zHesMub6QJfrObbfd1tJ2PsutdjN7t3UphYCeufzyy1PsaVdN5/iM2Jpq/d3vfpfiv/zlL9l2mhrwNO6ll16aYh1uHRHxk5/8JMXXXntt1nbqqadGt9LrRVdgiIhYtmxZijX9GpFfO57y0bSaHnPfrjR1gbZ5mk7f269pTeENGzascf9+7/Z/X6fzFJunb5V+Vzqtj6b9nK4KEJHPMD516tSsTdPy69evb9xnO9EDBQAAUIkHKAAAgEo8QAEAAFTqmuKOG264IXs9c+bMFF988cVZ22te85oU33zzzSletWpVtp3m/n1Y/Oc+97kUH3vssVmb11yh9+y7777Z66VLl6b4l7/8ZYp9aK22PfvZz+6bD4dEpxLwGhStg9DpRiLymonrr78+xRdccEG2ndZV+dB6fb93vOMdWdvy5ctTvHHjxsbP34l0ygGvW9HaHz0+EXl9lNcraa2RTxnRVAPltUw6zYVPOaDD5Z1OceD3Z/0sXoOny4voeRoRccghhzS+XyfyqTu0BtGnpdDrRdtKS2FpLXJEviSTn4N6ng2UulR6oAAAACrxAAUAAFBpYPSDtYEPe9TXPlRTu/yPPPLIxn1q1/LKlSuzNh2CqUM6I/Ihtcxy3bt8qgkdcq3pvFmzZmXb6XGYPn1633w4JD/84Q9TfPrpp2dtmqZ705ve1LiPV77ylSn2tM+9996bYr82NaWgs5lHRPzsZz9LcbelcnUGbv8+NZXj9zO9D/r0IDrbuM9urqkynbpA036+/6FDh2Zt+jn973SfnsLTIfGeitLX8+bNy9q6LYXn54FOY+CpXKX33dIM/uPGjcte6++yr1Cgv9N+PPsLPVAAAACVeIACAACoxAMUAABApY6ugdLVoT2fqjUvvkTDZz7zmRSfddZZjfvXPOycOXOytpNOOinFusJ0RJ7L9WURUO+rX/1qij/0oQ9lbbfffnuKX/rSl6Z48uTJ2XYTJ05M8dy5c7O2F77whb3yOfE4vXZ8uYx//etfKb7mmmuytjvvvDPFOo3B7Nmzs+0OOuigbf5NRH6+fOMb36j41J3ty1/+coq1hiwiPya+nIcuabVly5asrbT0idZL6XQE/jdaR+W1LzpVgX8urd/xofQbNmxIsdd06fn4q1/9Kmt77WtfG93El8DRaQb0uEQ0T/uhx8j575/ee/14ak2bf67+Qg8UAABAJR6gAAAAKnV0Ck+7FH04pnbT6vD2iHzqgiuvvDLFPhzzb3/7W4p1htaIiKuvvjrFPkxWZ/wlhbfjDjvssBR//etfz9re8573pLj0vWsXv884r13VOvwZPaepHk/habr9wgsvzNruuuuuFOss5W9/+9uz7TQt89SnPjVr++lPf5ril73sZY2f0YfF+yzbneaMM85Isd/rrrrqqhQfccQRWdtFF12UYr0WIyIOPPDAFPv3qSk8jX1aGf0sw4cPz9r02vTUn6aOPI203377pfgNb3hD1vbc5z43xdOmTYtu5tPs6LFZsmRJ1qaz+PsxbDJhwoTstU6NoCt9RDwxrTwQ0AMFAABQiQcoAACASjxAAQAAVOroGijNyXr9gq7mrEtHRESceOKJKb7llltS7Hnd0aNHp/j5z39+1qYrfPtwTJ2ufsqUKc3/ALREl2W59tprs7bTTjstxVqD4UtCLFiwIMV777131jZQhsx2Eq1J9Ho0rTObOXNm1nbcccdtcx++VJO2+TB16ti27fjjj99mvD0333xzY5veI0tLuSi/V+v92ZeKUcOGDcte6z1+7dq1WduaNWsa94PH+bWi16pPWXH33XenWL/7El9KSadJ0KkzIiIWL16cYq9p7i/0QAEAAFTiAQoAAKBSR6fwtNtWU2oR+VBc70a88cYbU6zdzH/961+z7RYuXJhiHfoaEXH++eenWFNMERHHHHPM9j46Kmj65oYbbsjaNKWnQ6x9KLbOWL5o0aKsTWdJ9hmN0TM6c7in38aMGZNinX04Ip+KQlOruupARJ7qGTVqVNbmKwM06fRpC5xPM6BK34VO++HD0pVPjaD3ZC238FUj9Prz46yf2T+/vh43blzWpsPlR44cmbVpesj/3d12TngKT685n4l83rx5KT722GNb2r+XUuix3rp1a9ampTClGe7biR4oAACASjxAAQAAVOIBCgAAoFJH10Bpbt6Homue3Ye0Tpw4McXPec5zUuzLCGgdzQEHHJC1zZ8/P8UvfvGLs7axY8du97OjdaW8/D/+8Y8U63H13L4ut6NL+fj+0Tuuu+66FPuQZ12yQWsifFtdZsKXS9LaKR/6PmfOnBT/6U9/ytr0eu+2pVxK/z69Bnx5D60vKg0v9/pBndZAj7Nfb3qtlqZC8OVadD9+LH3bJp1+zLfHp4ZYsWJFin26F51uwuvKmvh2en3rlAbO65b7Cz1QAAAAlXiAAgAAqNTRKTwdVumpAJ1V/G1ve1vWpl3NOvupdx/rrKw6Q3JExOWXX55in/7gK1/5ynY/O1qn6Rsf3qozIWt3vw+p1q7qqVOnZm1NMyajdZ7a0bSMHqOIvHte0/C+re7T9//ggw+m2IfFP/rooyn24e2q21J4JaV/u5Y2+DQDmkr1e3DTcVi6dGm23caNG1Ps0yTofdzT8prO92uYa7o1fm1q2k6vsYjmlGyJ34c3bNiQYk3lR+Qznw+UMhh6oAAAACrxAAUAAFCJBygAAIBKHV0Ddeedd6ZYh1hGRPz4xz9OsebRI/Ip5DWP7sOtte7C88EHHnhgiv/whz9kbV/72tdS/MlPfrL5H4CWaK2b16kdfvjhKdapJXS5noiIU045JcXvfe97szamMdhxXhtz2223pfjkk0/O2pYsWZJirYmIyIc963Hxa1NfX3DBBVnbF7/4xRRPmzat8TP7MjLdrLTMi9ZA+ZQRWp/oNUr6/Wqs986IiClTpjTuQ/fvdXBar+P3BbTGp+7R30qvj7rnnntSrMuolXgt2qGHHppinW4kImLfffdt/Lv+wh0CAACgEg9QAAAAlTo6hXfZZZel2Lt3NU2gK3NH5ENldTimz16rw6M9Dahd2cuWLcvaNF2EHffNb34zxd51rKt961BsnQk7ImL27NkpXr9+fdZ20UUXpfj888/fsQ/bpW655Zbs9RVXXJFiT69fc801Kdah7hERJ5xwQopvvfXWFM+aNSvbTlN4PpvyiSee2Pg5NVXVzdMW1ND0rA41j3hiGkb5fbfJ8uXLU+ylEuPHj0+xpxk1vaf3ge3huD/Oyxd0Sp7p06dnbYsWLUrxOeec09L+/Xd57ty5KdZpgiLye3sp9d5O9EABAABU4gEKAACgEg9QAAAAlXYqDU/tA219sxL9d+u0BRF53leXFfCh0pp/9xy7D7cdQHorwT9gjmWJ1k9MnDgxxVofFxHxrGc9K8VekzOA9WaxRp8ez7POOit7ffHFF6fYV1a/4447UuzLHp1xxhkp1mM7efLkbLsjjzwyxWvWrMnatI5Dz4mIfq+BGrDXZqvfi9e66fQBXtOix2zUqFEp3rRpU7bd7bff3timQ+d1mHtExIwZM1LsS8C0waC5Np3WtPm0FD3hdU4lem4NsFq0bX4YeqAAAAAq8QAFAABQqd0pPAAAgEGPHigAAIBKPEABAABU4gEKAACgEg9QAAAAlXiAAgAAqMQDFAAAQCUeoAAAACrxAAUAAFCJBygAAIBKPEABAABU4gEKAACgEg9QAAAAlXiAAgAAqMQDFAAAQCUeoAAAACrxAAUAAFCJBygAAIBKPEABAABU4gEKAACgEg9QAAAAlXiAAgAAqMQDFAAAQKVd2vx+j7X5/R5/48fyt77ttttSfP/992dt06dPT/Hw4cNT/KQn5c+b//73v1P8z3/+M2vbaaedUjxjxoyszffTZjttf5OW9NuxLNm0aVP2+qGHHkrxXnvtleL//ve/2XZbt25NsR67iPI50M9661hGDNDj6dft6tWrU3zvvfemeNSoUdl2Y8aMSfEAO2YlHX1tPvzww9nrj3/84ynefffdU7znnntm261bty7FQ4YMydo+9alPpXiXXdr9c1bUkdemXo+PPvpoS9vtvPPOLe9ff1N9/09+8pNTvOuuu2Ztfs/uA9t8g0FzZwEAABgoBtQje1/yp9nvf//7Kb711luzNv3f66RJk1I8duzYbDvt7dD/GUdEjB49OsUXXnhh1qb/20Lv0p6kiIg1a9akWI+rnw+rVq1Ksf+PSXug0F733Xdf9vr0009PsV5/3gP10Y9+NMVHH310H306OO8xvOeee1L87ne/O2u77rrrUqzXmF9vK1asSLFftwsXLkzxd7/73axNe7La0EPRFfR4/uY3v8na9FjofXfChAnZdo888kiK9Z4cEbHbbrul2O/DM2fOTPExxxyTtfXX8aUHCgAAoBIPUAAAAJV4gAIAAKjUNTVQbuTIkSnWUXcRee5VR474SDutZZo6dWrWNnny5G3uD31r6dKl2eu5c+emeNq0aSnW0XkREevXr0+xj9A7+OCDe/ETYnu0jmb27NlZ25YtW1K8efPmFC9btizb7tJLL03xUUcdlbVRD9O7brnllhSfeeaZWZseF61vicjvmXqP1PpR/zuvcfz73/+e4qc97WlZ26GHHpri8847L2vTewGa+b3wkksuSfH8+fOzNr1u99hjjxRv2LAh205HS+oIS9+H1zUuWLAgxYcddljW5qMz24UeKAAAgEo8QAEAAFTqmhSeD6/VbmEfZqndlv/5z39S7BO86XY6yVdExLhx41I8iCby6xH/bvV1u//ta9euzV4vWbIkxTq01ofHz5kzJ8WesiPl0146zHnWrFlZ25QpU1KsqQBPk2v3v17D/neod8YZZ2SvdTj7U57ylKxNSyU8zfLAAw+kWK8xnUwxIuLBBx9MsU+gqKUSnt5btGhRik888cSs7SMf+UiK/d+Dx+m0BRERt99+e4o1TReRTz+h16NPPaHHeujQoY1tTqcK0nt5RH5faKfO/mUHAADoAzxAAQAAVOIBCgAAoFLXFgNoPl4XmY3I8/haP+FLuWhu1+uAuonnrfuzZkhrLiLyOrWVK1em2OtiSvVs6Ft+7eh1pXUyEREjRoxIsdYyHXTQQdl2++yzT4qpYdtxOu3H5ZdfnrXpdC5e56Q1kD4kXq9VPeZeo6ZtXlOp+ywtkeV/94Mf/CDF73//+7M2auQe57VGem/0JXe0rrh0DQ8bNizFfm2WvnudUsinq9FauHYeP3qgAAAAKvEABQAAUKlr+iq9+3jjxo0p9iGQOjxTh9T6ytHaXe0zW/vwT7SHTzWh6SHtftah0RH5sfSZkEkB9S2/Nu+///4U+7HQNLpOReEpmkmTJqW4m9PrveXuu+9Osae/S1MQ6L3UpzjQ46LD3n3/et36tajb+lQWun//u3vvvTfF+m+L6L8h8QORfzeaHvOSFj32ek37lDGaavW0q54jfl/QVKBPV9Nf6IECAACoxAMUAABAJR6gAAAAKnVNDZTnwDX36sPWNc+rsa8mrvvwvL0Pp0d7aP1MRMSKFStSrDl0H56r+XvfR7tp7YbX7+h53Cm1WV43s2nTphT7FCM6FYX++70G8c9//nOKjz322KxN6+Q65Tvsa7/73e9SrMPJI8rTByg/l3XJHuX1bHpv9Xu11sn4/nz4fNPfXX311Vnb+973vsa/6zZ+rJumKojIryWtR/PfTT2+paW+fDoCrVP1Gtb+Qg8UAABAJR6gAAAAKnVNCs+7CjVt4F2R2qbdzt5FvGXLlhR7d7H+XambshP4cNNSWqSvUybr1q3LXutn0zZPJ+i0Frrqt2/bjpSPvp+fm9qtPZjTT/pv1OsoImLZsmUp9iHQ2nWvQ9E97brrrrumeOHChVnbIYcckmIfWo9tu+2221Ls35neLz3touk3Twfp+VuasVxRuqUtAAAPP0lEQVS3K+2jNF1FU7owIuLKK6/MXnd7Ck+/Ry9F0TSap271GOr16PcpPV+89KVEt201bdzXOvuXHQAAoA/wAAUAAFCJBygAAIBKXVMD5XlYrVnyGoxRo0alWHOtnmPXXK4Pq+yUWpUmrdYFtfvfrvUzEflwWh3qXsq933rrrdnrV7ziFb306VqjtQSdujK8fv9ec6ZTTOg0FBERW7duTbHWQPm1qXU6fn3ra1+eqVO/7x3197//PcX+Xeu9z78/rVny2imdkkDPB68Z1evWl2tRpWVevK5K97Nq1arGfXajpukIIvLaQp9SQpft0aXMhg0b1ridHxfdZ2maBNdfyzXRAwUAAFCJBygAAIBKXdNf7V2FOtuxD4kcOnRoijVN592ZpVlZN2zYkOLSbNKD1UD9N/gq3ZoO0OPn54OmHrxruj95OqO/uqp7m6bQfcV3veb82tR/v6aHfLspU6ak2Gcp12kN/Fhr+r6b+f1MpwCpmfqhNM2AngN6nnt6XV+XpvXwa1r/zt9bj7veq/3vSinDbuDpb/3efJUAPTb6fftKADo1ydy5c7O2SZMmpdivW03Z10yd05fogQIAAKjEAxQAAEAlHqAAAAAqdU0NlK/4rsOhfXmApqVcfDsd0uk52FK9Tacv7aL6u/6raRi1L/2hdW+e9283/c78+9PXg7k+Q68/n3pC65l8Jfem5Ry8LkeHUfs5N2/evMb3ftWrXtW4z27i10dpuRatfSkti1WqUWpa1iUiX4bFzwd/P6Xv53VVpfu6Xv+jR49u3H83KE334t+b3kP1HNF6xIj83NLpMSLyujj9fY3I73c1Uxz0pe75JQcAAOglPEABAABU6poUnq/Gra89vaddjNoNXJrp1mciX79+/Tb30Sn03+RppnanKPX9Pb2gs+Bql7Mfc+221+Gy/a2/v9ve4teATjfh104p9aLHRq9hH/Ksrz0VsHnz5hRrqi8iPw/Gjx8f3eqOO+5obPP7YOl46THyKSNavS/qNaApnoj8evdrRffvKZ+mIfcRETfddFOKTz311JY+Y6fyY63Xkt9rm34PR44cmW2n9zBPA+o54ulaPU4jRowofs52GZx3YwAAgH7EAxQAAEAlHqAAAAAqdU0NlNe1lGqgmuonSvlaz8E2rTTeKfTf6//2/lzmZfny5Y1tumSI12Po8gJek9NupdXQB+tSLl7vostn+LQRTXVr/lrjUq2YL+9TGiq9Zs2aFI8bNy5rG6jLF/WFP/3pT41tfk6Wao1KS2E1KS3l4kr7Ly0jU5oO5Nprr01xt9dA7bnnntlrrXvy46LHQs8Dv9fqa1+CSXmbvrdft9RAAQAADBI8QAEAAFTqmhTeypUrs9fajejdlK0Ox9QUgs6sHJGnDXyIdanbcrAYLOmMptmOPTWkbUOGDOn7D9ZlPIWiKTxPofdkeLvT4+nd/WPGjEnxxo0bs7Y5c+akeMaMGVmb76eT3X777dlrHVJemtXb02E+FF1pmqeUbtPzwc8NPSbeVprRX1O8Phx/1qxZjZ+52+iUHxH5d+X3SU2Na8mM/1boDP9+7Zdm/9ffYp8pX49vO3+b6IECAACoxAMUAABApa5J4fmihdq17Ck27VbU7mrvNhw+fPg244i8K9u7QT0ViN7jM87r8dMu53Xr1mXbafezdytjx3m3unbHl1JCpcWoNX3jo79KI7c0Ze/ni54Hvo9uSuEtWrSosc2vj1K6tJSObUrhOT0OPhN/qa10HpVG7+m9oZsXgo944rEupdE0vaffU2k289IqAf6bqr/TvoJAf+muswEAAKAX8AAFAABQiQcoAACASl1TA+U1L5r39pmnNdequXMfYlla0Vtzuz7TMnqX1kF4PcOoUaNSrFMX6MzjEXnOfsWKFVlbfw2R9ffuj/fvK63Wv7imuhavpdDj6TMha62Gny++bTfReh//XvR1afoIbytdm3oc9L29zqg0jYG+n7eVVkvQ88/PHR2CrzVxERFDhw6NTqffqddAac2gfzf6+6j78OOpx8LrqPT1HnvskbXpcfL37q8VGuiBAgAAqMQDFAAAQKWOTuFpt96qVauytrFjx6a4NFS11GWpbb4PnW3cZ7adOXNmijslJdOf9Lv31NyIESNSrKla77bXrmrvVu5PnXJ+lGYj9pmgNdXjUwloW2lIeWkqCt1HKVXVbfT+tn79+pb/rnQcStNJtJp2KV0DpfS6vvZzzKe9aNrnkiVLsrZDDz20/GE7gH5vvkqHlkH49D9679W/Kx0XT9Op0rXvx48UHgAAwCDBAxQAAEAlHqAAAAAqdXQNlNbGrF69OmvTXLbXw2juVWspvAZq06ZN29zO27ympjRcGPU0L+9TRhxwwAEp1uPsx0vrOHyVcew4r4PQa8BrHXQqAa+vaVqCw2sgtN5Nl22KiBg2bFiKN27c2Lj/Tqk/a9XixYsb2/S79vuZHi8f5q/H1utE9bsuTVWgx8HbtLapVG9VUqrBW7hwYdbWDTVQei2tXbs2aytN/6B0Kgi/NvWa9vuwfvf+e6vnXel+0k70QAEAAFTiAQoAAKBSR6fwtLvRUzvare9dhfp32i3sQy732WefFHu3tg7pLM2sSwpvx+mQaz8Oevy029dX89a0jk59EFGeJRk9o9MY+HdaSg3o8dSpQjSOyFP2vn9NOel0JhERe++9d4q77drUYen+nemULf69DB8+PMV+HDRFo6n2ktJs5p66KbWVZikvzbatKT1PYfXnqgT9wb83/W30aQxK37fS703vuxH577Sng/Uc3N7nbBd+DQAAACrxAAUAAFCJBygAAIBKHV0Dpfnx0nBor5tpGjZbGobrefRSjl0/V6neA63RZXq0viUiYsyYMSnW/L3XY+jf6RD4iP7Lr0c8sR5ksNZd+L9Dpw/wmprS9631KVqDMXny5Gy7+fPnp9ivMd2/ny8TJkxI8WD9rnuqdK/TOlGvc9JaFa8n9Skkmuj92e/Vet22uqxLRP5v8L/TpUb8/qyvZ8+e3djm506n0O9q5MiRWduCBQtS7HXFPZlGwu+1el8unQet1tP1NXqgAAAAKvEABQAAUKmjU3jazeerN2vXoXfxa5e07sO7erUL11cv125Q7+rUWVp1ODd6Rrt6ffVw7yL+H//eSzNQ92cKr1PTSHpcPCWkx9PTAlu3bk3xXnvttc04Ik/n+HBovW497aND8jv1u28yc+bMFH/4wx/O2ubNm5din+Zj+fLlKb711luzNp/lW+lx1vuzH5PSPkrTCuhrHwL/9Kc/PcVLly7N2mbMmJHic889N2vrtpILT+Hp9Dx+r9VUvF5zpeklPA2q5TSeDtb9+O95f6EHCgAAoBIPUAAAAJV4gAIAAKjU0TVQmzdvTrHXUmgef9y4cVmb5rk1R+s1UJoP9qkQlOfwdRkRr91APa1v8O9a8+ba5kNkNS+vdTARPV/lHY/zuha9dpxef17roG0TJ05Msde0lZb48CWZmtq6rQZKv8MPfOADWVtpmY53vetdjfvU+6d/n03LIpWWa/H3Lk1/oNNceK3N+eefn2Kv6Sp95m7j159+H6VlrfT69n2U/k6PtS8VU1rKxe8v7UIPFAAAQCUeoAAAACp1dApv3bp1KfZVn3WY5fjx47M2TcdpGtBp+s1TR7qKt86GHfHEaQ1Qx7trb7nllhT70Fed8VrPAZ/eQFNDnsLr9m783uBpUL3G/NrRqUP877RNj5OnCXT/ni7U88BTO0wr8v+VpgTwFIwO+7/xxhuzNp2J3EsgdJ+llGtpyHpTit4/p892P3bs2G1uh5ynTPX683utfv96rEvpNU+f6v3aZ7HXKUx8apL+ukdz5gAAAFTiAQoAAKASD1AAAACVOqoGqrQEgE9JrzUwXiOhS61obt5z+Fpn4atD6/IUnq/Vffpnpt6mntatTJkyJWvT11oH4TVQOpWF5tr7W6ecH6XlcLzuSGtefCizXktaS+jfkx5r38emTZtSrMuXROTX7WD9rttN74t+LPU79DokpcuC+Lmix8/robyGrUmppgs5vZZGjRqVtelUAj51j/7eauzHU88Dr4HS4+K/t/q5vDaSaQwAAAAGCR6gAAAAKnVUCs9pKs67lkspNh3iqrOGH3DAAdl2mvbxlNDq1atT7N2L/n7YMXpsvbtYp7IodffrUObRo0dnbaX0U1/rlFRD6Tv0Nk2pe1d906zwfv3pEGhv0/PFr0UfCo/t07RaKcXmqVS95vQc8GkFdHb40nlUStP5Na335E65xvqCf99aClMqTVGl9JqvxKHlE34uaUrPpzjor6ko6IECAACoxAMUAABAJR6gAAAAKnVUwr80jYHna6dNm5biww8/PGvTJSJ0qOaQIUOy7XT5D5/GQId7ah1VRF6b1SnD1NvJvzNdUsCPg26rw959iKwuITB58uSsjWOy43wldZ1KwGuUdGklr8HQmhodAu1TT+hx9/pHrZfwa5pjXe+ggw5KsR9Lfe11Kjp1Qak+Ue+zXkelr0vLjvhx7a9h74NNaXoQX8pFf9f02Prvn9aj+bJm+n6rVq3K2vQ88BqopvqrvkYPFAAAQCUeoAAAACp1dApPV12fPn161nbwwQenWKctiMi7mrXb0LuBS0Mnx4wZk2JPL/iQWtTx4zBp0qQUa9dxRD4UVtO4elwj8i7gAw88MGtrdbZj5PR69NSLplA97arD1v2a1lTgmjVrUrx58+ZsO00v+EoDeqz92JLCq3f88cenWMsfIiKWL1+eYj/O+l3rdapTzETk54oPbVeejtUSjmc+85lZW38Nex8M9JrzFTw0Ha5p+Ij8+z700ENT7OUzetyf8YxnZG36Wzlv3rysTc+DqVOnNr53O3EWAQAAVOIBCgAAoBIPUAAAAJV2avNwzj59M/+36BQEXiOhud3SCuK6T99/KY+uQ2o9b6/5Wq/FaYPeKvIYMOOAtabFVwjXuguNPWeubbpET0Q+ZLa/a2RsCYre/DC9fjz1s/px+c53vpNivza9ZlBpnYtew36N6RI+Pi3Ffvvtl+KTTz45a/P6mzYb9NdmaSoBr4HS46zXcGkfXkun55jXP2odqrfpcPw+MqCvzZbf2H7z9FiUnh20ttB/J5t+XyPy88DPF73GfRqDNtyjt7lTeqAAAAAq8QAFAABQqd0pPAAAgEGPHigAAIBKPEABAABU4gEKAACgEg9QAAAAlXiAAgAAqMQDFAAAQKX/B4334JFM02cmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x288 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "for epoch in range(n_epochs):\n",
    "    # train\n",
    "    for batch, train_x in tqdm(\n",
    "        zip(range(N_TRAIN_BATCHES), train_dataset), total=N_TRAIN_BATCHES\n",
    "    ):\n",
    "        model.train(train_x)\n",
    "    # test on holdout\n",
    "    loss = []\n",
    "    for batch, test_x in tqdm(\n",
    "        zip(range(N_TEST_BATCHES), test_dataset), total=N_TEST_BATCHES\n",
    "    ):\n",
    "        loss.append(model.compute_loss(train_x))\n",
    "    losses.loc[len(losses)] = np.mean(loss, axis=0)\n",
    "    # plot results\n",
    "    display.clear_output()\n",
    "    print(\n",
    "        \"Epoch: {} | MSE: {}\".format(\n",
    "            epoch, losses.MSE.values[-1]\n",
    "        )\n",
    "    )\n",
    "    plot_reconstruction(model, example_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-13T20:53:25.573916Z",
     "start_time": "2019-05-13T20:53:25.362394Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Epoch')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtMAAAEKCAYAAADQLdlLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8nGW99/HPb2ayb20m6ZY0S9tAKYUCDYVCiyDnQHGhsgmIgj4obqCP2xGf53iOx0fPETdEFhUFRRQBQaQiCEJBKEtpSwu0dEm6J12ydMm+zMz1/DF3QhrbZppMMlm+79crr7mXa+78xvtl+M7V674uc84hIiIiIiLHzpfoAkRERERERiqFaRERERGRflKYFhERERHpJ4VpEREREZF+UpgWEREREeknhWkRERERkX5SmBYRERER6SeFaRERERGRflKYFhERERHpp0CiCzgWeXl5rqSkJNFliIiIiMgotmrVqjrnXH4sbUdUmC4pKWHlypWJLkNERERERjEz2x5rWw3zEBERERHpJ4VpEREREZF+UpgWEREREeknhWkRERERkX5SmBYRERER6SeFaRERERGRflKYFhERERHpJ4XpPjyzbg93v7g50WWIiIiIyDAUU5g2s0VmttHMKs3s5sOcTzGzh7zzy82spNf5IjNrMrOvxnrN4eLFilrufF5hWkRERET+WZ9h2sz8wJ3ARcAs4Gozm9Wr2fXAfufcDOBW4JZe528FnjrGaw4LxbkZHGzt5GBLZ6JLEREREZFhJpae6XlApXNui3OuA3gQWNyrzWLgPm/7EeB8MzMAM/sQsAVYd4zXHBaKgukAbN/XnOBKRERERGS4iSVMFwA7e+xXeccO28Y5FwIOAkEzywC+DvxXP64JgJndYGYrzWxlbW1tDOXGV3FXmK5vGfLfLSIiIiLDWyxh2g5zzMXY5r+AW51zTf24ZvSgc3c758qdc+X5+fl9FhtvRbnRML1jn8K0iIiIiBwqEEObKmBqj/1CYNcR2lSZWQDIAfYBZwCXm9n3gXFAxMzagFUxXHNYSE8OMCErhW11GuYhIiIiIoeKJUyvAMrMrBSoBq4CPtKrzRLgOuBV4HJgqXPOAQu7GpjZt4Am59wdXuDu65rDRnEwne3qmRYRERGRXvoc5uGNgb4ReBpYDzzsnFtnZt82s4u9ZvcQHSNdCXwZOOpUd0e6Zv8/xuAqys1gh8ZMi4iIiEgvsfRM45x7Eniy17H/6LHdBlzRxzW+1dc1h6viYDqPvtFGW2eY1CR/ossRERERkWFCKyDGoGtGDz2EKCIiIiI9KUzHoDiYAWh6PBERERE5lMJ0DIpzu+aa1oweIiIiIvIuhekYjEtPIis1oGEeIiIiInIIhekYmFl0ejwN8xARERGRHhSmY1Scm6FhHiIiIiJyCIXpGBUH06na30ooHEl0KSIiIiIyTChMx6g4mE4o4th9sC3RpYiIiIjIMKEwHaOiXE2PJyIiIiKHUpiOUdfCLdv3ady0iIiIiEQpTMdoUnYqyQGfeqZFREREpJvCdIx8PqMoN10zeoiIiIhIN4XpY1Ccq7mmRURERORdCtPHoCiYzo59LTjnEl2KiIiIiAwDCtPHoDg3nZaOMHVNHYkuRURERESGAYXpY1Ac7JoeT+OmRURERERh+ph0T4+ncdMiIiIigsL0MSkcn47PYPs+hWkRERERUZg+JskBH5Nz0tihYR4iIiIigsL0MSsOpqtnWkREREQAheljVhzUXNMiIiIiEhVTmDazRWa20cwqzezmw5xPMbOHvPPLzazEOz7PzNZ4P2+a2SU93rPNzN72zq2M1wcabMXBDPY1d9DY1pnoUkREREQkwfoM02bmB+4ELgJmAVeb2axeza4H9jvnZgC3Ard4x9cC5c65U4BFwC/MLNDjfec5505xzpUP8HMMmeJczeghIiIiIlGx9EzPAyqdc1uccx3Ag8DiXm0WA/d5248A55uZOedanHMh73gqMOKXDizypsfboXHTIiIiImNeLGG6ANjZY7/KO3bYNl54PggEAczsDDNbB7wNfKZHuHbAM2a2ysxuONIvN7MbzGylma2sra2N5TMNqncXblGYFhERERnrYgnTdphjvXuYj9jGObfcOXcicDrwDTNL9c6f7Zw7jejwkc+b2TmH++XOubudc+XOufL8/PwYyh1cmSkBghnJWgVRRERERGIK01XA1B77hcCuI7XxxkTnAPt6NnDOrQeagdne/i7vtQZ4jOhwkhFBM3qIiIiICMQWplcAZWZWambJwFXAkl5tlgDXeduXA0udc857TwDAzIqB44FtZpZhZlne8QzgAqIPK44IxcEMjZkWEREREQJ9NXDOhczsRuBpwA/c65xbZ2bfBlY655YA9wD3m1kl0R7pq7y3LwBuNrNOIAJ8zjlXZ2bTgMfMrKuGB5xzf4v3hxssRbnp/HlNNe2hMCkBf6LLEREREZEE6TNMAzjnngSe7HXsP3pstwFXHOZ99wP3H+b4FmDOsRY7XBQH03EOqva3Mj0/M9HliIiIiEiCaAXEfigOds01rYcQRURERMYyhel+KMrV9HgiIiIiojDdL3mZyWQk+xWmRURERMY4hel+MDOKNKOHiIiIyJinMN1PxbnpGjMtIiIiMsYpTPdTcTCdnftaCUd6LwYpIiIiImOFwnQ/FQXT6QhH2NPQluhSRERERCRBFKb7qSTYNaOHhnqIiIiIjFUK0/1UlBuda3qHZvQQERERGbMUpvtpyrg0kvzGds3oISIiIjJmKUz3k99nFI7XjB4iIiIiY5nC9AAU5aZr4RYRERGRMUxhegBKgunsqG/BOU2PJyIiIjIWKUwPQFEwg8b2EPtbOhNdioiIiIgkgML0ABR7M3po3LSIiIjI2KQwPQDFwa4wrXHTIiIiImORwvQATM1VmBYREREZyxSmByA1yc/knFS279MwDxEREZGxSGF6gIpy07UKooiIiMgYpTA9QMXBdK2CKCIiIjJGxRSmzWyRmW00s0ozu/kw51PM7CHv/HIzK/GOzzOzNd7Pm2Z2SazXHCmKgxnUNrbT3B5KdCkiIiIiMsT6DNNm5gfuBC4CZgFXm9msXs2uB/Y752YAtwK3eMfXAuXOuVOARcAvzCwQ4zVHhCLvIcQd6p0WERERGXNi6ZmeB1Q657Y45zqAB4HFvdosBu7zth8Bzjczc861OOe6umxTga6lAmO55ohQEswANKOHiIiIyFgUS5guAHb22K/yjh22jReeDwJBADM7w8zWAW8Dn/HOx3LNEaEo2NUzrRk9RERERMaaWMK0HeaYi7WNc265c+5E4HTgG2aWGuM1oxc2u8HMVprZytra2hjKHVo5aUmMS09Sz7SIiIjIGBRLmK4CpvbYLwR2HamNmQWAHGBfzwbOufVAMzA7xmt2ve9u51y5c648Pz8/hnKHXnFuusK0iIiIyBgUS5heAZSZWamZJQNXAUt6tVkCXOdtXw4sdc457z0BADMrBo4HtsV4zRGjKJihhVtERERExqA+w7Q3xvlG4GlgPfCwc26dmX3bzC72mt0DBM2sEvgy0DXV3QLgTTNbAzwGfM45V3eka8bzgw2lkmA6uw600RmOJLoUERERERlCgVgaOeeeBJ7sdew/emy3AVcc5n33A/fHes2Rqig3nXDEUb2/lZK8jESXIyIiIiJDRCsgxkGxNz3etnoN9RAREREZSxSm46A4qIVbRERERMYihek4mJCVQmqSTzN6iIiIiIwxCtNxYGYU52YoTIuIiIiMMQrTcVIUTNcqiCIiIiJjjMJ0nHQt3BKJHHYhRxEREREZhRSm46Q4mE57KEJNY3uiSxERERGRIaIwHSdF3vR42zU9noiIiMiYoTAdJyXe9HjbNT2eiIiIyJihMB0nU8al4fcZOzSjh4iIiMiYoTAdJ0l+HwXj0rQKooiIiMgYojAdR8XBdK2CKCIiIjKGKEzHUZE3PZ6IiIiIjA0K03FUHEznYGsnB1s6E12KiIiIiAwBhek4Ku6aHk8rIYqIiIiMCQrTcVTsTY+3TUM9RERERMYEhek4KsqNhukdmtFDREREZExQmI6j9OQA+VkpeghRREREZIxQmI6z4tx0rYIoIiIiMkYoTMdZcTBDqyCKiIiIjBEK03FWHExnT0MbbZ3hRJciIiIiIoMspjBtZovMbKOZVZrZzYc5n2JmD3nnl5tZiXf8X81slZm97b2+t8d7XvCuucb7mRCvD5VIXTN6aCVEERERkdGvzzBtZn7gTuAiYBZwtZnN6tXsemC/c24GcCtwi3e8Dvigc+4k4Drg/l7vu8Y5d4r3UzOAzzFslOZF55p+Y/v+BFciIiIiIoMtlp7peUClc26Lc64DeBBY3KvNYuA+b/sR4HwzM+fcaufcLu/4OiDVzFLiUfhwNXtKDidOyeb2pZW0hzTUQ0RERGQ0iyVMFwA7e+xXeccO28Y5FwIOAsFebS4DVjvn2nsc+7U3xOObZmbHVPkw5fMZN180k+oDrfzutR2JLkdEREREBlEsYfpwIdcdSxszO5Ho0I9P9zh/jTf8Y6H387HD/nKzG8xspZmtrK2tjaHcxFtYls+CGXncsbSChrbORJcjIiIiIoMkljBdBUztsV8I7DpSGzMLADnAPm+/EHgMuNY5t7nrDc65au+1EXiA6HCSf+Kcu9s5V+6cK8/Pz4/lMw0LX180k/0tndz9jy2JLkVEREREBkksYXoFUGZmpWaWDFwFLOnVZgnRBwwBLgeWOuecmY0D/gp8wzn3cldjMwuYWZ63nQR8AFg7sI8yvJxUmMMH50zhV8u2UNPQluhyRERERGQQ9BmmvTHQNwJPA+uBh51z68zs22Z2sdfsHiBoZpXAl4Gu6fNuBGYA3+w1BV4K8LSZvQWsAaqBX8bzgw0HX73gOEJhx0+eq0h0KSIiIiIyCMy53sOfh6/y8nK3cuXKRJdxTP7z8bX8bvkOnvnSOUzPz0x0OSIiIiLSBzNb5Zwrj6WtVkAcZDedX0ZqwMcPn96Y6FJEREREJM4UpgdZXmYKn1w4jafW7mH1Di3kIiIiIjKaKEwPgU+dM41gRjLfe2oDI2lYjYiIiIgcncL0EMhMCfCF88tYvnUfL2waGXNli4iIiEjfFKaHyNXziijKTeeWpzYQjqh3WkRERGQ0UJgeIskBH1+98Hg27Gnk8TXViS5HREREROJAYXoIfeCkycwuyOZHz2yirTOc6HJEREREZIAUpoeQz2fcvOgEqg+08rvXtie6HBEREREZIIXpIbagLI+FZXnc8XwlDW2diS5HRERERAZAYToBvr5oJgdaOvnFPzYnuhQRERERGQCF6QSYXZDDB+dM4Z5lW6lpaEt0OSIiIiLSTwrTCfLVC44jFHb85LmKRJciIiIiIv2kMJ0gxcEMrjmjiIdW7GRzbVOiyxERERGRflCYTqCbzi8jNeDjh09vTHQpIiIiItIPCtMJlJeZwqfOmcZTa/fwxo79iS5HRERERI6RwnSCfXLhNPIyk/nKw29SWaPhHiIiIiIjicJ0gmWmBLjrmrk0tHbyoTtf5ul1exJdkoiIiIjESGF6GJhXmstfblrAtPwMPn3/Kn70zEbCEZfoskRERESkDwrTw8SUcWk8/On5fLi8kNuXVnL9fSs42KIVEkVERESGM4XpYSQ1yc8tl53Mdz40m5cr67j4zmVs2NOQ6LJERERE5AgUpocZM+OjZxbz4A1n0tIR5pI7X+GJt3YluiwREREROYyYwrSZLTKzjWZWaWY3H+Z8ipk95J1fbmYl3vF/NbNVZva29/reHu+Z6x2vNLOfmpnF60ONBnOLc/nrTQuYNSWbGx9YzX8/uZ5QOJLoskRERESkhz7DtJn5gTuBi4BZwNVmNqtXs+uB/c65GcCtwC3e8Trgg865k4DrgPt7vOdnwA1AmfezaACfY1SakJ3KHz51Jh87s5i7X9zCdb9+nX3NHYkuS0REREQ8sfRMzwMqnXNbnHMdwIPA4l5tFgP3eduPAOebmTnnVjvnusYorANSvV7syUC2c+5V55wDfgt8aMCfZhRKDvj4fx+azfcvP5kV2/bzwduXsbb6YKLLEhERERFiC9MFwM4e+1XescO2cc6FgINAsFeby4DVzrl2r31VH9cEwMxuMLOVZraytrY2hnJHpw+XT+WRz8zHOcdlP3uFR1dV9f0mERERERlUsYTpw41l7j0J8lHbmNmJRId+fPoYrhk96Nzdzrly51x5fn5+DOWOXicXjmPJTQs4tWgcX/njm/z475uIduyLiIiISCLEEqargKk99guB3tNLdLcxswCQA+zz9guBx4BrnXObe7Qv7OOachh5mSn87vozuGJuIT99roL/++e1WuBFREREJEFiCdMrgDIzKzWzZOAqYEmvNkuIPmAIcDmw1DnnzGwc8FfgG865l7saO+d2A41mdqY3i8e1wOMD/CxjRsDv4/uXn8xnz53OA8t38Pnfv0FbZzjRZYmIiIiMOX2GaW8M9I3A08B64GHn3Doz+7aZXew1uwcImlkl8GWga/q8G4EZwDfNbI33M8E791ngV0AlsBl4Kl4faiwwM76+aCbf/MAs/rZuDx//9es0tGnFRBEREZGhZCNpzG15eblbuXJlossYdh5fU81XHn6TsolZ3PeJ05mQnZrokkRERERGLDNb5Zwrj6WtVkAcBRafUsA9Hz+d7fXNXPbzV9hW15zokkRERETGBIXpUeI9x+XzwKfOpKktxGU/e4W3qzQXtYiIiMhgU5geRU6ZOo5HPnsWqUl+rrr7VV6urEt0SSIiIiKjmsL0KDM9P5NHP3sWhePT+cSvV/DEW5pxUERERGSwKEyPQpNyUnn40/OZMzWHm/6wmt++ui3RJYmIiIiMSgrTo1ROehL3X38G58+cyH88vo4fPbNRqyWKiIiIxJnC9CiWmuTn5x89jQ+XF3L70kq+9NAamtpDiS5LREREZNQIJLoAGVwBv49bLjuZqePTufXZTbxZdZDbrz6V2QU5iS5NREREZMRTz/QYYGbcdH4ZD3zqTFo6Qlx61yv85uWtGvYhIiIiMkAK02PImdOCPPXFc1hQlse3/vIOn75/FQdaOhJd1rBS39TO/a9uIxLRFw0RERHpm8L0GJObkcw915Xz7+8/gec31vD+ny5j1fZ9iS5r2PjvJzfwzcfX8dTaPYkuRUREREYAhekxyMz45MJpPPKZs/D7jA//4jXufL5yzPfGVuxt5LHVVQDcvrRizP/vISIiIn1TmB7D5kwdxxNfWMBFsyfxg6c3cu29r1PT2JboshLmx3/fRFqSn39//wls2NPIcxtqEl2SiIiIDHMK02NcdmoSt199Kt+79CRWbt/H+257iZcqahNd1pB7q+oAT63dwycXTuPjZ5VQlJvO7Usr9JCmiIiIHJXCtGBmXDWviCU3LmB8ejLX3vs63//bBjrDkUSXNmR++MwmxqUn8cmFpQT8Pj537nTeqjrIixV1iS5NREREhjGFael23MQslty4gCvLp3LXC5u55K6X+ePKnTSP8oVeXttSz4ubavncudPJSk0C4NLTCpmSk8rtz6l3WkRERI5MYVoOkZbs53uXncztV59KU1uIrz3yFqd/91m+8vCbvLq5ftQ9lOec44dPb2RidgrXzi/pPp4c8PGZc6ezcvt+Xtui2U5ERETk8LQCohzWB+dM4QMnT2bV9v08sqqKJ97azaNvVFE4Po1LTyvkstMKKA5mJLrMAXthUy0rt+/nOx+aTWqS/5BzHy6fyu1LK7l9aQXzpwcTVKGIiIgMZzaS/gm7vLzcrVy5MtFljEmtHWGeeWcPj6yqYlllHc7BvNJcLj+tkPedPJnMlJH3vSwScXzwjmU0tHXy3JfPJTnwz/9Q86uXtvCdv67n0c/OZ25xbgKqFBERkaFmZqucc+WxtNUwD4lJWrKfxacUcP/1Z/Dy19/L1y48nrrGdv7t0bc4/TvP8uWH1vByZR1tneFElxqzp9buYd2uBr70L8cdNkgDfOSMInIzkrl9aeUQVyciIiIjQUzdiWa2CLgN8AO/cs59r9f5FOC3wFygHrjSObfNzILAI8DpwG+cczf2eM8LwGSg1Tt0gXNOE/uOAFPGpfH582bwuXOn88aOA94wkF38aXV19HxOKqX5GZQEMyjNy2Catz01N50k//D4/hYKR/jR3zdSNiGTxacUHLFdenKA6xeU8oOnN/JW1QFOLhw3hFWKiIjIcNdnmDYzP3An8K9AFbDCzJY4597p0ex6YL9zboaZXQXcAlwJtAHfBGZ7P71d45zTuI0RysyYWzyeucXj+c8PzuKFjbVs2tvI1rpmttY185c3d9HQ9u5MIH6fUZSbTkkwndK8TErzMzizNJeyiVlDXvufVlezpbaZn390Ln6fHbXttfOL+cU/NnPH0kruvjamf/ERERGRMSKWnul5QKVzbguAmT0ILAZ6hunFwLe87UeAO8zMnHPNwDIzmxG/kmU4Sk3ys2j2JBbNntR9zDnH/pbO7nC9ta6JbXUtbKlr5rUt+2jtDBPwGV+78Hg+tXAavj5Cbby0h8Lc9mwFJxfmcOGJE/tsn5WaxCfOLuW25ypYv7uBEyZnD0GVIiIiMhLEEqYLgJ099quAM47UxjkXMrODQBDoa8WLX5tZGHgU+I4bSU9DSp/MjNyMZHIzkplbPP6Qc845qg+08p0n1vM/T21g+dZ9/PCKOeRmJA96XX9YvoPqA61877KTMIstwH/i7BLuWbaVO5+v5I6PnDbIFYqIiMhIEcsA1sOljd6hN5Y2vV3jnDsJWOj9fOywv9zsBjNbaWYra2vH3jLXo5WZUTg+nZ999DS+vfhEllXU8b7bXmLFtsGd07mlI8Qdz1dy5rRcFszIi/l949KT+dj8Yv769m4qa5oGsUIREREZSWIJ01XA1B77hcCuI7UxswCQAxw1FTnnqr3XRuABosNJDtfubudcuXOuPD8/P4ZyZSQxM66dX8KfPncWKUk+rrr7Ne58vnLQFof59cvbqGvq4GsXHh9zr3SXTy4oJSXg464XNLOHiIiIRMUSplcAZWZWambJwFXAkl5tlgDXeduXA0uPNmTDzAJmludtJwEfANYea/EyeswuyOGJmxZw0exJ/ODpjVz369epa2qP6+842NrJL/6xmfNnTujXnNHBzBSuOaOYx9fsYkd9S1xrExERkZGpzzDtnAsBNwJPA+uBh51z68zs22Z2sdfsHiBoZpXAl4Gbu95vZtuAHwMfN7MqM5sFpABPm9lbwBqgGvhl/D6WjERZqUncfvWpfPeS2Szfuo/33fYSr26uj9v1f/niFhraQnzlguP7fY0bzpmG32f87B/qnRYRERGtgCjD1Du7GrjxgTfYVt/MF88/jhvfO6PPKeyOpraxnff84HnOP2Eit1996oBq++af1/Lgih288LXzKBiXNqBriYiIyPCjFRBlxJs1JZslNy3g4jlTuPXZTVx773JqGtv6fb27XqikPRThS/9SNuDaPnPudJyDu/+xecDXEhERkZFNYVqGrcyUALdeeQrfv+xkVm3fz/tuW8ayir5mW/xn1Qda+f1rO7j8tEKm5WcOuK6CcWlcdlohf1ixk5qG/gd8ERERGfliWk5cJFHMjA+fPpU5U8fx+Qfe4KP3LGdyTionTM7mhMlZ3ms2JcGMIw4D+emzFQB8IQ690l0+d950/rhqJ798aQv/9/2z4nZdERERGVkUpmVEOH5SFktuPJsHlu9gbfVB1u9u5B+bagl7U+ilJvk4flI2s3oE7JmTsqhpbOeRN6q4dn5xXMc3FwczWHxKAb97bQefec90gpkpcbu2iIiIjBwK0zJipCcH+OTCad377aEwFXubWL+7gfW7G1m/u4Gn1u7hD6+/u2BnRrKflICPz50b/xXtP3/edP68ppp7X97K1y6cGffri4iIyPCnMC0jVkrAz+yCHGYX5HQfc86xp6GtO2Bv2NPIOWV55GfFv+d4xoQs3jd7Mve9sp1PLZxGenKAUCRCKOIIhR2hcITOiCMcdnRGIoTCjs5w9PyErBSmaCYQERGREU9T44kMwPrdDVx020vH/L6Az/j6oplcv6AU3wCm/BMREZH4O5ap8dQzLTIAJ0zO5tYr57CtroUkvxHw+wj4LPrj90WP+XwEDnk1/riyiu8+uZ5XNtfxwyvmaMy1iIjICKWeaZEEcM7xu+U7+H9PvMP49CR+cuWpzJ8eTHRZIiIighZtERn2zIyPnVnMnz93NhkpAT7yq9f48d83dc9OIiIiIiODwrRIAs2aks1fblzApacW8tPnKrj6l6+x+2DrkP3+SMTxzq4GfvGPzXzl4TdZvqV+yH63iIjIaKBhHiLDxJ/eqOLf/7yWlICPH14xh/NPmDgov6emoY2XKupYVlnHSxV11DW1A9FpBJs7wlwwayI3XzQzLqtFioiIjETHMsxDYVpkGNlS28SND6zmnd0NXL+glK8vmklyYGD/gNTWGeb1rft4qaKWlyrq2LCnEYBgRjILyvJYWJbPghl55KQlce/LW7nr+UraQxGuOaOIL5xfFpeHI9s6wzy1djc76lv5+Nkl5KQlDfiaIiIig0VhWmQEa+sM8z9Prue+V7dzUkEOd3zkVIqDGX2+zzlHY3uImoZ2ahraeLv6IC9V1PH6tn10hCIk+32cXjqehWX5LCzL44RJ2Yedlq+2sZ3bntvEH17fSXqSn8+eN53/dXYpqUn+Y/4sa6sP8uCKHTy+ZheNbSEAinLTueua0w6ZH1xERGQ4UZgWGQWeXreHf3vkLcIRx3c+NJuTCnPY29AWDcuNbextaI/uN0bD896Gdlo7w4dc4/iJWV7vcx5nlAZJS449EFfWNPG9pzbw7Pq9TMlJ5WuLjmfxnII+58U+2NLJn9dU89CKnbyzu4GUgI+LZk/iw6dPJdnv48YHVrOvpYNvffBErp43FTPNsy0iIsOLwrTIKFF9oJUv/GE1q7bv/6dzaUl+JmanMCE7lQlZKUzMTo3uZ6UyITuF6fmZTMxOHXANr26u57+fXM/b1QeZXZDN/3nfCZw1Pe+QNpGI49Ut9Ty0Yid/W7eHjlCE2QXZXFk+lYtPKThkWEd9UztfevhNXtyzhHRqAAAU6ElEQVRUy6WnFvCdS2aTnqwp70VEZPhQmBYZRULhCE+8tRuACVnR8DwxO4XMlMCQ9epGIo4lb+7iB09vpPpAK/9ywgRuvmgm6ckBHllVxR9X7WTnvlayUwNccmoBHz59KidOOfIwjkjEccfzldz67CZm5Gfys4+exowJWUPyWURERPqiMC0ig6KtM8yvX97GXc9X0tIZxjlHxMFZ04NcefpULjxx0jGNrX65so4vPrialo4w/3PpSSw+pWAQqxcREYmNwrSIDKr6pnbufXkrfp+Py08rpCiY3u9r7W1o46YHVvP6tn1cc0YR3/zArH497CgiIhIvCtMiMqKEwhF++Mwmfv6PzcwuyOauj8wdUEAXEREZCIVpERmRnn1nL19+eA0O+NEVc7jgxElHbBsKR9h9sI2q/a1UH2ilan8L1ftb6QxHmDk5m1mTs5k1JZu8OMyTLSIiY0vcw7SZLQJuA/zAr5xz3+t1PgX4LTAXqAeudM5tM7Mg8AhwOvAb59yNPd4zF/gNkAY8CXzR9VGMwrTI6LdzXwuff+AN3qo6yKcWlnLu8ROo3h8Ny1X7W6k60Er1/lb2NLQRjhz6J2Nidgo+M3YfbOs+NiErhVlTouH6BC9glwQz8PcxxV88tYfCpAQ0dEVEZKSIa5g2Mz+wCfhXoApYAVztnHunR5vPASc75z5jZlcBlzjnrjSzDOBUYDYwu1eYfh34IvAa0TD9U+fcU0erRWFaZGxoD4X57l/X89tXt3cf8xlMzkmjYFwahePTKBjvvY5Lp3B8GpPHpXYH1gMtHbyzu4F3djV0v1bWNBHywndakp+Zk7O6e6/PnBZkWl5GXGdH2VbXzF/f3s1f39rNO7sbmDkpi3OOiy6Yc3pJrsaFi4gMY/EO0/OBbznnLvT2vwHgnPufHm2e9tq8amYBYA+Q39XTbGYfB8q7wrSZTQaed87N9PavBs51zn36aLUoTIuMLat37KetM0Lh+DQm5aSS5O//0urtoTCVNU2HBOz1uxto8FZmLBiXxjnH5XNOWR5necurH6utdc082SNAA5xaNI4zSoO8VXWAldv20xGOkBLwMa80l4Xecu4zJ2UN6jSHneEIW2qb2bCngQ17Gtmwu4Etdc2UF+dywznTOH6SpiUUEenpWMJ0LCslFAA7e+xXAWccqY1zLmRmB4EgUHeUa1b1uqbmxBKRQ5xaND5u10oJ+DlxSs4h818759he38Kyyjpe3FTLX97cxR9e34HfZ5wydRwLy/I457h85hSOO+KwkCMF6H9//wlcdNJkCsaldbdt6QixfOs+XtpUx0sVtfz3kxuADeRnpbBwRh4Lj8tjwYx88rP6N87bOUdtYzvrvcC8cU8j6/c0UlnTSGc42nGS5Dem52dSNiGLJ9/ezaNvVHHe8fl8+j3TOaM0d8StSBkKRwgM4EuWiMhAxRKmD/eXtXd3dixt+tXezG4AbgAoKio6yiVFRI6NmVGSl0FJXgYfPbOYznCENTsP8OKmWl7cVMttz1Xwk2cryElLYsGMvO5w3R6KxByge0pPDnDe8RM47/gJAOw+2MqyijpeqqjjhU21/Gl1NQAnTM6mcPzhr3EkTW0hNu5tZF9zR/exSdmpzJycxTnH5XHCpGxmTs5iWl4myYFo+Nzf3MH9r23nvle2cdXdrzGnMIdPv2c6F544aUjHlB+L2sZ2XttSz6tb6nltcz3b6ps5qXAcZ08PcvaMPOYWj9cQGhEZUhrmISJyBPubO7p7rV+sqGVvQ/sh508tGsf7T5rM+06azJQjBOhYRSKOd3Y38GJFLS9X1rGvufOY3p+a5OP4iVnMnJTF8ZOymTkpi/EZyTG9t60zzCOrqvjVS1vYVt9CcTCdTy6cxhVzCxMeTPc1d0TD8+ZogK6saQIgMyXAvNJcpudnsHrHAVbvPEA44kgJ+CgvGc/ZM/I4e3oeswtyhu0XAxEZvuI9ZjpA9AHE84Fqog8gfsQ5t65Hm88DJ/V4APFS59yHe5z/OD3CtHdsBXATsJzoA4i3O+eePFotCtMikijOOSpqmnhxUy0+MxbNnjTgAD3chCOOZ9bt4ecvbuHNnQcIZiRz7fwSrp1fHHMwH6iDLZ28tjUanl/bUs+GPY0ApCf7KS/JZf60IPOnB5k9JfuQ4R1N7SFe31rPsop6Xtlc1/2+7NQA871e67Nn5MX9QVMRGZ0GY2q89wE/ITo13r3Oue+a2beBlc65JWaWCtxPdOaOfcBVzrkt3nu3AdlAMnAAuMA5946ZlfPu1HhPATdpajwRkcRzzvH61n384sUtLN1QQ2qSjyvLp3L53KlMykllfHrSgMYphyOOXQda2VbfzLb6FrbVNbOtrpmt9c1srWvGuWhPe3lxLvOnBzlzWpCTC3OO6QHU2sZ2Xtlcx8uVdbxcWU/1gVYAJuekcuKUHErz0ikOZlASzKAkL53JOWnqwRaRblq0RURE4mLT3kbufnELj6+p7n6I0Qxy05MJZiYTzEghLyuFYEYy+d5rXmYKwcxkstOS2HOwja1eWO4KzzvqW+gIR7p/R2qSLxpqgxmcMDmb+dODzJmaE7e5ubseNH15cx2vbK6nYm8j2+tbaA+9W0Oy38fU3DRK8zK8kJ0eHU8fzGDKOAVtkbFGYVpEROJqb0Mbq7bvp76pndqmDuqb2qlraqe+qaP7tbE9dMT3pwR83b3A0ddoUC3Ny2BCVgq+IQ6rkYhjT0Mb2+qb2d7VO961Xd9MW+e7QdtnMN778pCbEf0C0b2dGf0CEczoOp/CuLSkIf88IhJfCtMiIjLk2jrD1Dd3UNfYTn1zOwdaOpmUk0ppXgYTs1JHTMCMRBw1je3RnvS6ZqoPtFLfHP0Csa+5g/qmDuqbOzjYeviHRP0+o2BctJe7NC/ay12an0lpMIOC8erlFhkJ4j3PtIiISJ9Sk/wUjEs74tSAI4XPZ0zKSWVSTipnTgsesV1nOML+5g4vaHdQ3/xuT/2OfdEe7pXb9tHcEe5+z7vDSTIpzUunNC+Tkrx0puSkkZ7iJyM5QFqSf8R88RjODrZ08sTbuzCMS08rSPjMNDJ6KUyLiIj0Q5Lfx4TsVCZkpx6xjXOO2qZ2ttZGh5Fs6Ro/XtfCixW1dPQYt91TWpKfjBQ/6ckB0pP9ZKREX9OTo4E7PcVPbkYKeZneGHVvyEleZjI5aUkxz1jinKOhNUSd90Wgvqm9+8tBa2eY0rx0ZkzIomxiJtmpx74q6FALhSO8WFHLo6uq+fs7e7vH5v/k2U189tzpXD2vKK6h+q2qA/z+tR2Y0T1jTO4QzXwjw4eGeYiIiCRAJOLY3dDG1tpm9ja00dIZprUjRHN7mJaOEM0dYVo7wjS3h2jpCNPcEYrud4RoagtxoLWTw/0nPOCz7odDg17YzstMJhyB+uboUJW6pneHrYQih88BSX7rfugUoosAlU3MZMaE6AqaZRMzKZuQybj0xIfHDXsaeHRVFY+t3kVdUzu5GclcPGcKl88tpLEtxG3PbeK1LfuYkJUy4FDdGY7w9Lo93LtsK2/sOEBGsh+/z2hoC2EGJ07JZsGMfBaWaRGhkUxjpkVEREa5UDjC/pbOQ4aXdIXk7v3mdx8W9Zl1z7QS9Hq1c3v0aHeF72BGMuMzkvGZUb2/lYqaRipqmqjY20Slt93SY+hKXmYKZRMyu8P1dC9s52UmD+qc3vuaO3h8TTWPvlHF2uoGAj7jvTMncNncQs47fkL3Sp9dXt1cP6BQvb+5gz+s2MH9r25n98E2ioPpXDe/hCvKC0lL8vNW9UGWVdSxrKKON3bsJxRxpCb5mFcaZOGMPBaU5TFzUtag/m/inKOuqYOKmkZwMK80d0DTWI5lCtMiIiIyKCIRx66DrVTUNFG5t+mQsN3UY0aXnLSk7pA9PT+TsolZzJiQyZSc1H4Hyo5QhOc31vDoqiqWbqghFHHMLsjmstMKuXjOFIKZKX1e41hD9cY9jfzmla08trqats4IZ88I8omzSjlv5oQjPkza1B5i+ZZ6XqqoY1llXffKnXmZKSyYEWReaZDJOanRfzXIin6R6R3++1LX1M6mvY1U1jSxaW8jm/Y2UbG3kf0t7z4Ym5eZwsVzpnDpaQWcOCVbCxYdA4VpERERGVLOOfY2tFNREw14PcN2z4CXkexn+oTocJFJ2al0hCK0hcK0dUZo64y+tofCtHWGaQ+9e6ytM0xjW4jWzjB5mSlccuoULptbyMxJ2f2q92ihOhJxLN1Qw69f2crLlfWkBHxceloBHz+rlOMnZR3z79p9sDXaa10Z7bmub+74pzY5aUnRfyHITCE/893x8HlZKeRmJFPT0MamvdHgXFHTxL4e18hKCVA2MZPjJmZRNjGLsgmZtHSEeGx1NUs31NAZdpRNyOTS0wr50KlTmJwzsh8SHgoK0yIiIjJs1De1vxuwe/zUNrWTEvCRmuQn1XtNSfKTmuTrcTy6n5rkJy3Zz8KyPM4py4/b8IWeoTo/K4XFc6bw9/V72V7fwuScVD42v5irTy9ifJweLIxEHNUHWqltaqeuMTo0p84bilPX1E5dYwd1zdFzDW2Hzt3eFZq7xqwfNzGL4yZmMTE75Yi9zgdaOnjird08trqaVdv3YwbzpwW55NQCLjppMpkpiZuLojMcobk9+nxAS3uIJu/5gOhr9PmB0rwMzp6RN+S1KUyLiIiIHIOeoXpu8Xg+cXYJF5446ZiWsY+39lCY+qYO9jV3EMxMZlJ2/4fIAGyra+ax1dX8eU012+tbSE3yccGsSVxyWgELZ+R1f0HpDEdoaO2ksS1EQ1snDa1drz2PddLaGSYUdoQijlAk0mPbEQpHul/DEUdn2NEZjnQ/TNvSHj5kJdQjuXxuIT+8Yk6/P3N/KUyLiIiI9MPB1k5y0ob/NIAD4ZzjjR37+dMb1Tzx1m4OtnYyLj2JlICPhtboUJqj8RlkpSaRluQn4DcCPiPg93mvht/nI8ln+H1Gkt/nvRoBn697PvWMlAAZ3rSPXdNAZqa8OxVk1/ms1CTSkod+RhSFaRERERHpU3sozPMbanl2/V78ZmSnBchOTSIrNUB2WhLZqUnR17QAWalJZKcGyEgOjPqFhbQCooiIiIj0KSXgZ9HsSSyaPSnRpYxYmnxQRERERKSfFKZFRERERPpJYVpEREREpJ8UpkVERERE+klhWkRERESknxSmRURERET6SWFaRERERKSfFKZFRERERPppRK2AaGa1wPYE/Oo8oC4Bv1cSQ/d77NE9H1t0v8cW3e+xJV73u9g5lx9LwxEVphPFzFbGuqSkjHy632OP7vnYovs9tuh+jy2JuN8a5iEiIiIi0k8K0yIiIiIi/aQwHZu7E12ADCnd77FH93xs0f0eW3S/x5Yhv98aMy0iIiIi0k/qmRYRERER6SeF6T6Y2SIz22hmlWZ2c6Lrkfgys3vNrMbM1vY4lmtmfzezCu91fCJrlPgxs6lm9ryZrTezdWb2Re+47vkoZGapZva6mb3p3e//8o6Xmtly734/ZGbJia5V4sfM/Ga22sye8PZ1v0cxM9tmZm+b2RozW+kdG9K/6QrTR2FmfuBO4CJgFnC1mc1KbFUSZ78BFvU6djPwnHOuDHjO25fRIQR8xTl3AnAm8Hnv/9O656NTO/Be59wc4BRgkZmdCdwC3Ord7/3A9QmsUeLvi8D6Hvu636Pfec65U3pMiTekf9MVpo9uHlDpnNvinOsAHgQWJ7gmiSPn3IvAvl6HFwP3edv3AR8a0qJk0Djndjvn3vC2G4n+B7cA3fNRyUU1ebtJ3o8D3gs84h3X/R5FzKwQeD/wK2/f0P0ei4b0b7rC9NEVADt77Fd5x2R0m+ic2w3R8AVMSHA9MgjMrAQ4FViO7vmo5f2T/xqgBvg7sBk44JwLeU30d310+Qnwb0DE2w+i+z3aOeAZM1tlZjd4x4b0b3pgMC8+Cthhjmn6E5ERzswygUeB/+2ca4h2Xslo5JwLA6eY2TjgMeCEwzUb2qpkMJjZB4Aa59wqMzu36/Bhmup+jy5nO+d2mdkE4O9mtmGoC1DP9NFVAVN77BcCuxJUiwydvWY2GcB7rUlwPRJHZpZENEj/3jn3J++w7vko55w7ALxAdKz8ODPr6kzS3/XR42zgYjPbRnRY5nuJ9lTrfo9izrld3msN0S/M8xjiv+kK00e3AijzngROBq4CliS4Jhl8S4DrvO3rgMcTWIvEkTd+8h5gvXPuxz1O6Z6PQmaW7/VIY2ZpwL8QHSf/PHC510z3e5Rwzn3DOVfonCsh+t/rpc65a9D9HrXMLMPMsrq2gQuAtQzx33Qt2tIHM3sf0W+2fuBe59x3E1ySxJGZ/QE4F8gD9gL/CfwZeBgoAnYAVzjnej+kKCOQmS0AXgLe5t0xlf+H6Lhp3fNRxsxOJvrwkZ9o59HDzrlvm9k0oj2XucBq4KPOufbEVSrx5g3z+Kpz7gO636OXd28f83YDwAPOue+aWZAh/JuuMC0iIiIi0k8a5iEiIiIi0k8K0yIiIiIi/aQwLSIiIiLSTwrTIiIiIiL9pDAtIiIiItJPCtMiIiOAmYXNbE2Pn5vjeO0SM1sbr+uJiIwlWk5cRGRkaHXOnZLoIkRE5FDqmRYRGcHMbJuZ3WJmr3s/M7zjxWb2nJm95b0WeccnmtljZvam93OWdym/mf3SzNaZ2TPeioEiItIHhWkRkZEhrdcwjyt7nGtwzs0D7iC6Yive9m+dcycDvwd+6h3/KfAP59wc4DRgnXe8DLjTOXcicAC4bJA/j4jIqKAVEEVERgAza3LOZR7m+Dbgvc65LWaWBOxxzgXNrA6Y7Jzr9I7vds7lmVktUNhzOWUzKwH+7pwr8/a/DiQ5574z+J9MRGRkU8+0iMjI546wfaQ2h9PeYzuMnqkREYmJwrSIyMh3ZY/XV73tV4CrvO1rgGXe9nPAZwHMzG9m2UNVpIjIaKSeBxGRkSHNzNb02P+bc65rerwUM1tOtIPkau/YF4B7zexrQC3wCe/4F4G7zex6oj3QnwV2D3r1IiKjlMZMi4iMYN6Y6XLnXF2iaxERGYs0zENEREREpJ/UMy0iIiIi0k/qmRYRERER6SeFaRERERGRflKYFhERERHpJ4VpEREREZF+UpgWEREREeknhWkRERERkX76/zk75hXeFf5YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize = (12,4))\n",
    "ax.plot(losses.MSE)\n",
    "ax.set_xlabel('MSE')\n",
    "ax.set_xlabel('Epoch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}