04.Dueling_DQN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dueling Deep Q Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from IPython.display import clear_output\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from timeit import default_timer as timer\n",
    "from datetime import timedelta\n",
    "import math\n",
    "\n",
    "from utils.wrappers import *\n",
    "from agents.DQN import Model as DQN_Agent\n",
    "from utils.ReplayMemory import ExperienceReplayMemory\n",
    "\n",
    "from utils.hyperparameters import Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = Config()\n",
    "\n",
    "config.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "#epsilon variables\n",
    "config.epsilon_start = 1.0\n",
    "config.epsilon_final = 0.01\n",
    "config.epsilon_decay = 30000\n",
    "config.epsilon_by_frame = lambda frame_idx: config.epsilon_final + (config.epsilon_start - config.epsilon_final) * math.exp(-1. * frame_idx / config.epsilon_decay)\n",
    "\n",
    "#misc agent variables\n",
    "config.GAMMA=0.99\n",
    "config.LR=1e-4\n",
    "\n",
    "#memory\n",
    "config.TARGET_NET_UPDATE_FREQ = 1000\n",
    "config.EXP_REPLAY_SIZE = 100000\n",
    "config.BATCH_SIZE = 32\n",
    "\n",
    "#Learning control variables\n",
    "config.LEARN_START = 10000\n",
    "config.MAX_FRAMES=1000000\n",
    "\n",
    "#Nstep controls\n",
    "config.N_STEPS=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DuelingDQN(nn.Module):\n",
    "    def __init__(self, input_shape, num_outputs):\n",
    "        super(DuelingDQN, self).__init__()\n",
    "        \n",
    "        self.input_shape = input_shape\n",
    "        self.num_actions = num_outputs\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(self.input_shape[0], 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "\n",
    "        self.adv1 = nn.Linear(self.feature_size(), 512)\n",
    "        self.adv2 = nn.Linear(512, self.num_actions)\n",
    "\n",
    "        self.val1 = nn.Linear(self.feature_size(), 512)\n",
    "        self.val2 = nn.Linear(512, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        adv = F.relu(self.adv1(x))\n",
    "        adv = self.adv2(adv)\n",
    "\n",
    "        val = F.relu(self.val1(x))\n",
    "        val = self.val2(val)\n",
    "\n",
    "        return val + adv - adv.mean()\n",
    "    \n",
    "    def feature_size(self):\n",
    "        return self.conv3(self.conv2(self.conv1(torch.zeros(1, *self.input_shape)))).view(1, -1).size(1)\n",
    "    \n",
    "    def sample_noise(self):\n",
    "        #ignore this for now\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(DQN_Agent):\n",
    "    def __init__(self, static_policy=False, env=None, config=None):\n",
    "        super(Model, self).__init__(static_policy, env, config)\n",
    "\n",
    "    def declare_networks(self):\n",
    "        self.model = DuelingDQN(self.env.observation_space.shape, self.env.action_space.n)\n",
    "        self.target_model = DuelingDQN(self.env.observation_space.shape, self.env.action_space.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards, losses, sigma, elapsed_time):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s. time: %s' % (frame_idx, np.mean(rewards[-10:]), elapsed_time))\n",
    "    plt.plot(rewards)\n",
    "    if losses:\n",
    "        plt.subplot(132)\n",
    "        plt.title('loss')\n",
    "        plt.plot(losses)\n",
    "    if sigma:\n",
    "        plt.subplot(133)\n",
    "        plt.title('noisy param magnitude')\n",
    "        plt.plot(sigma)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAE/CAYAAADWnCU3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xt8XHd57/vPo7tly5JGvlu2NWlCQu6JrTGUUiiEJmWzCXRDmx52CgdomgM9u1DOBlL2aXug2aeUQ8uhKU1TLqX7pGWzgZScEiChUC5NsXxJ4jhxEpxIjjW2Y8XSSLI1us6z/1hrnIkykiXNSGsu3/frNS/PrPVbM89Iy5r1zO/3/H7m7oiIiIiIiCxETdQBiIiIiIhI+VACISIiIiIiC6YEQkREREREFkwJhIiIiIiILJgSCBERERERWTAlECIiIiIismBKIFaAmV1sZg+b2aiZ/aeo45HlZWZ9ZnZd1HEUm5m9w8zujzoOEZFSV6mfAyJZSiBWxoeBH7h7i7t/NupgZjOzu8zsSTPLmNm78uz/oJmdNLMRM/uimTXm7Osysx+Y2ZiZPTH7D2Yhx1YjM3uFmT1gZoNmNmBm/8PMNufsNzP7pJmdDm+fNDOb5/nWm9nfm9mwmQ2Z2d0LjKPLzNzM6rLb3P1ud//lwt7h0iz2XCnWeWdml5vZd83seTPLu2iOmd1kZofN7KyZPW1mr176OxURESl9SiBWxg7gsbl2mlntCsaSzyPA+4ADs3eY2fXAR4HXE7yPC4D/K6fJPwAPAR3Ax4Cvmdn6Qo9djNyL3JW0TK/bDtwFdBH8zEaBL+XsvwV4C3AVcCXw74Hfnuf5vgGcBLYDG4D/p+gRr4wFnytFPu+mgK8C75njtd4AfBL4X4EW4BeBZxbzxkRERMqOu+u2jDfg+8AMMA6cAV4G/C3wV8B9wFngOuDfEVzUjADHgD/KeY4uwAkuUo4BQ8CtQDdwEEgBd8x63XcDh8O23wV2LCDWnwDvmrXt74H/mvP49cDJ8P7LgAmgJWf/j4FbCz12AbH2AR8J3/8EUAdsAb4ODAC9wH8K2zYBaWBd+PhjwDSwNnz8CeAz4f2F/B7eAzwL/CjcfjNwFDgdPncfcF2Rzp9rgdGcxw8Ct+Q8fg/w0zmO/eUwltolvO6z4Xs9E95eCbwL+ElOGydIPH9GkOh8Avi5MMYRggvvhpz2bwIeDs/XB4ErFxjLos6V5TjvgAsBz7P9QeA9xfhd66abbpVzy34OAI3AZ4Dj4e0zQGPYZh3wT+HfxMHwb1FNuO8jQDL82/ok8Pqo35NuuuXe1AOxzNz9dQR/FH7H3de4+1Phrv8FuJ3gW8ufECQSvwm0EVzE/m9m9pZZT7cbuAj4dYI/Qh8j+AN1GfBrZvYaADO7Efh94FeB9eHr/8MS38JlBD0UWY8AG82sI9z3jLuPztp/WRGOXYjfIPhZtQEZ4P8Pn2MrwUXjB8zsencfB/YCrwmPew3BBf+rch7/MLy/kN/Da4CXA9eb2aUEyeDNBAlMB9CZbWhmv2BmqUW8p9l+kRf3XuX7mc71M3sFwQfPl8PhTnuz58gCXxegLTxv/22OdtcDO8PX+jBB78l/BLYBlxP8jjCza4AvEvSWdAB/DdybHVpkZp8zs8/N8Rrznit5fsZLPu8W8/sKew53AevN7IiZ9ZvZHWa2aiHHi0hV+BjB38erCXqOE8B/Cfd9COgn+JzeSPC57WZ2MfA7QLe7txD8ne1b2bBF5qcEIjrfdPd/dfeMu4+7+7+4+6Ph44MEF/yzL/Y+Eba9n+BC9x/c/ZS7JwmShGvCdrcC/7e7H3b3aeC/Aleb2Y4lxLkGGM55nL3fkmdfdn9LEY5diM+6+zF3TxP0xqx394+7+6S7PwP8DXBT2PaHwGvCYUdXAp8NHzeFx/4IYIG/hz9y97Ph674N+Cd3/5G7TwD/J0EyQ/h8P3H3tkW8p3PM7ErgD4D/nLM53890zRx1EJ0EvRA/ADYBnwa+aWbrlhLPHP7U3Ufc/THgEHC/uz/j7sPAt3nhnLwF+Gt33+PuM+7+ZYKegFcAuPv73P19c7zGvOdKnp/xks+7Rf6+NgL1BOfAqwkuEK7hhYsDEZF3AB8PP6sHCIZT3hzumwI2E4wQmHL3H7u7E4xaaAQuNbN6d+9z96cjiV5kDkogonMs94GZ7Q4LOwfMbJggCZh9ofdczv10nsdrwvs7gP/XzFLht6mDgBF8M79YZ4C1OY+z90fz7Mvuz367W8ixC5H7M9wBbMm+5/B9/z7BRR4ECcRrCYYEPQo8QJAYvAI44u6nYcG/h9zX3ZL72N3PEgxlOi8z225mZ7K3WfsuJLgA/113/3HOrnw/0zPhh85saaDP3b8Qfjh9JYz1VXnaLtVizskPzfr9bCP4+Z3PYs+V5T7vstLhv3/h7ifc/Xngz4A3LuG5RKQybSHo8c46ygt/9z4FHAHuN7NnzOyjAO5+BPgA8EfAKTP7ipkt5G+lyIpRAhGd2Rd8fw/cC2xz91bgToKL/qU4Bvy2u7fl3Fa5+4NLeK7HCLpds64CngsvuB8DLjCzlln7HyvCsQuR+zM8BvTOes8t7p69mHsQuBh4K/BDd3+coLD4jbwwfAkW9nvIfd0TBBfCAJhZM8EQnfMH7/5sODxojbtnL7QJe4q+R9Dj9N9mHZbvZzrXz+wgLz3P8s4klC+8BbZbqGPA7bN+P83uvpChdYs9V5b7vAPA3YcIhh/k/qyK/XMTkfJ2nOALlKzt4TbcfdTdP+TuFwBvBn7PzF4f7vt7d/+F8FgnmKxBpGQogSgdLcCgu4+bWYKgRmKp7gRuM7PsuO5WM3v7XI3NrCEcymNAvZk1mVn23Pg74D1mdqmZtREMz/hbgLCe42HgD8Nj3kowPOjrRTh2sXqAUTP7iJmtMrPacArO7vD1xoD9wPt5IWF4kKCHITeBWOzv4WvAm8Kx8w3Axyng/5WZbSUovL/D3e/M0+TvCD5ktobfSH2I8Geaxz1Au5m9M/x5vI1gWNO/LiCUAYKhWBcs9j3M4W+AW8MeHjOz1Wb272ZdyOe1hHOlaOddGGsT0BA+brKcKWEJZsj6381sg5m1Ax8kKIoUEYFgGOx/sWBK7XUEw1L/PwAze5OZXRgOQR0mGLqUsWDtqNeFf2vGCXo7M3M8v0gklECUjvcBHzezUYI/MF9d6hO5+z0E31Z8xcxGCMam/8o8h9xP8Afq5wmKYNOERbTu/h3gTwnG0T9L0P36hznH3kRQSDoE/AnwtnCcZ0HHWrBo2YK/FXb3GYJZfq4mmIHpeeDzQGtOsx8SjFnvyXncQlj/EFrU7yEc+/9+gp6LE+F76c/uN7NXzx6edB7vJbho/6M5hjf9NUGx+KMEv9dvhduyr3fGwnUI3H2Q4Fut/4Pgw+mjwI3hUBvM7E4zy5ekZBOu24F/DYccvWIR7yHf8+0Dfgu4g+BndIRgVqds3HPGEprvXHnRz7jA827272sHwf+H7LmYJihMz/oEQYH+UwSznj1E8HMTEQH4Y2AfQY/wowTTpf9xuO8igt7mM8C/AZ9z9x8Q1D/8CcHn2EmCKbhvW9mwReZn+YdOi4iIiIiIvJR6IEREREREZMGUQIiIiIiIyIIpgRARERERkQVTAiEiIiIiIgumBEJERERERBasLuoAimndunXe1dUVdRgiIiVp//79z7v7+qjjiJI+J0RE8lvMZ0RFJRBdXV3s27cv6jBEREqSmR2NOoao6XNCRCS/xXxGaAiTiIiIiIgsmBIIERERERFZMCUQIiIiIiKyYEogRERERERkwZRAiIiIiIjIgimBEBERERGRBVMCISIiIiIiC1ZwAmFmbzezx8wsY2a7crY3mNmXzOxRM3vEzF47x/ExM3vAzH4W/tsebjcz+6yZHTGzg2Z2baGxioiIiIhIYYrRA3EI+FXgR7O2/xaAu18BvAH4tJnle72PAv/s7hcB/xw+BvgV4KLwdgvwV0WIVUREREREClBwAuHuh939yTy7LgW+H7Y5BaSAXXna3Qh8Obz/ZeAtOdv/zgM/BdrMbHOh8ebz+PER7t5T9Qu0ikgJm5zO8OUH+3h64EzUoYiISJVbzhqIR4A3m1mdmcWBncC2PO02uvuJ8P5JYGN4fytwLKddf7jtRczsFjPbZ2b7BgYGlhToA48/x3/5x0MMp6eWdLyIyHI7MZzmD+99jP1Hh6IORUREqtyCEggz+56ZHcpzu3Gew75IcNG/D/gM8CAwM9/ruLsDvsDYs8fc5e673H3X+vXrF3PoOd3xdtzhgD6YRaREJYfSAHS2rYo4EhERqXZ1C2nk7tct9ondfRr4YPaxmT0IPJWn6XNmttndT4RDlE6F25O8uMeiM9xWdNdsa6e+1tjTO8gvXbJhOV5CRKQg/akggdjargRCRESitWxDmMys2cxWh/ffAEy7++N5mt4LvDO8/07gmznbfzOcjekVwHDOUKeiWtVQy5WdbfT0nl6OpxcRKVhyKI0ZbGptijoUERGpcsWYxvWtZtYPvBL4lpl9N9y1AThgZoeBjwA35xzz+ZwpX/8EeIOZ/Qy4LnwMcB/wDHAE+BvgfYXGOp9EPMbB/mHSk/OOshIRiUQylWb9mkYa62qjDkVERKrcgoYwzcfd7wHuybO9D7h4jmPem3P/NPD6PG0ceH+h8S1UIh7jr/7laR56doifv3DdSr2siMiCJIfSGr4kIiIlQStRh3buaKfGYE/vYNShiIi8RDKVZqsKqEVEpAQogQitbarn5ZvXsrdPCYSIlJZMxjkxrB4IEREpDUogciTiMQ48O8TkdCbqUEREzjk1OsHUjGsKVxERKQlKIHLsjscYn8rwaHI46lBERM5JagpXEREpIUogcnR3xQDoUR2EiJSQcwlEW3PEkYiIiCiBeJGONY1cuGGN1oMQkZKSXYVaPRAiIlIKlEDMkojH2Nc3xEzGow5FRASAZGqM1lX1rGkseOZtERGRgimBmCXRFWN0YponTo5EHYqICBCuAaECahERKRFKIGZJxFUHISKlJZnSFK4iIlI6lEDMsqVtFZ3tq5RAiEhJcHf1QIiISElRApFHIh6jp3cQd9VBiEi0htNTnJ2cUQIhIiIlQwlEHrvjMU6fneTpgbNRhyIiVa5fMzCJiEiJUQKRRyLeAagOQkSi98IaEEogRESkNCiByKOro5l1axrZ26cEQkSipTUgRESk1CiByMPM2B3WQYiIRCmZStNUX0PH6oaoQxEREQGUQMwpEY+RTKXpHxqLOhQRqWLHU2m2tK3CzKIORUREBFACMSetByEipSCZ0hSuIiJSWpRAzOHijS2sbapTAiEikUoOpelU/YOIiJQQJRBzqKmxc+tBiIhEIT05w+mzk+qBEBGRkqIEYh7dXTGeef4sA6MTUYciIlXo3BSu6oEQEZESogRiHtk6CE3nKiJReGENiOaIIxEREXlBQQmEmb3dzB4zs4yZ7crZ3mBmXzKzR83sETN77RzHf8rMnjCzg2Z2j5m1hdu7zCxtZg+HtzsLiXOpLt/ayqr6Wg1jEpFIaA0IEREpRYX2QBwCfhX40aztvwXg7lcAbwA+bWb5XusB4HJ3vxJ4CrgtZ9/T7n51eLu1wDiXpL62hp072tmjBEJEIpBMjVFbY2xsaYw6FBERkXMKSiDc/bC7P5ln16XA98M2p4AUsGt2I3e/392nw4c/BToLiWc5JOIxnjg5wnB6KupQRKTKJIfSbFrbRF2tRpuKiEjpWK5PpUeAN5tZnZnFgZ3AtvMc827g2zmP42b2kJn90MxevUxxnlciHsMd9h9VL4SIrCytASEiIqXovAmEmX3PzA7lud04z2FfBPqBfcBngAeBmXle42PANHB3uOkEsN3drwF+D/h7M1s7x7G3mNk+M9s3MDBwvrezaFdva6O+1jSMSURWXHIoXfH1D2a2zcx+YGaPhzV1v5unzY1hrdzD4d/7X8jZN5NTL3fvykYvIlKd6s7XwN2vW+yThsOSPph9bGYPEtQ4vISZvQt4E/B6d/fw+AlgIry/38yeBl5GkJDMfq27gLsAdu3a5YuN9Xya6mu5qrNNhdQisqKmZzKcHBmvhh6IaeBD7n7AzFqA/Wb2gLs/ntPmn4F73d3N7Ergq8Al4b60u1+9wjGLiFS1ZRnCZGbNZrY6vP8GYHrWh0G23Q3Ah4E3u/tYzvb1ZlYb3r8AuAh4ZjliXYhEPMaj/cOMTU6fv7GISBGcHBkn45U/A5O7n3D3A+H9UeAwsHVWmzPZL5iA1UDRvywSEZGFK3Qa17eaWT/wSuBbZvbdcNcG4ICZHQY+Atycc8znc6Z8vQNoAR6YNV3rLwIHzexh4GvAre4eWRdAIh5jOuM89GwqqhBEpMqcm8K18nsgzjGzLuAaYE+efW81syeAbxHUzGU1hcOafmpmb5njeZd1qKuISLU57xCm+bj7PcA9ebb3ARfPccx7c+5fOEebrwNfLyS2Ytq5o50agz29g7zqwnVRhyMiVaDaVqE2szUEf/c/4O4js/dnP2/M7BeBTwDZ4bU73D0Z9lZ/38wedfenZx27rENdRUSqjeYGXICWpnou29LKXtVBiMgKqaYeCDOrJ0ge7nb3b8zX1t1/BFxgZuvCx8nw32eAfyHowRARkWWkBGKBurtiHHh2iMnpTNShiEgVSKbSrFvTQFN9bdShLCszM+ALwGF3/7M52lwYtsPMrgUagdNm1m5mjeH2dcCrgJfU24mISHEpgVigRDzGxHSGR5OqgxCR5VdFa0C8iqBO7nU507G+0cxuNbNbwzb/ATgU1sX9JfDrYVH1y4F9ZvYI8APgT/JN2CEiIsVVUA1ENenuageCOoidO2IRRyMilS45lOaSzS1Rh7Hs3P0ngJ2nzSeBT+bZ/iBwxTKFJiIic1APxAJ1rGnkog1rtB6EiCw7dyeZSrOltSp6IEREpMwogViERDzG/r4hZjKaxENEls/zZyaZmM5UzQxMIiJSXpRALEIiHmN0YprDJ14yw6CISNGcm8K1OmogRESkzCiBWIREPKh90DAmEVlO56ZwVQ+EiIiUICUQi7C5dRXbYquUQIjIsjoe9kB0tjVHHImIiMhLKYFYpERXBz19gwQzCIqIFF8ylWZNYx1rV2miPBERKT1KIBZpdzzG4NlJnh44E3UoIlKh+oeCNSDCtdNERERKihKIRcrWQezRMCYRWSbJVFr1DyIiUrKUQCzSjo5mNrQ0slcJhIgsk+TQmGZgEhGRkqUEYpHMjO54jD29qoMQkeIbHZ9iZHxaPRAiIlKylEAswe54jBPD4/SHUy2KiBSL1oAQEZFSpwRiCbQehIgsl+waEFuUQIiISIlSArEEL9vQQuuqeiUQIlJ02R6ITg1hEhGREqUEYglqaozurhg9fUogRKS4kkNpGmprWL+mMepQRERE8lICsUS74zF6nz/LqdHxqEMRkQrSn0qzua2JmhqtASEiIqVJCcQSdYd1EHt7hyKOREQqyfFUWgXUIiJS0pRALNFlW9bS3FBLT+/pqEMRkQqSHFICISIipU0JxBLV19awc0e7VqQWkaKZmJ7h1OiE1oAQEZGSVlACYWZvN7PHzCxjZrtytjeY2ZfM7FEze8TMXjvH8X9kZkkzezi8vTFn321mdsTMnjSz6wuJc7kkumI8+dwoqbHJqEMRkQpwIhXUVKkHQkRESlmhPRCHgF8FfjRr+28BuPsVwBuAT5vZXK/15+5+dXi7D8DMLgVuAi4DbgA+Z2a1BcZadIl4DHfY16c6CBEp3LlF5NQDISIiJaygBMLdD7v7k3l2XQp8P2xzCkgBu/K0m8uNwFfcfcLde4EjQKKQWJfDVdvaaKitYa+mcxWRIsguItfZ1hxxJCIiInNbrhqIR4A3m1mdmcWBncC2Odr+jpkdNLMvmll7uG0rcCynTX+4raQ01ddy1bZW1UGISFH0p9KYwabWpqhDERERmdN5Ewgz+56ZHcpzu3Gew75IcNG/D/gM8CAwk6fdXwE/B1wNnAA+vdg3YGa3mNk+M9s3MDCw2MMLlojHOJQc5uzE9Iq/tohUluRQmo0tTTTUaX4LEREpXef9lHL369z98jy3b85zzLS7fzCsa7gRaAOeytPuOXefcfcM8De8MEwpyYt7LDrDbfle6y533+Xuu9avX3++t1N0iXgH0xnnoWdTK/7aIlJZkqkxtrSp90FERErbsnzNZWbNZrY6vP8GYNrdH8/TbnPOw7cSFGUD3AvcZGaN4RCoi4Ce5Yi1UDt3tFNjaD0IESlYMpVma7vqH0REpLQVOo3rW82sH3gl8C0z+264awNwwMwOAx8Bbs455vM5U77+aTjV60Hgl4APArj7Y8BXgceB7wDvd/d8Q6Ait6axjsu3qg5CRAozk3FOpMY1hauIiJS8ukIOdvd7gHvybO8DLp7jmPfm3L85X5tw3+3A7YXEt1ISXTH+20+PMjE9Q2Ndyc02KyJl4NToONMZ1xSuIiJS8lSpVwTd8RgT0xke7R+OOhQRKVPHU9kpXJVAiIhIaVMCUQTdXTEADWMSkSXrH9IiciIiUh6UQBRBbHUDL9u4hh4lECKyROdWoVYPhIiIlDglEEWSiMfYf3SI6ZlM1KGISBlKDqVpa65ndWNBpWkiIiLLTglEkSTiHZyZmObwidGoQxGRMpRMpdX7ICIiZUEJRJEkwjqInj4NYxKRxUsOKYEQEZHyoASiSDa1NrE91qwF5URk0dw9XEROCYSIiJQ+JRBFlIjH6OkdxN2jDkVEykhqbIqxyRn1QIiISFlQAlFEiXiMobEpjpw6E3UoIlJGNAOTiIiUEyUQRbQ7rvUgRGTxtAaEiIiUEyUQRbQ91szGtY1aD0JEFkU9ECIiUk6UQBSRmdHdpToIEVmc46k0TfU1xFY3RB2KiIjIeSmBKLLd8RgnR8bPDUkQETmf7BSuZhZ1KCIiIuelBKLIEvEOQHUQIrJwwRSuzVGHISIisiBKIIrsog1raGuu13oQIrJgWoVaRETKiRKIIqupeaEOQkTkfMYmpxk8O0mnZmASEZEyoQRiGeyOx+g7PcapkfGoQxGREndcMzCJiEiZUQKxDLq7gvUgevrUCyEi89MaECIiUm6UQCyDy7aspbmhVsOYROS8qn0NCDPbZmY/MLPHzewxM/vdPG1uNLODZvawme0zs1/I2fdOM/tZeHvnykYvIlKd6qIOoBLV1dawc0e7EggROa/kUJraGmPj2qaoQ4nKNPAhdz9gZi3AfjN7wN0fz2nzz8C97u5mdiXwVeASM4sBfwjsAjw89l53H1rpNyEiUk3UA7FMdsdjPHFylNTYZNShiEgJS6bSbFrbRG1Nda4B4e4n3P1AeH8UOAxsndXmjL+wOudqgmQB4HrgAXcfDJOGB4AbViZyEZHqpQRimWTXg9jbpy/CRGRuyaG06h9CZtYFXAPsybPvrWb2BPAt4N3h5q3AsZxm/cxKPkREpPgKSiDM7O3hmNWMme3K2d5gZl8ys0fN7BEze+0cx//3cEzrw2bWZ2YPh9u7zCyds+/OQuKMwpWdrTTU1Wg9CBGZVzKVprNK6x9ymdka4OvAB9x9ZPZ+d7/H3S8B3gJ8YpHPfUtYO7FvYGCgOAGLiFSxQmsgDgG/Cvz1rO2/BeDuV5jZBuDbZtbt7pncRu7+69n7ZvZpYDhn99PufnWB8UWmqb6Wqzvb6FEPhIjMYWomw3Mj41XfA2Fm9QTJw93u/o352rr7j8zsAjNbBySB1+bs7gT+Jc8xdwF3Aezatctn7xcRkcUpqAfC3Q+7+5N5dl0KfD9scwpIERS55WVmBvwa8A+FxFNqEvEYh5LDnJ2YjjoUESlBJ4fHyXj1zsAE5/7+fwE47O5/NkebC8N2mNm1QCNwGvgu8Mtm1m5m7cAvh9tERGQZLVcNxCPAm82szsziwE5g2zztXw085+4/y9kWN7OHzOyHZvbqZYpzWSXiMWYyzoFn1QshIi91bgrX6u6BeBVwM/C6nGGrbzSzW83s1rDNfwAOhcNc/xL4dQ8MEgxn2hvePh5uExGRZXTeIUxm9j1gU55dH3P3b85x2BeBlwP7gKPAg8DMPC/zG7y49+EEsN3dT5vZTuAfzeyyfONizewW4BaA7du3n+/trKhrd7RTW2P09A7y6ovWRx2OiJSY5FB1rwEB4O4/AeadgsrdPwl8co59XyT4zBERkRVy3gTC3a9b7JO6+zTwwexjM3sQeCpfWzOrI6ij2Jlz/AQwEd7fb2ZPAy8jSEhmv1bJjm1d01jH5VvWskfrQYhIHtkeiC1VnECIiEj5WZYhTGbWbGarw/tvAKZnLQqU6zrgCXfvzzl+vZnVhvcvAC4CnlmOWJdbIh7j4WMpJqbn64ARkWqUHEqzbk0jTfW1UYciIiKyYIVO4/pWM+sHXgl8y8yyxWsbgANmdhj4CMH41uwxn8+d8hW4iZcWT/8icDAc7/o14NZyHdfa3RVjcjrDwf7h8zcWkaqSTGkNCBERKT8FTePq7vcA9+TZ3gdcPMcx7531+F152nydYEq/stfdFQOgp3fw3H0REQgSiEs3r406DBERkUXRStTLrH11AxdvbFEdhIi8SCbj6oEQEZGypARiBSTiMfb3DTI9kzl/YxGpCs+fnWByOsOW1qaoQxEREVkUJRArIBGPcXZyhsdPvGQWWhGpUuemcG1vjjgSERGRxVECsQIS8RfqIEREIGcROU3hKiIiZUYJxArYuLaJHR3NSiBE5JzjWoVaRETKlBKIFZLoirG3b5BMpqTWuhORiCSH0rQ01tG6qj7qUERERBZFCcQKScRjDI1NcWTgTNShiEgJ0AxMIiJSrpRArJDd8Q4ATecqIgD0D6VV/yAiImVJCcQK2RZbxaa1TaqDEBFAPRAiIlK+lECsEDMjEY/R03sad9VBiFSzkfEpRsen1QMhIiJlSQnECuqOx3huZIJjg+moQxGRCL2wBoQSCBERKT9KIFbQ7nA9iD29pyOORESidC6BUA+EiIiUISUQK+jC9Wtob65XHYRIlUtqDQgRESljSiBWUE2N0d0Vo6dPCYRINUum0jTU1rBudWPUoYiIiCyaEogVlojHOHp6jJPD41GHIiIRSQ6l2dLWRE30gnp5AAAgAElEQVSNRR2KiIjIoimBWGHZ9SDUCyFSvTSFq4iIlDMlECvs5ZtbWN1Qy17VQYhUrWRKi8iJiEj5UgKxwupqa9jZFVMhtUiVGp+aYWB0gq1tzVGHIiIisiRKICKwOx7jyedGGTo7GXUoIrLCToT1TxrCJCIi5UoJRAQS4XoQe1UHIVJ1tAaEiIiUOyUQEbiys5WGuhoNYxKpQsnUGACd6oEQEZEypQQiAo11tVyzrU09ECJVKDmUpsZgU2tT1KGIiIgsScEJhJl9ysyeMLODZnaPmbXl7LvNzI6Y2ZNmdv0cx8fNbE/Y7r+bWUO4vTF8fCTc31VorKUkEY9x6PgIZyamow5FRFZQfyrNxrVN1Nfq+xsRESlPxfgEewC43N2vBJ4CbgMws0uBm4DLgBuAz5lZbZ7jPwn8ubtfCAwB7wm3vwcYCrf/ediuYiTiMWYyzoGjQ1GHIiIrKDmkKVxFRKS8FZxAuPv97p79Gv2nQGd4/0bgK+4+4e69wBEgkXusmRnwOuBr4aYvA2/JOf7L4f2vAa8P21eEa7e3U1tjqoMQqTJaRE5ERMpdsfvQ3w18O7y/FTiWs68/3JarA0jlJCC5bc4dH+4fDttXhNWNdVy+tZU9vaejDkUqyINHnueWv9vH9Ewm6lAkj5mMc3J4nC3qgRARkTK2oATCzL5nZofy3G7MafMxYBq4e7mCnSO2W8xsn5ntGxgYWMmXLtjueIxHjg0zPjUTdShSIf7H/n7uf/w5Dp8YjToUyeO5kXGmM64hTCIiUtYWlEC4+3Xufnme2zcBzOxdwJuAd7i7h4clgW05T9MZbst1Gmgzs7o8bc4dH+5vDdvPju0ud9/l7rvWr1+/kLdTMhJdMSZnMhzsH446FKkQ2SFxPZrhqyQdT4VrQGgIk4iIlLFizMJ0A/Bh4M3uPpaz617gpnA2pThwEdCTe2yYbPwAeFu46Z3AN3OOf2d4/23A93OSk4qwq6sdgB4NY5Ii6B8aIxleoOqcKk3Z30+neiBERKSMFaMG4g6gBXjAzB42szsB3P0x4KvA48B3gPe7+wyAmd1nZlvC4z8C/J6ZHSGocfhCuP0LQEe4/feAjxYh1pLS1tzAJZta2KNCaimC7Loil29dS0/vIBWWb1eE/iH1QIiISPmrO3+T+YXTrM6173bg9jzb35hz/xlmzc4Ubh8H3l5ofKUuEY/xtf39TM9kqNO88FKAnt5B1jbV8R937+Cj33iUI6fOcNHGlqjDkhzJVJr25nqaGwr+0ysiIhIZXbFGLBGPMTY5w2PHR6IORcrcnt5BurtivOKCjnOPpbQkhzSFq4iIlD8lEBFLdMUAtB6EFGRgdIJnBs6SiMfY0dHMhpZGnVMlKJnSInIiIlL+lEBEbMPaJuLrVmvWHCnIvvD8ScRjmBmJeEx1ECXG3cNVqJujDkVERKQgSiBKQHdXO3v7BslkdLEnS7Ond5BV9bVcvrUVCNYYOTkyfq5oV6I3NDZFempGQ5hERKTsKYEoAYl4B6mxKX526kzUoUiZ6ukdZOeOdurDQvxEXHUQpSaZnYFJQ5hERKTMKYEoAbvj2ToIzd0vizecnuLwyRG6w3oagIs2rKF1Vb3OqRKSTAXL5CiBEBGRcqcEogR0tq9ic2uTvi2WJdl/dBD3oP4hq6bG6O6KqZC6hGgNCBERqRRKIEqAil6lEHt6B6mvNa7Z3vai7bvjMfpOj3FqZDyiyCTX8dQ4q+praW+ujzoUERGRgiiBKBGJeIxToxMcPT0WdShSZnp6B7mqs42m+toXbc/2SGiGr9KQTI2xtX0VZhZ1KCIiIgVRAlEizq0HoYs9WYT05AyP9g+/aPhS1mVb1tLcUKthTCVCa0CIiEilUAJRIi7csIbY6gZd7MmiPPTsENMZpztPAlFXW8POHe06p0qEVqHOz8y2mdkPzOxxM3vMzH43T5t3mNlBM3vUzB40s6ty9vWF2x82s30rG72ISHVSAlEizIzuLl3syeLs6R2kxmDnjva8+xNdMZ44OUpqbHKFI5NcY5PTDI1NqQciv2ngQ+5+KfAK4P1mdumsNr3Aa9z9CuATwF2z9v+Su1/t7ruWP1wREVECUUIS8Q6eHRzjxLAW/5KF6ekd5NIta1nblL8wNzu0aW/f0EqGJbNk14DoVA/ES7j7CXc/EN4fBQ4DW2e1edDdsyfxT4HOlY1SRERyKYEoIS+sB6FeCDm/yekMB54dItHVMWebq7a10VBbo/UgItaf0iJyC2FmXcA1wJ55mr0H+HbOYwfuN7P9ZnbL8kUnIiJZSiBKyMs3r2VNY50SCFmQR5MpJqYzeQuos5rqa7l6Wxs96oGIVFJrQJyXma0Bvg58wN1H5mjzSwQJxEdyNv+Cu18L/ArB8KdfzHPcLWa2z8z2DQwMLEP0IiLVRQlECamtMXbuaGevZmKSBejpDZKC7q789Q9ZiXiMQ8lhzk5Mr0RYkkcylaauxtjQ0hR1KCXJzOoJkoe73f0bc7S5Evg8cKO7n+tSc/dk+O8p4B4gMftYd7/L3Xe5+67169cvx1sQEakqSiBKTCIe46nnzjB4VkWvMr+e3tNcuGENHWsa522XiMeYyTgHnlUvRFSSQ2k2tzVRW6M1IGazYGGMLwCH3f3P5mizHfgGcLO7P5WzfbWZtWTvA78MHFr+qEVEqpsSiBKz+1zRq3ohZG4zGWdf39C8w5eyrt3RTo2ptiZKyVSaLa0avjSHVwE3A68Lp2J92MzeaGa3mtmtYZs/ADqAz82arnUj8BMzewToAb7l7t9Z8XcgIlJl6qIOQF7sis5WGutq6Okd5PrLNkUdjpSowydGGJ2YPpdwzmdNYx2Xb21ljxKIyCSH0vz8hXMXu1czd/8JMG/XjLu/F3hvnu3PAFe99AgREVlO6oEoMY11tVyzvU3fFsu8sudHd9f5EwgI1oN4+FiKiemZ5QxL8piayfDc6DidmoFJREQqhBKIEpSId/DY8WFGx6eiDkVKVE/vINtiq9iywIvSRDzG5HSGg/3DyxyZzHZyeBx3zcAkIiKVQwlECUp0xcg4HHg2FXUoUoLcnb19g/Ou/zBbtqdCPVsrrz87hWtbc8SRiIiIFEdBCYSZfcrMnjCzg2Z2j5m15ey7zcyOmNmTZnb9HMffHe4/ZGZfDKfyw8xea2bDOQV1f1BInOXm2h1t1NWYFv+SvJ4eOMvps5Mk4vNP35qrfXUDF29sUR1EBJIprQEhIiKVpdAeiAeAy939SuAp4DYAM7sUuAm4DLiBYOaM2jzH3w1cAlwBrOLFRXI/dverw9vHC4yzrDQ3BEWv+rZY8smeF4n44opyu+Pt7O8bZHomsxxhyRyyi8htbtUaECIiUhkKSiDc/X53z65O9VOgM7x/I/AVd59w917gCPkX97nPQwRT8HXOblOtdsdjPHJsmPEpFb3Ki/X0nmZ9SyNdHYsbEpOId3B2cobHT+Rd5FeWSTI1xvqWRprq832HIiIiUn6KWQPxbuDb4f2twLGcff3htrzCoUs3A7nzd7/SzB4xs2+b2WVFjLMsJOIxJmcyPHxMdRDyAndnT+8giXiMYP2thUuoDiISyVSarZqBSUREKsh5Ewgz+15YozD7dmNOm48B0wRDkpbic8CP3P3H4eMDwA53vwr4C+Af54nvFjPbZ2b7BgYGlvjypWfXjhimxb9klv6hNCeGxxe0/sNsm1qb2NHRrHNqhSWH0qp/EBGRinLeheTc/br59pvZu4A3Aa8PhyIBJIFtOc06w235jv9DYD3w2zmvOZJz/z4z+5yZrXP35/PEdxdwF8CuXbt89v5y1dpcz8UbW7QitbxI9nxYyArU+SS6Ynzv8HNkMk5NzeJ6MGTxMhnneGpci0KKiEhFKXQWphuADwNvdvexnF33AjeZWaOZxYGLCGocZh//XuB64DfcPZOzfZOF4zPMLBHGWXVTEu2Ox9h/dIgpFb1KqKd3kNZV9bxsQ8uSjk/EYwyNTXFk4EyRI5N8nj8zweRMRj0QIiJSUQqtgbgDaAEeCKdbvRPA3R8Dvgo8TlDX8H53nwEws/vMbEt4/J3ARuDfZk3X+jbgkJk9AnwWuCmnd6NqJOIdjE3O8NhxFb1KoKd3kO6u9iX3HmR7LjSd68roD6dw3dKqBEJERCrHeYcwzcfdL5xn3+3A7Xm2vzHnft7Xd/c7CJKTqtYdzvPf03uaq7e1nae1VLpTo+M88/xZbkpsO3/jOWyPNbNxbSM9vYPc/IodRYxO8jmuNSBERKQCaSXqErahpYkL1q1W0asAsLd3CFj8+g+5zIxEvIO9vYNUYafeisuuAaEEQkREKokSiBLX3RVjb98QmYwu9qpdT+9pmhtquWzL2oKeJxGPcXJknGOD6SJFJnNJptK0NNWxtqk+6lBERESKRglEiUvEYwynp3jq1GjUoUjEevqG2Lmjnfrawv7b7j5XB1F18xKsuOSQ1oAQEZHKowSixGWLXjWMqboNj03xxMkRuruWNn1rrgvXr6G9uV7n1ApIptJ0aviSiIhUGCUQJa6zfRVbWps0a06V23d0EPelr/+Qq6bG2NUVo0drjCw79UCIiEglUgJR4oKi1xg9Knqtaj29gzTU1hRtNq7d8RhHT4/x3Mh4UZ5PXmo4PcXoxLQKqEVEpOIogSgDiXgHA6MT9J0eO39jqUh7ege5alsrTfW1RXk+DY1bfudmYGprjjgSERGR4lICUQYSOetBSPUZm5zmUHK4KMOXsi7dvJbVDbVKIJZRUmtAiIhIhVICUQZ+bv0aYqsb6AnXAZDq8tCzKaYzXpQC6qy62hp2dsWUQCyj5FDQY6gaCBERqTRKIMqAmZHoitHTpx6IarSnd5Aag5072ov6vLvjMZ58bpShs5NFfV4JJFNpGupqWLemIepQREREikoJRJlIxGMcG0xzPKXFv6pNT+9pLtvSSkuRFyPLDonaq9mYlkUyFczAZGZRhyIiIlJUSiDKhC72qtPE9AwPPZsqav1D1pWdrTTU1WgY0zJJpsY1fElERCqSEogy8fLNa2lprNN6EFXm0f5hJqYzy5JANNbVcvW2NiWly0RrQIiISKVSAlEmamuMnV3t+ra4ymQTxmIWUOfaHY9x6PgIZyaml+X5q9X41AzPn5nQDEwiIlKRlECUkUQ8xpFTZzh9ZiLqUGSF7O0b5KINwSxcyyERjzGTcQ4c1QxfxZStVVIPhIiIVCIlEGVk97k6CF3sVYOZjLOvb2hZhi9lXbu9ndoaU89WkWkNCBERqWRKIMrIFVvbaFTRa9U4fCIYWrScCcTqxjou39qqc6rIXliFWgmEiIhUHiUQZaShroZrt7drPYgqka1/WM4EAiDR1c7D/SnGp2aW9XWqSTKVpsZgU2tT1KGIiIgUnRKIMpOIx3j8+Agj41NRhyLLrKf3NNtjzWxuXd5vsRPxDianMxzsH17W16kmyaE0m9Y2UV+rP7EiIlJ59OlWZhLxGBmH/Sp6rWjuzt5lrn/I6u4KVrju6VXPVrH0p9KqfxARkYqlBKLMXLO9jboaY6/GrFe0pwfOMHh2ksQyTd+aq625gUs2tWiNkSLSGhAiIlLJlECUmeaGOq7oVNFrpVup+oesRDzG/qNDTM9kVuT1Ktn0TIaTI+PqgRARkYpVcAJhZp8ysyfM7KCZ3WNmbTn7bjOzI2b2pJldP8fxf2tmvWb2cHi7OtxuZvbZ8PiDZnZtobFWikQ8xiMqeq1oPb2DbGhpZEdH84q8XiIeY2xyhseOj6zI61WyU6MTzGScLeqBEBGRClWMHogHgMvd/UrgKeA2ADO7FLgJuAy4AficmdXO8Rz/2d2vDm8Ph9t+BbgovN0C/FURYq0Iu+Mxpmach55NRR2KLAN3Z88zgyTiMcxsRV4zO1Rqb596tgqV1CJyIiJS4QpOINz9fnefDh/+FOgM798IfMXdJ9y9FzgCJBbx1DcCf+eBnwJtZra50Hgrwc4dMczQMKYK1T+U5uTI+LmFA1fChrVNdHU0qw6iCLJrQHRqCJOIiFSoYtdAvBv4dnh/K3AsZ19/uC2f28NhSn9uZo1LOL6qtK6q55JNa7UeRIV6of6hY0VfNxGPsbdvkEzGV/R1K022B0JDmEREpFItKIEws++Z2aE8txtz2nwMmAbuXmQMtwGXAN1ADPjIYg42s1vMbJ+Z7RsYGFjkS5ev3fEYB46mmFLRa8XZ2ztI66p6LtqwZkVfNxHvIDU2xc9OnVnR1600/UNpYqsbaG6oizoUERGRZbGgBMLdr3P3y/PcvglgZu8C3gS8w92zX18mgW05T9MZbpv93CfCYUoTwJd4YZjTQo+/y913ufuu9evXL+TtVIREPEZ6aoZDSS3+VWl6+gbp7opRU7My9Q9Z2SFTWg+iMMmUpnAVEZHKVoxZmG4APgy82d3HcnbdC9xkZo1mFicohu7Jc/zm8F8D3gIcyjn+N8PZmF4BDLv7iULjrRTdXdmLPY1ZrySnRsbpff7sitY/ZHW2r2Jza5PqIAqUHBpTAiEiIhWtGDUQdwAtwAPhNKx3Arj7Y8BXgceB7wDvd/cZADO7z8y2hMffbWaPAo8C64A/DrffBzxDUHz9N8D7ihBrxVjf0sgF61crgagwPX0ru/5DLjOjuyuog3ihI1EWw92DHggVUIuISAUreJCuu184z77bgdvzbH9jzv3XzXGsA+8vNL5Ktjse458OnmAm49Su8HAXWR49vYM0N9Ry2Za1kbx+Ih7j3keO8+zgGDs6VkcSQzkbPDvJ+FRGPRAiIlLRtBJ1GevuijE6Ps2TJ0ejDkWKpKd3kJ072qmrjea/ZnbolIYxLc25NSDUAyEiIhVMCUQZyw5z0eJflSE1NsmTz42eW9QtChduWENsdYOGxi1Rdg0I9UCIiEglUwJRxjrbm9natkoXexViX98Q7tHUP2QFdRDtOqeWKNsDoUXkRESkkimBKHOJeIw9vSp6rQQ9fYM01NZw1ba2SONIxDt4dnCMk8PjkcZRjpKpNM0NtbSuqo86lLJhZtvM7Adm9riZPWZmv5unzTvCxUYfNbMHzeyqnH03mNmTZnbEzD66stGLiFQnJRBlLhGP8fyZCXqfPxt1KFKgPb2DXL2tjab62kjjyA6h6tHQuEVLDgVrQASzUssCTQMfcvdLgVcA7zezS2e16QVe4+5XAJ8A7gIws1rgL4FfAS4FfiPPsSIiUmRKIMpcIq71ICrB2YlpDiWHIx2+lPXyzS2saazTgnJLoClcFy9cTPRAeH8UOAxsndXmQXcfCh/+lGBhUQgWHj3i7s+4+yTwFeDGlYlcRKR6KYEocxesW826NSp6LXcHnh1iJuMlkUDU1dawc4fqIJZCq1AXxsy6gGuAPfM0ew/w7fD+VuBYzr5+ZiUfIiJSfEogylx28S8NNylve3sHqTG4dkd71KEAQc/WU8+dYfDsZNShlI2zE9OkxqbUA7FEZrYG+DrwAXcfmaPNLxEkEB9Z5HPfYmb7zGzfwMBA4cGKiFQ5JRAVIBGP0T+UPjcDjJSfPb2DXL61lTWNBa/tWBS7NUXwop1bA0I9EItmZvUEycPd7v6NOdpcCXweuNHds+PrksC2nGad4bYXcfe73H2Xu+9av359cYMXEalCSiAqwLn1IDTkpCxNTM/w0LFUpOs/zHZFZyuNdTU6pxYhuwaEpnBdHAsqzr8AHHb3P5ujzXbgG8DN7v5Uzq69wEVmFjezBuAm4N7ljllEpNqVxtedUpBLNq2lpamOPb2DvOUaDf8tNwf7h5mczpRE/UNWY10tV29r09C4Reg/1wPRHHEkZedVwM3Ao2b2cLjt94HtAO5+J/AHQAfwuXCGq+mwR2HazH4H+C5QC3zR3R9b6TcgIlJtlEBUgNqasA5Cs+aUpWyxcncJ9UBAMIzpjh8c4czEdMkMrSplyaE09bXGhpbGqEMpK+7+E2DeeW/d/b3Ae+fYdx9w3zKEJiIic9AQpgrR3RXj6YGzPH9mIupQZJH29A7yso1raF/dEHUoL5KId5Bx2H906PyNhWQqzebWVdTUaA0IERGpbEogKkR2+Ms+DTkpK9MzGQ4cHSqp4UtZ1+5oo67G1LO1QMmhMRVQi4hIVVACUSGu2NpKU30Ne1T0WlYOnxjlzMQ0iXhH1KG8RHNDHZdvbdV6EAt0PDXOFiUQIiJSBZRAVIiGuhqu3a7Fv8rNnvDb/VKagSnX7niMR44NMz41E3UoJW1yOsNzo+NaA0JERKqCEogKkojHePzECCPjU1GHIgvU0zvIjo5mNrU2RR1KXt1dMSZnMjxyLBV1KCXt5PA47tCpHggREakCSiAqSCIewx3296notRxkMs7evsGS7X2AIIEwQz1b59GfGgNQD4SIiFQFJRAV5Jpt7dTXmuogysSRgTMMjU3RXYIF1FmtzfVcvLFF60GcR3YRORVRi4hINVACUUFWNdRyxdZW9upiryxkv9XfXcIJBATx7T86xNRMJupQSlYyXERuc1tpDkUTEREpJiUQFSYR7+Bgf4r0pIpeS11P7yAb1zayPVbaKxcn4h2MTc7w2PGRqEMpWcmhNBtaGmmsq406FBERkWWnBKLC7I7HmJpxHjqmOohS5u709A6SiHdgVtoLj3XH2wHYq6Fxc0qm0qp/EBGRqlFQAmFmnzKzJ8zsoJndY2ZtOftuM7MjZvakmV0/x/E/NrOHw9txM/vHcPtrzWw4Z98fFBJnNdnZ1a6i1zJwbDDNyZHxklxAbrYNLU3E161Wbc08kqm06h9ERKRqFNoD8QBwubtfCTwF3AZgZpcCNwGXATcAnzOzl/Ttu/ur3f1qd78a+DfgGzm7f5zd5+4fLzDOqrG2qZ5LN69VAlHisus/lHr9Q1aiK8bevkEyGY86lJKTyTgnUloDQkREqkdBCYS73+/u0+HDnwKd4f0bga+4+4S79wJHgMRcz2Nma4HXAf9YSDwS6O6KceDZISanVfRaqnp6B2lrrufC9WuiDmVBEvEYw+kpnjo1GnUoJWfgzASTMxmtASEiIlWjmDUQ7wa+Hd7fChzL2dcfbpvLW4B/dvfcKs1XmtkjZvZtM7usiHFWvN3xGONTGQ4dH446FJnD3r5Burti1NSUdv1DVnaolXq2Xqo/O4WreiBERKRKnDeBMLPvmdmhPLcbc9p8DJgG7l5iHL8B/EPO4wPADne/CvgL5umZMLNbzGyfme0bGBhY4stXlm5d7JW050bG6Ts9VjbDlwA621expbVJdRB5HA+ncN2iHggREakS500g3P06d788z+2bAGb2LuBNwDvcPTtAOglsy3maznDbS5jZOoLhTd/Kec0Rdz8T3r8PqA/b5YvvLnff5e671q9ff763UxXWrWnk59avVgJRorK/l3IooM4yMxLxGHt7B3nhv7nAC2tAqIhaRESqRaGzMN0AfBh4s7uP5ey6F7jJzBrNLA5cBPTM8TRvA/7J3cdznneThXNbmlkijPN0IbFWm0S8g719g8yo6LXk9PQOsrqhlks3r406lEVJxDs4NTrB0dNj529cRZJDadY21dHSVB91KCIiIiui0BqIO4AW4IFwutU7Adz9MeCrwOPAd4D3u/sMgJndZ2Zbcp7jJl48fAmCpOKQmT0CfBa4yfW156Ik4u2Mjk/zxEkt/lVqenoH2dkVo662vJZhSYTrQahn68WCNSBKezFAERGRYqor5GB3v3CefbcDt+fZ/sZZj1+bp80dBMmJLFEi3gEEF3uXbWmNOBrJGjo7yZPPjfLvr9ocdSiL9nPr1xBb3cCe3kF+rXvb+Q+oEsmhNNtKfDVxERGRYiqvr0Blwba2rWJr2yr29unb4lKy72iwQng2wSsnZkaiK0ZPn0YTZrk7yVSaTs3AJCIiVUQJRAXbHY/Ro6LXktLTe5qGuhqu7CzPXqFEPMaxwTQnhtNRh1ISRtLTnJmYVgG1iIhUFSUQFSwRj/H8mUmeef5s1KFIqKd3kKu3tdFU/5KF2cuC1oN4sf5UUFCuNSBERKSaKIGoYLrYKy1nJqY5dHykrNZ/mO3lm9fS0lincyqUHNIUriIiUn2UQFSw+LrVrFvToIu9EnHg6BAzGS+r9R9mq60xdna165wKnVsDQj0QIiJSRZRAVLDs4l+62CsNPb2D1NYY125vjzqUgiTiMX526gynz0xEHUrkkkNpmupr6FjdEHUoIiIiK0YJRIVLdMVIptL0D2nxr6j19A1y+Za1rG4saPbkyGWHYO3tG4o4kugdH06zpW0V4bqXIiIiVUEJRIXLTheq6VyjNT41w8PHUmU9fCnriq1tNNbV6Jwi6IFQ/YOIiFQbJRAV7uJNLaxtUtFr1A72DzM5nSnL9R9ma6ir4drtqoOAcBVqJRAiIlJllEBUuNoao7srxh5d7EWqpzdYfK27q7zrH7IS8RiPHR9mdHwq6lAiMz41w/NnJpVAiIhI1VECUQW64zGeGTjLwKiKXqOyp3eQSza10NZcGcW2iXiMjMP+o9VbB6EZmEREpFopgagCiXNFr+qFiML0TIYDR4fo7ir/+oesa7a3UVdjVT2MSWtAiIhItVICUQUu39LKqvraqr7Yi9LjJ0Y4OzlTEQXUWc0NdVzR2VrV55R6IEREpFopgagCDXU1XLujraov9qKU/blXUgIBwfs52D/M+NRM1KFEIjmUprbG2LS2KepQREREVpQSiCqR6Org8MkRhtPVW/QalT29g3R1NLOxwi40d8djTM5kePhYKupQIpFMpdm0tom6Wv0ZFZH/2d7dx8hR33ccf3/qR2xj7MMGG5+NL8UKMpTE5GJDcSsaCAGaYNTwRyo1OE9NpUZKaIMSKK0S2qhKW0pIREvkQhOSEEAhuKCkJRgHFCVK8BMoGAzYscG+qx8I5ydsMJzv2z9mtt24Nt7b3ZnfsfN5SavbnZ0Zfb6e8+58b+Y3Y1Yt/uariIU9XUTA2hd9FKJMQ0PB6hcGOu7oA8C7TtynQcQAAAuwSURBVO9CorJHtnwPCDMzqyo3EBWxYM4UxoySL+daso27XmHPwTc6agB1zUknjOHMGZOr20DsedXjH8zMrJLcQFTE+DGjOKfb4yDKVrv/w6IOuIHc0Szq6WLti7t54/BQ6iilGjw8xI59r/kIhJmZVZIbiApZ2NPFU317efX1ag56TWHVC7uZMXk8s7s6c0dzYU8Xr75xmPX9e1NHKdXO/Yc4PBQ+AmFmZpXkBqJCFvZ0MTgUPLG1ujf/KlNEsGrLyyzs6UJS6jiFqJ2aVbV7jNTuAXGaj0CYmVkFuYGokHedPpXfEh4HUZKtAwfZue9QRw6grpl+4jjeNn1i5U6N699zEPBN5MzMrJrcQFTI5PFjmH9adQe9lq3WqC3q4AYCsvpWbRlgaChSRymN70JtZmZV1nIDIemfJD0r6ZeSlkuakk8/WdKjkl6RdOubLN8laYWkjfnPqfl0SfqapE35us9tNatlp5ys27qb1werNeg1hVVbBpg6YQxnnDIpdZRCvXtuF/teG+S5nftTRylN/55XOXniWE4YOyp1FDMzs9K14wjECuDsiDgHeB64Pp/+GvA3wLXHWf46YGVEzANW5q8BLgPm5Y9PAre1IWvlLerp4tDgEE/1V/PmX2VatWWAd8/t3PEPNbVTtKp0ZKtvty/hamZm1TW61RVExMN1L38BXJVPPwD8VNIZx1nFEuDC/PmdwGPA5/Pp34qIAH4haYqkmRGxvdXMVVYb9Pq9NX0MHPBdqYty4NAgWwcOcvX5p6eOUrjuqROYNeUEHlq/ozKDije/dIBzuk9KHcPMzCyJlhuII3wMuHeYy5xa1xTsAE7Nn88CttXN15dP+40GQtInyY5QMGfOnOHmrZyTJ41j/szJ3LN6G/es3nb8Bawli+dNSx2hFIvPmMa9a7bx880vp45Smg+eOyt1BDMzsyQaaiAkPQLMOMpbN0TEA/k8NwCDwF3NhomIkDSskZgRsQxYBtDb21udUZwt+O6fLqIvHwRqxZk0bjRzp01MHaMUNy45iw9X4GhLvbfPODF1hI4gaTbwLbI/HgWwLCK+esQ8ZwLfAM4l+965qe69F4D9wGFgMCJ6S4puZlZZDTUQEXHxm70v6SPA+4GL8lOOhmNn7dQkSTOBXfn0fmB23Xzd+TRr0ZQJY5kyYWzqGNZBxo8ZxdmzfEqPNWUQ+GxErJN0IrBW0oqIeKZungHg08CVx1jHH0TEr4sOamZmmXZchelS4HPAFRFxsIlVPAgszZ8vBR6om351fjWm84C9Hv9gZtZZImJ7RKzLn+8HNpCdrlo/z66IWA144JaZ2QjQjqsw3QqcCKyQ9KSkr9feyA8t3wx8RFKfpPn59Nsl1Q4zfxl4r6SNwMX5a4D/BDYDm4B/A/68DVnNzGyEkjQXWAA8PozFAnhY0tp8TJyZmRWsHVdhOuZVliJi7jGmf6Lu+cvARUeZJ4BPtZrPzMxGPkmTgO8D10TEvmEsujgi+iWdQvaHrGcj4idHrNsX2zAzayPfidrMzJKSNIasebgrIu4fzrIR0Z//3AUsBxYeZZ5lEdEbEb3Tp09vR2Qzs0pzA2FmZskou9PiHcCGiLh5mMtOzAdeI2kicAmwvv0pzcysXrvvA2FmZjYcFwAfBp6S9GQ+7a+AOQAR8XVJM4A1wGRgSNI1wHxgGrA8v9v7aOC7EfFQyfnNzCrHDYSZmSUTET8FdJx5dpBdyvtI+4B3FJHLzMyOzacwmZmZmZlZw9xAmJmZmZlZw9xAmJmZmZlZw5TdbqEzSHoJeLHJxacBv25jnJHO9XauKtUKrnc4To+ISl/HtMXviVT8O965qlQruN6RruHviI5qIFohaU1E9B5/zs7gejtXlWoF12udr2rbvEr1VqlWcL2dxKcwmZmZmZlZw9xAmJmZmZlZw9xA/J9lqQOUzPV2rirVCq7XOl/VtnmV6q1SreB6O4bHQJiZmZmZWcN8BMLMzMzMzBrmBgKQdKmk5yRtknRd6jxFkTRb0qOSnpH0tKTPpM5UBkmjJD0h6QepsxRN0hRJ90l6VtIGSeenzlQUSX+R/x6vl3S3pPGpM7WTpH+XtEvS+rppXZJWSNqY/5yaMqO1T6PbVtLSfJ6NkpYe5f0H639nRqpW6pU0QdIP88+5pyV9udz0jTnevoWkcZLuzd9/XNLcuveuz6c/J+l9ZeZuVrP1SnqvpLWSnsp/vqfs7MPVyrbN358j6RVJ15aVud0q30BIGgX8C3AZMB/4Y0nz06YqzCDw2YiYD5wHfKqDa633GWBD6hAl+SrwUEScCbyDDq1b0izg00BvRJwNjAI+lDZV230TuPSIadcBKyNiHrAyf22d4bjbVlIX8AVgEbAQ+EL9jrekPwJeKSduy1qt96b8c24BcIGky8qJ3ZgG9y0+DuyOiDOArwD/kC87n+zz7Cyyz4B/zdc3YrVSL9l9Ej4QEb8DLAW+XU7q5rRYa83NwH8VnbVIlW8gyD6UNkXE5oh4HbgHWJI4UyEiYntErMuf7yfbuZyVNlWxJHUDfwjcnjpL0SSdBPw+cAdARLweEXvSpirUaOAESaOBCcB/J87TVhHxE2DgiMlLgDvz53cCV5YayorUyLZ9H7AiIgYiYjewgrzJlDQJ+EvgSyVkbYem642IgxHxKGSfc8A6oLuEzMPRyL5F/b/BfcBFkpRPvyciDkXEFmBTvr6RrOl6I+KJiKh9fj9N9rk+rpTUzWll2yLpSmALWa1vWW4gsh3obXWv++jwnWqA/HDaAuDxtEkKdwvwOWAodZAS9AAvAd/IT9m6XdLE1KGKEBH9wE3AVmA7sDciHk6bqhSnRsT2/PkO4NSUYaytGtm2b/Z99XfAPwMHC0vYXq3WC2SnbQIfIDuKMZI0sm/xv/NExCCwFzi5wWVHmlbqrfdBYF1EHCooZzs0XWve6H8euLGEnIVyA1FB+S/w94FrImJf6jxFkfR+YFdErE2dpSSjgXOB2yJiAXCADj3FJT+NYQlZ03QaMFHSn6RNVa7ILqHny+i9hUh6JB+zc+TjN/56OdxtK+mdwG9HxPJ2Z25FUfXWrX80cDfwtYjY3KbYloiks8hO9fmz1FkK9EXgKxHxVjnV8JhGpw4wAvQDs+ted+fTOpKkMWTNw10RcX/qPAW7ALhC0uXAeGCypO9ERKfuaPYBfRFRO6p0Hx3aQAAXA1si4iUASfcDvwt8J2mq4u2UNDMitkuaCexKHcgaFxEXH+s9SY1s237gwrrX3cBjwPlAr6QXyL7XT5H0WERcSEIF1luzDNgYEbe0IW67NbJvUZunL2+GTgJebnDZkaaVemunGy8Hro6IXxUftyWt1LoIuErSPwJTgCFJr0XErcXHbi8fgYDVwDxJPZLGkg1cejBxpkLk59/dAWyIiJtT5ylaRFwfEd0RMZdsu/64g5sHImIHsE3S2/NJFwHPJIxUpK3AefnVWERWa0cOGD/Cg2SDDMl/PpAwi7VXI9v2R8AlkqbmR+EuAX4UEbdFxGn5Z91i4PnUzUMDmq4XQNKXyHbKrikhazMa2beo/ze4iuw7KvLpH8qv5NMDzANWlZS7WU3Xm5+G9kPguoj4WWmJm9d0rRHxexExN/+/egvw92/F5gGAiKj8A7gceB74FXBD6jwF1rmY7DDxL4En88flqXOVVPuFwA9S5yihzncCa/Jt/B/A1NSZCqz1RuBZYD3ZVTvGpc7U5vruJhvf8QbZ0aWPk50vvBLYCDwCdKXO6UfbtvdRty3QC9xeN9/HyAbVbgI+epT1zAXWp66nyHrJ/uIbZH80qH2XfSJ1TUep8f/tWwB/C1yRPx8PfC+vbRXwtrplb8iXew64LHUtRdYL/DXZKbdP1j1OSV1PUdu2bh1fBK5NXUuzD9+J2szMzMzMGuZTmMzMzMzMrGFuIMzMzMzMrGFuIMzMzMzMrGFuIMzMzMzMrGFuIMzMzMzMrGFuIMzMzMzMrGFuIMzMzMzMrGFuIMzMzMzMrGH/A8sBimv4icq3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start=timer()\n",
    "\n",
    "env_id = \"PongNoFrameskip-v4\"\n",
    "env    = make_atari(env_id)\n",
    "env    = wrap_deepmind(env, frame_stack=False)\n",
    "env    = wrap_pytorch(env)\n",
    "model = Model(env=env, config=config)\n",
    "\n",
    "episode_reward = 0\n",
    "\n",
    "observation = env.reset()\n",
    "for frame_idx in range(1, config.MAX_FRAMES + 1):\n",
    "    epsilon = config.epsilon_by_frame(frame_idx)\n",
    "\n",
    "    action = model.get_action(observation, epsilon)\n",
    "    prev_observation=observation\n",
    "    observation, reward, done, _ = env.step(action)\n",
    "    observation = None if done else observation\n",
    "\n",
    "    model.update(prev_observation, action, reward, observation, frame_idx)\n",
    "    episode_reward += reward\n",
    "\n",
    "    if done:\n",
    "        model.finish_nstep()\n",
    "        model.reset_hx()\n",
    "        observation = env.reset()\n",
    "        model.save_reward(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "        if np.mean(model.rewards[-10:]) > 19:\n",
    "            plot(frame_idx, all_rewards, losses, timedelta(seconds=int(timer()-start)))\n",
    "            break\n",
    "\n",
    "    if frame_idx % 10000 == 0:\n",
    "        plot(frame_idx, model.rewards, model.losses, model.sigma_parameter_mag, timedelta(seconds=int(timer()-start)))\n",
    "\n",
    "model.save_w()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}