11.DRQN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Recurrent Q Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from IPython.display import clear_output\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from timeit import default_timer as timer\n",
    "from datetime import timedelta\n",
    "import math\n",
    "import random\n",
    "\n",
    "from utils.wrappers import *\n",
    "\n",
    "from agents.DQN import Model as DQN_Agent\n",
    "\n",
    "from networks.network_bodies import SimpleBody, AtariBody\n",
    "\n",
    "from utils.ReplayMemory import ExperienceReplayMemory\n",
    "from utils.hyperparameters import Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = Config()\n",
    "\n",
    "config.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device = config.device\n",
    "\n",
    "#epsilon variables\n",
    "config.epsilon_start = 1.0\n",
    "config.epsilon_final = 0.01\n",
    "config.epsilon_decay = 30000\n",
    "config.epsilon_by_frame = lambda frame_idx: config.epsilon_final + (config.epsilon_start - config.epsilon_final) * math.exp(-1. * frame_idx / config.epsilon_decay)\n",
    "\n",
    "#misc agent variables\n",
    "config.GAMMA=0.99\n",
    "config.LR=1e-4\n",
    "\n",
    "#memory\n",
    "config.TARGET_NET_UPDATE_FREQ = 1024\n",
    "config.EXP_REPLAY_SIZE = 10000\n",
    "config.BATCH_SIZE = 32\n",
    "\n",
    "#Learning control variables\n",
    "config.LEARN_START = 10000\n",
    "config.MAX_FRAMES=1500000\n",
    "\n",
    "#Nstep controls\n",
    "config.N_STEPS=1\n",
    "\n",
    "#DRQN Parameters\n",
    "config.SEQUENCE_LENGTH=8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replay Buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RecurrentExperienceReplayMemory:\n",
    "    def __init__(self, capacity, sequence_length=10):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "        self.seq_length=sequence_length\n",
    "\n",
    "    def push(self, transition):\n",
    "        self.memory.append(transition)\n",
    "        if len(self.memory) > self.capacity:\n",
    "            del self.memory[0]\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        finish = random.sample(range(0, len(self.memory)), batch_size)\n",
    "        begin = [x-self.seq_length for x in finish]\n",
    "        samp = []\n",
    "        for start, end in zip(begin, finish):\n",
    "            #correct for sampling near beginning\n",
    "            final = self.memory[max(start+1,0):end+1]\n",
    "            \n",
    "            #correct for sampling across episodes\n",
    "            for i in range(len(final)-2, -1, -1):\n",
    "                if final[i][3] is None:\n",
    "                    final = final[i+1:]\n",
    "                    break\n",
    "                    \n",
    "            #pad beginning to account for corrections\n",
    "            while(len(final)<self.seq_length):\n",
    "                final = [(np.zeros_like(self.memory[0][0]), 0, 0, np.zeros_like(self.memory[0][3]))] + final\n",
    "                            \n",
    "            samp+=final\n",
    "\n",
    "        #returns flattened version\n",
    "        return samp, None, None\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DRQN(nn.Module):\n",
    "    def __init__(self, input_shape, num_actions, gru_size=512, bidirectional=False, body=AtariBody):\n",
    "        super(DRQN, self).__init__()\n",
    "        \n",
    "        self.input_shape = input_shape\n",
    "        self.num_actions = num_actions\n",
    "        self.gru_size = gru_size\n",
    "        self.bidirectional = bidirectional\n",
    "        self.num_directions = 2 if self.bidirectional else 1\n",
    "\n",
    "        self.body = body(input_shape, num_actions)\n",
    "        self.gru = nn.GRU(self.body.feature_size(), self.gru_size, num_layers=1, batch_first=True, bidirectional=bidirectional)\n",
    "        #self.fc1 = nn.Linear(self.body.feature_size(), self.gru_size)\n",
    "        self.fc2 = nn.Linear(self.gru_size, self.num_actions)\n",
    "        \n",
    "    def forward(self, x, hx=None):\n",
    "        batch_size = x.size(0)\n",
    "        sequence_length = x.size(1)\n",
    "        \n",
    "        x = x.view((-1,)+self.input_shape)\n",
    "        \n",
    "        #format outp for batch first gru\n",
    "        feats = self.body(x).view(batch_size, sequence_length, -1)\n",
    "        hidden = self.init_hidden(batch_size) if hx is None else hx\n",
    "        out, hidden = self.gru(feats, hidden)\n",
    "        x = self.fc2(out)\n",
    "\n",
    "        return x, hidden\n",
    "        #return x\n",
    "\n",
    "    def init_hidden(self, batch_size):\n",
    "        return torch.zeros(1*self.num_directions, batch_size, self.gru_size, device=device, dtype=torch.float)\n",
    "    \n",
    "    def sample_noise(self):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(DQN_Agent):\n",
    "    def __init__(self, static_policy=False, env=None, config=None):\n",
    "        self.sequence_length=config.SEQUENCE_LENGTH\n",
    "\n",
    "        super(Model, self).__init__(static_policy, env, config)\n",
    "\n",
    "        self.reset_hx()\n",
    "    \n",
    "    def declare_networks(self):\n",
    "        self.model = DRQN(self.num_feats, self.num_actions, body=AtariBody)\n",
    "        self.target_model = DRQN(self.num_feats, self.num_actions, body=AtariBody)\n",
    "\n",
    "    def declare_memory(self):\n",
    "        self.memory = RecurrentExperienceReplayMemory(self.experience_replay_size, self.sequence_length)\n",
    "        #self.memory = ExperienceReplayMemory(self.experience_replay_size)\n",
    "\n",
    "    def prep_minibatch(self):\n",
    "        transitions, indices, weights = self.memory.sample(self.batch_size)\n",
    "\n",
    "        batch_state, batch_action, batch_reward, batch_next_state = zip(*transitions)\n",
    "\n",
    "        shape = (self.batch_size,self.sequence_length)+self.num_feats\n",
    "\n",
    "        batch_state = torch.tensor(batch_state, device=self.device, dtype=torch.float).view(shape)\n",
    "        batch_action = torch.tensor(batch_action, device=self.device, dtype=torch.long).view(self.batch_size, self.sequence_length, -1)\n",
    "        batch_reward = torch.tensor(batch_reward, device=self.device, dtype=torch.float).view(self.batch_size, self.sequence_length)\n",
    "        #get set of next states for end of each sequence\n",
    "        batch_next_state = tuple([batch_next_state[i] for i in range(len(batch_next_state)) if (i+1)%(self.sequence_length)==0])\n",
    "\n",
    "        non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, batch_next_state)), device=self.device, dtype=torch.uint8)\n",
    "        try: #sometimes all next states are false, especially with nstep returns\n",
    "            non_final_next_states = torch.tensor([s for s in batch_next_state if s is not None], device=self.device, dtype=torch.float).unsqueeze(dim=1)\n",
    "            non_final_next_states = torch.cat([batch_state[non_final_mask, 1:, :], non_final_next_states], dim=1)\n",
    "            empty_next_state_values = False\n",
    "        except:\n",
    "            empty_next_state_values = True\n",
    "\n",
    "        return batch_state, batch_action, batch_reward, non_final_next_states, non_final_mask, empty_next_state_values, indices, weights\n",
    "    \n",
    "    def compute_loss(self, batch_vars):\n",
    "        batch_state, batch_action, batch_reward, non_final_next_states, non_final_mask, empty_next_state_values, indices, weights = batch_vars\n",
    "\n",
    "        #estimate\n",
    "        current_q_values, _ = self.model(batch_state)\n",
    "        current_q_values = current_q_values.gather(2, batch_action).squeeze()\n",
    "        \n",
    "        #target\n",
    "        with torch.no_grad():\n",
    "            max_next_q_values = torch.zeros((self.batch_size, self.sequence_length), device=self.device, dtype=torch.float)\n",
    "            if not empty_next_state_values:\n",
    "                max_next, _ = self.target_model(non_final_next_states)\n",
    "                max_next_q_values[non_final_mask] = max_next.max(dim=2)[0]\n",
    "            expected_q_values = batch_reward + ((self.gamma**self.nsteps)*max_next_q_values)\n",
    "\n",
    "        diff = (expected_q_values - current_q_values)\n",
    "        loss = self.huber(diff)\n",
    "        \n",
    "        #mask first half of losses\n",
    "        split = self.sequence_length // 2\n",
    "        mask = torch.zeros(self.sequence_length, device=self.device, dtype=torch.float)\n",
    "        mask[split:] = 1.0\n",
    "        mask = mask.view(1, -1)\n",
    "        loss *= mask\n",
    "        \n",
    "        loss = loss.mean()\n",
    "\n",
    "        return loss\n",
    "\n",
    "    def get_action(self, s, eps=0.1):\n",
    "        with torch.no_grad():\n",
    "            self.seq.pop(0)\n",
    "            self.seq.append(s)\n",
    "            if np.random.random() >= eps or self.static_policy or self.noisy:\n",
    "                X = torch.tensor([self.seq], device=self.device, dtype=torch.float) \n",
    "                self.model.sample_noise()\n",
    "                a, _ = self.model(X)\n",
    "                a = a[:, -1, :] #select last element of seq\n",
    "                a = a.max(1)[1]\n",
    "                return a.item()\n",
    "            else:\n",
    "                return np.random.randint(0, self.num_actions)\n",
    "\n",
    "    #def get_max_next_state_action(self, next_states, hx):\n",
    "    #    max_next, _ = self.target_model(next_states, hx)\n",
    "    #    return max_next.max(dim=1)[1].view(-1, 1)'''\n",
    "\n",
    "    def reset_hx(self):\n",
    "        #self.action_hx = self.model.init_hidden(1)\n",
    "        self.seq = [np.zeros(self.num_feats) for j in range(self.sequence_length)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards, losses, sigma, elapsed_time):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s. time: %s' % (frame_idx, np.mean(rewards[-10:]), elapsed_time))\n",
    "    plt.plot(rewards)\n",
    "    if losses:\n",
    "        plt.subplot(132)\n",
    "        plt.title('loss')\n",
    "        plt.plot(losses)\n",
    "    if sigma:\n",
    "        plt.subplot(133)\n",
    "        plt.title('noisy param magnitude')\n",
    "        plt.plot(sigma)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwkAAAE/CAYAAAANJ48VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt85Xdd4P/XO8nMJNPOTMJ0epmTwhRb1NIFXMbS3RVRirZUpKzCUq/tclOBVVQWqRVEtMrFHyhbLnYBAUVKRZBZLdByVVZaKG6FttyGUmwyvTeZazIzmbx/f3y/Z3omJJkk55x8z0lez8fjPOac7+28vzlnku/7+/5cIjORJEmSpLqeqgOQJEmS1FlMEiRJkiQdwyRBkiRJ0jFMEiRJkiQdwyRBkiRJ0jFMEiRJkiQdwyShhSLi+yPilojYGxG/XnU8aq+IuDMinlZ1HK0WEb8QEddXHYckdbKV+jdAqjNJaK1XAJ/JzA2Z+Zaqg5kpIq6OiG9ExHREXDbL+t+MiHsiYk9EvDsi1jWs2xYRn4mIAxHx9Zm/GJvZdzWKiPMi4oaIeCgi7o+Iv42I0xrWR0S8PiIeLB+vj4iY41i/GxH7Gh4T5Wd80gLi2BYRGRF99WWZ+f7M/MnWnOniLPa70qrvXURcGhFfLo8zEhFvaPyZRMRfR8Td5fpvRsQLWnPGkiR1JpOE1noUcNtcKyOidxljmc2/AS8G/nXmioi4AHglcD7FeTwa+IOGTT4A/D9gM3AF8KGI2NLsvovReNG2nNr0vkPA1cA2ip/ZXuAvG9a/CHgW8HjgccBPA78y24Ey848z88T6A3g98NnMfKANcbfbgr8rLf7erQdeBpwEPKk85ssb1v8JsC0zNwLPBP4oIp64lBOUJKkrZKaPFjyATwNHgElgH/AY4D3A24HrgP3A04Cforhw2QPcBbym4RjbgAT+e7luDPhV4IeBrwDjwFUz3vd5wNfKbT8BPGoBsX4euGzGsr8B/rjh9fnAPeXzxwAHgQ0N6/8Z+NVm911ArHcCv1Oe/0GgD9gK/B1wP/Ad4NfLbfuBCeCk8vUVwBSwsXz9h8Cflc8X8jk8H/h34J/K5b8EfBd4sDz2ncDTWvT9+Y/A3obX/wK8qOH184EbF3CcAO4ALl3g+/57ea77ysd/Ai4DPt+wTVIkl9+iSGb+EPi+MsY9wLXA2obtnwHcUn5f/wV43AJjWdR3pc3fu98C/s8c674fuBv4b6347H348NGdj/rfAGAd8GfArvLxZ8C6cpuTgH8ofx8+VP4e6inX/Q4wWv5e/QZwftXn5MNH48NKQotk5lMp/vO/NIs7ut8sV/08cCWwgeLifD/wy8AgxYXqr0XEs2Yc7knAWcBzKX7ZXEHxi+ixwH+LiKcARMTFwO8CPwNsKd//A0s8hcdSVBrq/g04JSI2l+vuyMy9M9Y/tgX7LsTPUfysBoFp4P+Ux6hRXBi+LCIuyMxJ4EvAU8r9nkJxUf9fGl5/rny+kM/hKcAPAhdExNkUCd8vUSQpm4Hh+oYR8SMRMb6Ic5rpRzm2CjXbz3QhP7MnAydTJFELfV+AwfJ7+4U5trsAeCJwHkWzuquBXwROB86h+IyIiB8C3k1R9dgM/AWwo94MKCLeFhFvm+M95v2uzPIzXvL3bgGf18zPox77AeDrFEnCdfPsL2n1uILid+MTKKq/5wK/V677bWCE4m/0KRR/szMivh94KfDDmbmB4nfsncsbtjQ/k4T2+2hm/t/MnM7Mycz8bGZ+tXz9FYqL+qfM2OcPy22vp7iY/UBm3peZoxSJwA+V2/0q8CeZ+bXMnAL+GHhCRDxqCXGeCOxueF1/vmGWdfX1G1qw70K8JTPvyswJiqrKlsx8bWYeysw7gP8NXFJu+zngKWUToccBbylf95f7/hPAAj+H12Tm/vJ9nw38Q2b+U2YeBF5FkbBQHu/zmTm4iHM6KiIeB7wa+J8Ni2f7mZ44V7+EBpcCH8rMfUuJZR5vyMw9mXkbcCtwfWbekZm7gY/x8HfyRcBfZOZNmXkkM99LcUf/PIDMfHFmvniO95j3uzLLz3jJ37v5Pq+IeB6wHfjTxuVl3BsoErEPl+clSb8AvLb8O30/RbPHXyrXHQZOo6jyH87Mf87MpGh5sA44OyLWZOadmfntSqKX5mCS0H53Nb6IiCeVnSnvj4jdFBf6MzuY3tvwfGKW1yeWzx8F/HlEjJd3RR+iaG5SW0Kc+4CNDa/rz/fOsq6+vn6Xtpl9F6LxZ/goYGv9nMvz/l2KOzRQJAk/RtF856vADRQX/+cBOzPzQVjw59D4vlsbX2fmfopmR8cVEY9s7Fg8Y92ZFBfZv5GZ/9ywaraf6b7yj8tc77MeeA7w3oXEtUiL+U7+9ozP53SKn9/xLPa70vLvXVlN+hPg6TlLn44y8fk8RRXp1+Y7lqRVYytF1bruuzz8O++NwE7g+oi4IyJeCZCZOyn6Qb0GuC8iromIhfyelJaNSUL7zbyo+xtgB3B6Zm4C3kFxYb8UdwG/kpmDDY+BzPyXJRzrNooyad3jgXvLi+rbgEdHxIYZ629rwb4L0fgzvAv4zoxz3pCZF5Xr/4Wizfh/BT6XmbcDjwQu4uGmRrCwz6Hxfe+muNgFjl6Qb15Q8Jn/nsd2LK4f41HAJykqR381Y7fZfqbH+5n9V4pE8bMLiase3iK2XYi7gCtnfD7rM3MhzeAW+11p6fcuIi6kqEr9dGZ+9Tix9lH0y5CkXRQ3SOoeWS4jM/dm5m9n5qMpBj34rYg4v1z3N5n5I+W+STHohNQxTBKW3wbgocycjIhzKfosLNU7gMsjot7OelNEPGeujSNibdnsJoA1EdEfEfXvwPuA50fE2RExSNGe8j0AZf+KW4DfL/f5rxRNef6uBfsu1heBvRHxOxExEBG9EXFORPxw+X4HgC8DL+HhpOBfKCoFjUnCYj+HDwHPKNuyrwVeSxP/fyKiRtHZ/arMfMcsm7yP4o9Jrby79NuUP9N5XAq8b75qwyzup2g29ehF7DOf/w38almpiYg4ISJ+asbF+qyW8F1p2fcuIp4KvB/42cz84ox1J0fEJRFxYvl9u4CiD8anjndOklaFDwC/FxFbohh6+tXAXwNExDMi4syyqehuimZG01HMq/TUsr/WJEVFdnqO40uVMElYfi8GXhsReyl+kVy71ANl5kco7jxcExF7KNqKP32eXa6n+EX0nyk6nk5QdlzNzI8DbwA+QzHizXeB32/Y9xKKdtpjwOuAZ5dtL5vaN4qJuxZcVcjMIxSj5zyBYmSjB4B3ApsaNvscsIYioai/3kDZH6G0qM+hbIv/EooKxN3luYzU10fEk2c2JTqOF1BcmL9mjqZIf0HRQfurFJ/rP5bL6u+3LyKe3PC6BjyV4sL5GBHxjoiYLRGpJ1VXAv+3bB503iLOYbbj3Qy8ELiK4me0k2K0pOPGUprvu3LMz7jJ793Mz+tVFN+h6xo+j4/V34qiadFIeaw/BV6WmTsW+nORtKL9EXAzxSh8X6UYZvyPynVnUVSM9wFfAN6WmZ+h6I/wOoq/YfdQDDhx+fKGLc0vFnfTUZIkSdJKZyVBkiRJ0jFMEiRJkiQdwyRBkiRJ0jFMEiRJkiQdwyRBkiRJ0jH6qg5gKU466aTctm1b1WFIUsf58pe//EBmbqk6jqr5d0KSZrfQvxNdmSRs27aNm2++ueowJKnjRMR3q46hE/h3QpJmt9C/EzY3kiRJknQMkwRJkiRJxzBJkCRJknQMkwRJkiRJxzBJkCRJknQMkwRJkiRJxzBJkCRJknSMppKEiHhjRHw9Ir4SER+JiMGGdZdHxM6I+EZEXDDH/mdExE3ldh+MiLXNxCNJkiSpec1WEm4AzsnMxwHfBC4HiIizgUuAxwIXAm+LiN5Z9n898ObMPBMYA57fZDySJEmSmtTUjMuZeX3DyxuBZ5fPLwauycyDwHciYidwLvCF+sYREcBTgZ8vF70XeA3w9mZiklaDL3z7Qe58cH/VYSybE9f18VP/4TR6eqLqUCRJWhWaShJmeB7wwfJ5jSJpqBsplzXaDIxn5tQ82xwVES8CXgTwyEc+shXxSl3pyHRy6V9+kUNT01WHsqy2Dg7wxEcNVR2GJEmrwnGThIj4JHDqLKuuyMyPlttcAUwB729teA/LzKuBqwG2b9+e7XofqdPdu2eSQ1PTXP70H+DiJ8yZV68Y331wP8+9+kbueuiASUIHiYgLgT8HeoF3ZubrZqxfB7wPeCLwIPDczLwzIjYDHwJ+GHhPZr60YZ8nAu8BBoDrgN/IzCzX/Q/gJcAR4B8z8xXtPUNJWt2OmyRk5tPmWx8RlwHPAM6v/zIHRoHTGzYbLpc1ehAYjIi+spow2zaSZhgdnwDgB07byKmb+iuOpv02DhS/purnreqVfczeCvwERRX4SxGxIzNvb9js+cBYZp4ZEZdQ9EF7LjAJvAo4p3w0ejvwQuAmiiThQuBjEfHjFM1YH5+ZByPi5PadnSQJmh/d6ELgFcAzM/NAw6odwCURsS4izgDOAr7YuG+ZUHyGh/sxXAp8tJl4pNVgdKy4WK4NDlQcyfJYv7aPR5ywlpExk4QOci6wMzPvyMxDwDUUF/GNLqboawZF5eD8iIjM3J+Zn6dIFo6KiNOAjZl5Y/n34X3As8rVvwa8ruznRmbe15azkiQd1ezoRlcBG4AbIuKWiHgHQGbeBlwL3A58HHhJZh4BiIjrImJruf/vAL9VdmzeDLyryXikFa9+R321JAlQnKuVhI5SA+5qeD1bn7Kj25TV4t0Uv+fnO+bIHMd8DPDkcsjsz0XEDzcRuyRpAZod3ejMedZdCVw5y/KLGp7fQXFHStICjYxNsPmEtQysnW1U4ZWpNjjAt+7bW3UYqk4f8AjgPIq+DNdGxKMbmrgCDnAhSa3kjMtSlxkdn6A2tHqqCAC1oaKSMOOaUNVZSL+zo9tERB+wiaIv2nzHHJ7jmCPAh7PwRWAaOGnmATLz6szcnpnbt2zZsojTkSTNZJIgdZnRsQOrqqkRFJWEycPTPLT/UNWhqPAl4KyIOCMi1lJMnrljxjY7KPqaQdH37NMz7/w3ysy7gT0RcV45j84v83A/tb8HfhwgIh4DrAUeaNXJSJK+l0mC1EUys6gkrLYkoayc2C+hM5R9DF4KfAL4GnBtZt4WEa+NiGeWm70L2Fz2Ofst4JX1/SPiTuBNwGURMRIRZ5erXgy8E9gJfBv4WLn83cCjI+JWik7Sl86XcEiSmtfKydQktdnYgcNMHp5efc2NyqRo1/gEjxserDgaAWTmdRTDlDYue3XD80ngOXPsu22O5TfzvcOiUo6g9ItNhCtJWiQrCVIXqQ9/unW1VRLK83UYVEmSlodJgtRFRseL6UhWW3OjwfVrWL+21+ZGkiQtE5MEqYvU76QPr7LmRhFRzJVgJUGSpGVhkiB1kdHxCU5Y28umgTVVh7Ls6sOgSpKk9jNJkLrI6FgxR0IxQuTq4qzLkiQtH5MEqYusxuFP62pDA4wfOMz+g1NVhyJJ0opnkiB1kdU423JdPTmymiBJUvuZJEhdYv/BKcYPHKY2uL7qUCpR76xt52VJktrPJEHqEvU76Ku3klAkRyNWEiRJajuTBKlL1O+gr9Y+CSdvWMea3rCSIEnSMjBJkLpE/Q76apsjoa6nJzht0wC7rCRIktR2JglSl9g1PsGa3mDLieuqDqUyDoMqSdLyMEmQusTo2ASnbRqgp2f1zZFQt9VZlyVJWhYmCVKXWM1zJNTVhga4d+8kh6amqw5FkqQVzSRB6hL12ZZXs+HBATLhnt2TVYciSdKKZpIgdYFDU9Pcu3fSSkKZJI2MH6g4EkmSVjaTBKkL3LN7kszVO0dC3dFZl+2XIElSW5kkSF2gfud8eJVXEk4b7AdwhCNJktrMJEHqAkcnUlvllYR1fb2cvGGdlQRJktrMJEHqAqPjE0TAaZtWd5IARaJkJUGSpPYySZC6wOjYBCdvWMfaPv/LOqGaJEnt5xWH1AWcI+FhtaEB7h6fZHo6qw5FkqQVyyRB6gKj4xPUhtZXHUZHGB4c4NCRaR7Yd7DqUCRJWrFMEqQONz2d3D0+ydZyZJ/Vbutgfa4EmxxJktQuJglSh3tg30EOHZle9cOf1tVHeHKEI0mS2sckQepw9Tvmq33407qjE6pZSZAkqW1MEqQOd3SOhEH7JABs6F/Dxv4+KwmSJLWRSYLU4UatJHyP2tB6KwmSJLWRSYLU4UbHJtg0sIYT1/VVHUrHqA0OWEmQJKmNTBKkDuccCd9ruJx1OdO5EiRJageTBKnDjY5N2NRohtrgAPsOTrFnYqrqUCRJWpFMEqQOlplWEmZRT5pGxg9UHIkkSSuTSYLUwfZMTLHv4BTDVhKOcXQYVPslSJLUFiYJUger3ym3knCseiVhlyMcSZLUFiYJUgc7OkeClYRjbD5hLev6ehwGVZKkNjFJkDpY/U75VisJx4iIYhhUkwRJktqiqSQhIt4YEV+PiK9ExEciYrBh3eURsTMivhERF8yx//vL9bdGxLsjYk0z8Ugrzej4BP1reth8wtqqQ+k4tSHnSpAkqV2arSTcAJyTmY8DvglcDhARZwOXAI8FLgTeFhG9s+z/fuAHgP8ADAAvaDIeaUUZHZ9g6+AAEVF1KB3HSoIkSe3TVJKQmddnZn2g8huB4fL5xcA1mXkwM78D7ATOnWX/67IEfLFhf0mUcyTY1GhWtcEBHth3iMnDR6oORZKkFaeVfRKeB3ysfF4D7mpYN1Ium1XZzOiXgI+3MB6p642OTzj86RzqnbmtJkiS1HrHTRIi4pNln4GZj4sbtrkCmKJoPrQUbwP+KTP/eZ44XhQRN0fEzffff/8S30bqHpOHj/DAvkNWEubgXAnViogLyz5lOyPilbOsXxcRHyzX3xQR28rlmyPiMxGxLyKumrHPEyPiq+U+b4kZ7ewi4rcjIiPipHaemyQJ+o63QWY+bb71EXEZ8Azg/LLZEMAocHrDZsPlstn2/31gC/Arx4njauBqgO3bt+d820orQf0OucOfzs5KQnXKPmZvBX6ColL8pYjYkZm3N2z2fGAsM8+MiEuA1wPPBSaBVwHnlI9GbwdeCNwEXEfRp+1j5XueDvwk8O/tOi9J0sOaHd3oQuAVwDMz80DDqh3AJeWdpDOAsyj6HMzc/wXABcDPZeZ0M7FIK83RORIG11ccSWc6dWM/vT1hJaEa5wI7M/OOzDwEXEPRF63RxcB7y+cfAs6PiMjM/Zn5eYpk4aiIOA3YmJk3ljec3gc8q2GTN1P8vfEmkSQtg2b7JFwFbABuiIhbIuIdAJl5G3AtcDtFP4OXZOYRgIi4LiK2lvu/AzgF+EK5/6ubjEdaMawkzK+vt4dTN/ZbSajGQvqdHd2mHOBiN7D5OMccme2YZfPW0cz8t/mCslmqJLXOcZsbzSczz5xn3ZXAlbMsv6jheVPvL61ko2MT9PYEp2xYV3UoHas26FwJK11ErAd+l6Kp0bxslipJreOMy1KHGh2f4NSN/fT1+t90LrUh50qoyEL6nR3dJiL6gE3Ag8c5ZuMw2PVjfh9wBvBvEXFnufxfI+LUJuKXJB2HVx9Sh3KOhOPbOtjPPXsmmTpil6Zl9iXgrIg4IyLWUkyeuWPGNjuAS8vnzwY+3TC4xffIzLuBPRFxXjmq0S8DH83Mr2bmyZm5LTO3UTRD+o+ZeU+Lz0mS1MAkQepQo+MT9kc4jtrgeo5MJ/fuPVh1KKtK2cfgpcAngK8B12bmbRHx2oh4ZrnZu4DNEbET+C3g6DCpZUXgTcBlETESEWeXq14MvJNiAs5v8/DcO5KkZWafAKkDTR2Z5p49k1YSjuPoMKhWXZZdZl5HMUxp47JXNzyfBJ4zx77b5lh+M987LOqC9pUktZaVBKkD3bv3IEem00rCcRydUG38wHG2lCRJi2GSIHWgh+dIMEmYj7MuS5LUHiYJUgeq3xm3kjC/gbW9bD5hrSMcSZLUYiYJUgeykrBwtaEBRqwkSJLUUiYJUgcaHZ/gpBPX0r+mt+pQOl5t0LkSJElqNZMEqQONOFrPgtUGB9g1PsE8Q/BLkqRFMkmQOpBzJCxcbWiAycPTPLj/UNWhSJK0YpgkSB0mM9k1biVhoeo/p102OZIkqWVMEqQO8+D+Q0wenjZJWKDGCdUkSVJrmCRIHaZ+sbvVJGFBHp5QzSRBkqRWMUmQOkz9Ytc+CQuzaWANJ6ztdRhUSZJayCRB6jD1tvXDg+srjqQ7RAS1IYdBlSSplUwSpA4zMjbBiev62DjQV3UoXaM2OGCfBEmSWsgkQeowo+XIRhFRdShdw0qCJEmtZZIgdZjRMedIWKza4Hp2Txxm38GpqkORJGlFMEmQOsyocyQsmsOgSpLUWiYJUgfZd3CK3ROHrSQs0sPDoB6oOBJJklYGkwSpg9TvhFtJWJxhKwmSJLWUSYLUQep3wq0kLM6WE9extreHETsvS5LUEiYJUgep3wkftpKwKD09wWmD/ewan6w6FEmSVgSTBKmDjIxPsLa3h5NOXFd1KF1n66YBRsfskyBJUiuYJEgdZHRsgtMG++npcY6ExXKuBEmSWsckQeogDn+6dLXBAe7be5BDU9NVhyJJUtczSZA6yC6ThCWrDQ2QCXfvtpogSVKzTBKkDnFoapr79h50ZKMlqnf2dhhUSZKaZ5IgdYi7d0+Q6RwJS1VPrhwGVZKk5pkkSB3i6ERqVhKW5LRNA0RYSZAkqRVMEqQOUb8DPjy4vuJIutPavh5O3rDOEY4kSWoBkwSpQ4yOTRABp27qrzqUrlUbHLCSIElSC5gkSB1idHyCUzb0s7bP/5ZLVRtabyVBkqQW8GpE6hCjYxP2R2hSbXCAu3dPMD2dVYciSVJXM0mQOoQTqTWvNjTA4SPJ/fsOVh2KJEldzSRB6gDT08nduyfYapLQlNpg0Z9jxH4JkiQ1xSRB6gD37T3I4SNpc6Mm1cqRoeyXIElSc0wSpA4wOn4AeHjWYC1NPclyhCNJkppjkiB1gNHxScCJ1Jp14ro+Ng2sOZp0SZKkpWkqSYiIN0bE1yPiKxHxkYgYbFh3eUTsjIhvRMQFxznOWyJiXzOxSN3s6GzLVhKa5lwJkiQ1r9lKwg3AOZn5OOCbwOUAEXE2cAnwWOBC4G0R0TvbASJiOzDUZBxSVxsdP8Dg+jWcsK6v6lC6Xm1owD4JkiQ1qakkITOvz8yp8uWNwHD5/GLgmsw8mJnfAXYC587cv0wc3gi8opk4pG43Oubwp61SryRkOleCJElL1co+Cc8DPlY+rwF3NawbKZfN9FJgR2be3cI4pK7jHAmtMzw0wP5DR9g9cbjqUFa0iLiwbE66MyJeOcv6dRHxwXL9TRGxrVy+OSI+ExH7IuKqGfs8MSK+Wu7zloiIcvmcTVslSe1x3CQhIj4ZEbfO8ri4YZsrgCng/Qt944jYCjwH+F8L3P5FEXFzRNx8//33L/RtpI6Xmc623EL1ZMu5EtqnrAK/FXg6cDbwc2Uz00bPB8Yy80zgzcDry+WTwKuAl89y6LcDLwTOKh8XlstnbdoqSWqf4yYJmfm0zDxnlsdHASLiMuAZwC/kw/X9UeD0hsMMl8sa/RBwJrAzIu4E1kfEznniuDozt2fm9i1btiz0/KSOt3viMPsPHbGS0CJHh0G1X0I7nQvszMw7MvMQcA1FM9NGFwPvLZ9/CDg/IiIz92fm5ymShaMi4jRgY2beWP4teR/wLJi3aaskqU2aHd3oQor+BM/MzMYxB3cAl5Tl5jMo7gh9sXHfzPzHzDw1M7dl5jbgQHnHSVpV6ne8h60ktEQ92dplktBOC2lSenSb8gJ/N7D5OMccOc4x4dimrZKkNmm2T8JVwAbghoi4JSLeAZCZtwHXArcDHwdekplHACLiurKpkSQevuNdny1YzXnECWvpX9PjMKgr0PGattosVZJap6nxFue785+ZVwJXzrL8ojm2P7GZWKRuVb+Y3TrYX3EkK0NEsHXQYVDbbCFNSuvbjEREH7AJePA4x2xsRnTMMRuatp7f0LT1GJl5NXA1wPbt2x3eSpKa4IzLUsVGxyfoX9PDI05YW3UoK0bNJKHdvgScFRFnRMRainlxdszYZgdwafn82cCn57q4ByhHudsTEeeVoxr9MlDv+zZX01ZJUps4c5NUsfocCeVoj2qB4aEBbt+1p+owVqzMnIqIlwKfAHqBd2fmbRHxWuDmzNwBvAv4q3JAiocoEgkAysEqNgJrI+JZwE9m5u3Ai4H3AAMU/Q7qfQ+uAtZRNG0FuDEzf7XtJypJq5hJglSxXbsnqA3ZH6GVaoMDPLj/EBOHjjCwdtbJ3tWkzLwOuG7Gslc3PJ+kGOZ6tn23zbH8ZuCcWZY7qIUkLTObG0kVc7bl1nMYVEmSmmOSIFVo4tARHtx/yOFPW6w+UpRJgiRJS2OSIFXo4eFPTRJa6WglwWFQJUlaEpMEqUJHkwQrCS11yoZ19PYEo+MOhCNJ0lKYJEgVqt/ptpLQWn29PZy6sd9KgiRJS2SSIFVodPwAfT3BKRudSK3VakPOlSBJ0lKZJEgVGh2b4NRN/fT2OEdCqw0PDrBrfLLqMCRJ6komCVKFRscn2GpTo7bYOjjAPXsmmToyXXUokiR1HZMEqUKjYxMMmyS0RW1ogCPTyT17rCZIkrRYJglSRQ4fmeaePZOObNQm9c7gdl6WJGnxTBKkityze5LpdGSjdnHWZUmSls4kQarILudIaCsrCZIkLZ1JglQRZ1tur/41vZx04lorCZIkLYFJglSR+h1uRzdqn9qgcyVIkrQUJglSRUbHJzjpxHX0r+mtOpQVqzY0YHMjSZKWwCRBqsjo+IT9EdqsXknIzKpDkSSpq5gkSBVxjoT2qw0OcHBqmgf2Hao6FEmSuopJglSBzLSSsAxqQ+uBh0eSkiRJC2OSIFXggX2HODg17chGbXZ0GFSTBEmSFsUkQapA/aLVkY3ay7kSJElaGpMEqQL1i1YrCe21caCPE9f1WUmQJGmRTBKkCoyOHwCcbbndIoLa4AAjVhIkSVoUkwSpAqNjE2xY18emgTVVh7Li1YacUE2SpMUySZAqMDo+aRVxWefPAAAb80lEQVRhmdQGBxgdO1B1GJIkdRWTBKkCo+MT9kdYJrWhAfZMTrF38nDVoUiS1DVMEqQKjI4dsJKwTBwGVZKkxTNJkJbZ3snD7JmcspKwTOrJmMOgSpK0cCYJ0jKr39G2krA8hq0kSJK0aCYJ0jJzjoTlddKJ61jb22MlQZKkRTBJkJaZlYTl1dMTbB3st5IgSdIimCRIy2x0bIK1vT2cdMK6qkNZNbYOOleCJEmLYZIgLbOR8Qm2DvbT0xNVh7JqFHMlmCRIkrRQJgnSMhsdm7Cp0TKrDQ1w396DHJw6UnUokiR1BZMEaZk5kdryq/+87x6frDgSSZK6g0mCtIwmDx/h/r0HqQ2urzqUVeXoXAn2S5AkaUFMEqRldM/u4k62zY2W13CZlNkvQZKkhTFJkJbR0eFPbW60rE7d1E9E0WlckiQdn0mCtIzqd7KHrSQsq7V9PZyyod9KgiRJC9RUkhARb4yIr0fEVyLiIxEx2LDu8ojYGRHfiIgL5tg/IuLKiPhmRHwtIn69mXikTjcyPkFPFHe2tbxqQwOMjh+oOgxJkrpCs5WEG4BzMvNxwDeBywEi4mzgEuCxwIXA2yKid5b9LwNOB34gM38QuKbJeKSONjo2wSkb+1nTaxFvudWcUE2SpAVr6kolM6/PzKny5Y3AcPn8YuCazDyYmd8BdgLnznKIXwNem5nT5fHuayYeqdONjh+wP0JFakMD3LN7kiPTWXUoK0JEXFhWindGxCtnWb8uIj5Yrr8pIraVyzdHxGciYl9EXDVjnydGxFfLfd4SEVEuf0RE3BAR3yr/HVqOc5Sk1ayVtzOfB3ysfF4D7mpYN1Ium+n7gOdGxM0R8bGIOKuF8UgdZ3TcidSqUhsc4PCR5P69B6sOpeuVleG3Ak8HzgZ+rqwgN3o+MJaZZwJvBl5fLp8EXgW8fJZDvx14IXBW+biwXP5K4FOZeRbwqfK1JKmNjpskRMQnI+LWWR4XN2xzBTAFvH+R778OmMzM7cD/Bt49TxwvKpOJm++///5Fvo1UvSPTyd3jk2y1klCJegXHfgktcS6wMzPvyMxDFE1FL56xzcXAe8vnHwLOj4jIzP2Z+XmKZOGoiDgN2JiZN2ZmAu8DnjXLsd7bsFyS1CZ9x9sgM5823/qIuAx4BnB++YsdYJSir0HdcLlsphHgw+XzjwB/OU8cVwNXA2zfvt32Auo69+2dZGo6bW5UkXoFZ2Rsgic+quJgut9s1eInzbVNZk5FxG5gM/DAPMccmXHMegX6lMy8u3x+D3DK0kOXJC1Es6MbXQi8AnhmZjbentsBXFK2ST2Domz8xVkO8ffAj5fPn0LR+VlakerDb9rcqBoPVxLsvNzNyptRs94osuIsSa3TbJ+Eq4ANwA0RcUtEvAMgM28DrgVuBz4OvCQzjwBExHURsbXc/3XAz0bEV4E/AV7QZDxSx6pfnA5bSajECev6GFy/xrkSWmMh1eKj20REH7AJePA4xxxueN14zHvL5kj1ZkmzDnKRmVdn5vbM3L5ly5YFnookaTbHbW40n7JD2lzrrgSunGX5RQ3Px4GfaiYGqVuMWEmonMOgtsyXgLPKSvEoxZDXPz9jmx3ApcAXgGcDn25okvo9MvPuiNgTEecBNwG/DPyvGcd6XfnvR1t4LpKkWTSVJEhauNHxCYbWr2H9Wv/bVaU2OMB3HthfdRhdr+xj8FLgE0Av8O7MvC0iXgvcnJk7gHcBfxURO4GHKBIJACLiTmAjsDYingX8ZGbeDrwYeA8wQDFaXn3EvNcB10bE84HvAv+t/WcpSaubVyvSMtnl8KeVqw0N8PmdD5CZlEPwa4ky8zrguhnLXt3wfBJ4zhz7bptj+c3AObMsfxA4v4lwJUmL5LSv0jIZHZtwZKOK1QYHOHDoCOMHDlcdiiRJHc0kQVoGmVlMpDa4vupQVrXhIUc4kiRpIUwSpGUwfuAwBw4dsblRxepJ2ogjHEmSNC+TBGkZ1O9c29yoWvUkbZeVBEmS5mWSIC2Do8OfmiRUamj9GvrX9NjcSJKk4zBJkJbB0UqCzY0qFRHFXAk2N5IkaV4mCdIyGB2bYGBNL0Pr11QdyqpXG1pvJUGSpOMwSZCWwej4AWpDA47N3wGcdVmSpOMzSZCWQTH8qU2NOsHw0AAP7T/EgUNTVYciSVLHMkmQlsHomLMtd4p6suYIR5Ikzc0kQWqzA4emGDtw2EpCh6gna86VIEnS3EwSpDar37EetpLQEerJmv0SJEmam0mC1GbOkdBZTtnYT19POAyqJEnzMEmQ2sw5EjpLb09w6qZ+KwmSJM3DJEFqs9GxCfp6gpM39Fcdikq1wQE7LkuSNA+TBKnNRscnOG2wn94e50joFLUhZ12WJGk+JglSm42OTbB1k02NOkltcIB79kxy+Mh01aFIktSRTBKkNhsdd46ETlMbHGA64Z7dk1WHIklSRzJJkNro8JFp7t0zybAjG3WUetJm52VJkmZnkiC10T27J5lORzbqNEfnSrBfgiRJszJJkNro4TkS1lcciRptdUI1SZLmZZIgtZFzJHSm/jW9nHTiOisJkiTNwSRBaqP6Rehpm5wjodPUhgasJEiSNAeTBKmNdo1PsGXDOvrX9FYdimYYHjRJkCRpLiYJUhuNjk8c7SSrzlKvJExPZ9WhSJLUcUwSpDZyjoTOVRsc4NDUNA/sP1h1KJIkdRyTBKlNpqeT0fEJ50joUPUKz65xJ1STJGkmkwSpTR7Yf5BDU9NHh9tUZ9nqXAmSJM3JJEFqk9GjcySYJHSih2ddPlBxJJIkdR6TBKlNnCOhs20aWMOGdX1WEiRJmoVJgtQmRysJJgkdy7kSJEmanUmC1Caj4xNs6O9jY/+aqkPRHGqDA4xYSZAk6XuYJEhtMjrmHAmdzkqCJEmzM0mQ2mR0fIJhmxp1tNrgAHsnp9gzebjqUCRJ6igmCVKbWEnofEdHOLLJkSRJxzBJkNpgz+Rh9h6cstNyh6s5V4IkSbMySZDa4OE5EtZXHInm8/BcCSYJkiQ1MkmQ2sDhT7vDSSesY21fj0mCJEkzNJUkRMQbI+LrEfGViPhIRAw2rLs8InZGxDci4oI59j8/Iv41Im6JiM9HxJnNxCN1iqMTqdknoaP19AS1QUc4kiRppmYrCTcA52Tm44BvApcDRMTZwCXAY4ELgbdFRO8s+78d+IXMfALwN8DvNRmP1BFGxydY29fD5hPWVh2KjmPrYL99EiRJmqGpJCEzr8/MqfLljcBw+fxi4JrMPJiZ3wF2AufOdghgY/l8E7CrmXikTlEf2ainJ6oORcdhJWFpIuLCslK8MyJeOcv6dRHxwXL9TRGxrWHdrJXmiPiNiLg1Im6LiJc1LH9CRNxYVp1vjojZ/p5IklqolX0Sngd8rHxeA+5qWDdSLpvpBcB1ETEC/BLwuhbGI1VmZNzhT7tFbXA99+89yOThI1WH0jXKyvBbgacDZwM/V1aQGz0fGMvMM4E3A68v95210hwR5wAvpLih9HjgGQ1NUN8A/EFZdX51+VqS1EbHTRIi4pPlnZ2Zj4sbtrkCmALev8j3/03goswcBv4SeNM8cbyovIN08/3337/It5GWl3MkdI965/K7d09WHElXORfYmZl3ZOYh4BqKCnKji4H3ls8/BJwfEcHcleYfBG7KzANlhfpzwM+U+1t1lqRl1ne8DTLzafOtj4jLgGcA52dmlotHgdMbNhsulzXutwV4fGbeVC76IPDxeeK4GrgaYPv27TnXdlLVJg8f4YF9Bx3ZqEs0zpVwxkknVBxN15itWvykubbJzKmI2A1sLpffOGPfGnArcGVEbAYmgIuAm8ttXgZ8IiL+lOLm1n9u6dlIkr5Hs6MbXQi8AnhmZh5oWLUDuKRsk3oGcBbwxRm7jwGbIuIx5eufAL7WTDxSJ9jlyEZdZfjoXAkHjrOl2ikzv0bRJOl6ihtGtwD1NmC/BvxmZp5OUYF+12zHsOIsSa3TbJ+Eq4ANwA1lh7J3AGTmbcC1wO0Uv+xfkplHACLiuojYWpaTXwj8XUT8G0WfhP/ZZDxS5Y4Of2oloSucuqmfnnDW5UU6brW4cZuI6KNoJvTgfPtm5rsy84mZ+aMUN5K+WW5zKfDh8vnfMvtAGGTm1Zm5PTO3b9myZYmnJkmCBTQ3mk/ZIW2udVcCV86y/KKG5x8BPtJMDFKneXi2ZZOEbrCmt4dTNvYz4ghHi/El4KyyUjxK0RH552dss4Pi4v4LwLOBT2dmRsQO4G8i4k3AVhoqzRFxcmbeFxGPpOiPcF55rF3AU4DPAk8FvtXGc5Mk0WSSIOl77RqfoCeKO9TqDrXBASsJi1D2MXgp8AmgF3h3Zt4WEa8Fbs7MHRRNgv4qInYCD1EkEpTb1SvNUzRUmikqy5uBw+Xy8XL5C4E/LysSk8CLludMJWn1MkmQWmxkfIJTN/azpreVIwyrnWpDA3z5u2NVh9FVMvM64LoZy17d8HwSeM4c+85VaX7yHNt/HnhiM/FKkhbHqxipxUbHJuyP0GVqgwPcs3uSI9MOnCZJEpgkSC036kRqXac2NMDUdHLfXudKkCQJTBKkljoyndyze5KtJgldZWvDXAmSJMkkQWqpe/dMMjWdNjfqMsP1JMERjiRJAkwSpJYadSK1rlRP6kasJEiSBJgkSC1Vb64ybCWhq6xf28fQ+jVWEiRJKpkkSC1Uv8i0T0L3qQ05V4IkSXUmCVILjYxN8IgT1rJ+rVOQdJva4ICVBEmSSiYJUgs5/Gn3qg2uZ3RsgkznSpAkySRBaqHRsQMmCV2qNjTAxOEjjB04XHUokiRVziRBapHMZNf4pMOfdqmacyVIknSUSYLUImMHDjNx+IiVhC5VH5FqdPxAxZFIklQ9kwSpRep3oK0kdKejlYTxyYojkSSpeiYJUovU70BbSehOg+vXMLCm1+ZGkiRhkiC1TH22XpOE7hQRxVwJNjeSJMkkQWqV0fEJ1q/tZXD9mqpD0RI5V4IkSQWTBKlFRseKORIioupQtETOuixJUsEkQWqR0fEJOy13udrgAGMHDnPg0FTVoUiSVCmTBKlFnG25+x0dBtVqgiRplTNJkFpg/8Epxg8ctpLQ5epJ3oj9EiRJq5xJgtQC9c6uVhK6W81KgiRJgEmC1BL1i8phKwld7eQN/fT1hCMcSZJWPZMEqQUeriSsrzgSNaO3JzhtsN9KgiRp1TNJkFpgdHyCNb3ByRvWVR2KmlQbHGCXlQRJ0ipnkiC1wOjYBKdtGqCnxzkSul1tcL3NjSRJq55JgtQCo+MTbB3srzoMtUBtsJ9790xy+Mh01aFIklQZkwSpBYrZlu2PsBLUhgaYTrhn92TVoUiSVBmTBKlJh6amuXfvpHMkrBD1ZG/EzsuSpFXMJEFq0j27J8mEYedIWBGOzpVgvwRJ0ipmkiA1aWT8AICVhBXitE1F3xKHQZUkrWYmCVKT6heTzra8MvSv6WXLhnWMlsmfJEmrkUmC1KR6s5TTHN1oxagNDtjcSJK0qpkkSE0aHZvg5A3rWNfXW3UoapHa0IDNjSRJq5pJgtSk0fEJ+yOsMMODA+wan2R6OqsORZKkSpgkSE3aNT5hf4QVpjY0wKEj0zyw72DVoUiSVAmTBKkJ09PJrnHnSFhp6kmf/RIkSauVSYLUhAf2HeTQkWkrCSvMVpMESdIqZ5IgNWFk3OFPV6KjE6rZeVmStEqZJEhNODpHgs2NVpSN/WvY0N9nJUGStGo1lSRExBsj4usR8ZWI+EhEDJbLN0fEZyJiX0RcNc/+j4iIGyLiW+W/Q83EIy23USsJK1Zt0GFQJUmrV7OVhBuAczLzccA3gcvL5ZPAq4CXH2f/VwKfysyzgE+Vr6WuMTo2wcb+Pjb0r6k6FLXY8JATqs0nIi6MiG9ExM6I+J7f3RGxLiI+WK6/KSK2Nay7vFz+jYi4oGH5b0TErRFxW0S8bMbx/kd5U+q2iHhDO89NkgR9zeycmdc3vLwReHa5fD/w+Yg48ziHuBj4sfL5e4HPAr/TTExzyUw++bX72nForWJfHd1NbWh91WGoDWqDA3zh2w9yw+33Lvt7r+vr4Ucfs2XZ33ehIqIXeCvwE8AI8KWI2JGZtzds9nxgLDPPjIhLgNcDz42Is4FLgMcCW4FPRsRjgB8EXgicCxwCPh4R/5CZOyPixyn+Xjw+Mw9GxMnLdKqStGo1lSTM8Dzgg4vc55TMvLt8fg9wylwbRsSLgBcBPPKRj1xSgC98381L2k+azzMfv7XqENQGZ56ygf2HjlTye+PUjf3c+LvnL/v7LsK5wM7MvAMgIq6huIhvTBIuBl5TPv8QcFVERLn8msw8CHwnInaWxxsGbsrMA+UxPwf8DPAG4NeA15X7kJne8ZGkNjtukhARnwROnWXVFZn50XKbK4Ap4P1LDSQzMyLmnN40M68GrgbYvn37kqZB/Yf/8SNLjE6a26O3nFB1CGqDnz/3kWx/1BBHKph1ua83lv09F6kG3NXwegR40lzbZOZUROwGNpfLb5yxbw24FbgyIjYDE8BFQD1Dewzw5Ii4kqI568sz80stPSNJ0jGOmyRk5tPmWx8RlwHPAM7PzMX+Nb03Ik7LzLsj4jSgbXeHIoJzapvadXhJK0xvT/CDp22sOoxVIzO/FhGvB64H9gO3AEfK1X3AI4DzgB8Gro2IR8/8m9OKirMkqdDs6EYXAq8AnlkvES/SDuDS8vmlwEebiUeStCxGgdMbXg+Xy2bdJiL6gE3Ag/Ptm5nvyswnZuaPAmMUA2JAUW34cBa+CEwDJ80MKjOvzsztmbl9y5bO7dMhSd2g2dGNrgI2ADdExC0R8Y76ioi4E3gTcFlEjJSd1YiId0bE9nKz1wE/ERHfAp5WvpYkdbYvAWdFxBkRsZaiI/KOGds03gR6NvDp8s7/DuCScvSjM4CzgC8C1DskR8QjKfoj/E25/98DP16uewywFnigTecmSaL50Y3mHL0oM7fNsfwFDc8fBDq6d54k6VhlH4OXAp8AeoF3Z+ZtEfFa4ObM3AG8C/irsmPyQxSJBOV211J0cp4CXpKZ9WZFf1f2SThcLh8vl78beHdE3Eox8tGlS2jeKklahFaObiRJWiUy8zrguhnLXt3wfBJ4zhz7XglcOcvyJ8+x/SHgF5uJV5K0OM02N5IkSZK0wpgkSJIkSTqGSYIkSZKkY5gkSJIkSTqGSYIkSZKkY5gkSJIkSTpGdONQ0xFxP/DdJe5+EqtnEp7VdK7g+a5kq+lcobnzfVRmrvrphpv8O1EFv+Mrm+e7cnXjuS7o70RXJgnNiIibM3P78bfsfqvpXMHzXclW07nC6jtfrb7P3PNd2VbT+a7kc7W5kSRJkqRjmCRIkiRJOsZqTBKurjqAZbSazhU835VsNZ0rrL7z1er7zD3flW01ne+KPddV1ydBkiRJ0vxWYyVBkiRJ0jxWTZIQERdGxDciYmdEvLLqeNopIk6PiM9ExO0RcVtE/EbVMbVbRPRGxP+LiH+oOpZ2i4jBiPhQRHw9Ir4WEf+p6pjaKSJ+s/we3xoRH4iI/qpjaqWIeHdE3BcRtzYse0RE3BAR3yr/HaoyRrXGQj/XiLi03OZbEXHpLOt3NH5fOlUz5xsR6yPiH8vfc7dFxOuWN/qFOd61RUSsi4gPlutviohtDesuL5d/IyIuWM64l2qp5xsRPxERX46Ir5b/PnW5Y1+KZj7fcv0jI2JfRLx8uWJupVWRJEREL/BW4OnA2cDPRcTZ1UbVVlPAb2fm2cB5wEtW+PkC/AbwtaqDWCZ/Dnw8M38AeDwr+Lwjogb8OrA9M88BeoFLqo2q5d4DXDhj2SuBT2XmWcCnytfqfsf9XCPiEcDvA08CzgV+v/HiOiJ+Bti3POE2rdnz/dPy99wPAf8lIp6+PGEvzAKvLZ4PjGXmmcCbgdeX+55N8bvssRT//99WHq9jNXO+FPMI/HRm/gfgUuCvlifqpWvyfOveBHys3bG2y6pIEih+8ezMzDsy8xBwDXBxxTG1TWbenZn/Wj7fS3ERWas2qvaJiGHgp4B3Vh1Lu0XEJuBHgXcBZOahzByvNqq26wMGIqIPWA/sqjielsrMfwIemrH4YuC95fP3As9a1qDULgv5XC8AbsjMhzJzDLiBMomMiBOB3wL+aBlibYUln29mHsjMz0Dxew74V2B4GWJejIVcWzT+DD4EnB8RUS6/JjMPZuZ3gJ3l8TrZks83M/9fZtZ/d99G8Tt93bJEvXTNfL5ExLOA71Ccb1daLUlCDbir4fUIK/iiuVFZ+voh4KZqI2mrPwNeAUxXHcgyOAO4H/jLsnnVOyPihKqDapfMHAX+FPh34G5gd2ZeX21Uy+KUzLy7fH4PcEqVwahlFvK5zvf36g+B/w840LYIW6vZ8wWKJpbAT1NUIzrJQq4tjm6TmVPAbmDzAvftNM2cb6OfBf41Mw+2Kc5WWfL5lgn97wB/sAxxts1qSRJWpfJL+nfAyzJzT9XxtENEPAO4LzO/XHUsy6QP+I/A2zPzh4D9rOCmKGWzg4spkqOtwAkR8YvVRrW8shiCzmHoukREfLLsPzPzccwdyMV+rhHxBOD7MvMjrY65Ge0634bj9wEfAN6SmXe0KGxVJCIeS9Ek51eqjqXNXgO8OTO7pWngrPqqDmCZjAKnN7weLpetWBGxhiJBeH9mfrjqeNrovwDPjIiLgH5gY0T8dWau1AvJEWAkM+uVoQ+xgpME4GnAdzLzfoCI+DDwn4G/rjSq9rs3Ik7LzLsj4jTgvqoD0sJk5tPmWhcRC/lcR4Efa3g9DHwW+E/A9oi4k+Jv98kR8dnM/DEq1Mbzrbsa+FZm/lkLwm21hVxb1LcZKROeTcCDC9y30zRzvvWmwR8Bfjkzv93+cJvWzPk+CXh2RLwBGASmI2IyM69qf9its1oqCV8CzoqIMyJiLUVnoR0Vx9Q2ZXu4dwFfy8w3VR1PO2Xm5Zk5nJnbKD7XT6/gBIHMvAe4KyK+v1x0PnB7hSG1278D55UjnQTF+a7YjtoNdlB07qP896MVxqLWWcjn+gngJyNiqKyk/STwicx8e2ZuLX/X/QjwzaoThAVY8vkCRMQfUVx0vWwZYl2KhVxbNP4Mnk3xNyrL5ZeUo+OcAZwFfHGZ4l6qJZ9v2WTsH4FXZub/XbaIm7Pk883MJ2fmtvL/658Bf9xtCQIAmbkqHsBFwDeBbwNXVB1Pm8/1RyjKul8BbikfF1Ud1zKc948B/1B1HMtwnk8Abi4/378HhqqOqc3n+wfA14FbKUbEWFd1TC0+vw9Q9Lc4TFEpej5FG95PAd8CPgk8ouo4fbTks571cwW2A+9s2O55FB1ZdwL/fZbjbANurfp82nm+FHdtk+KmQP3v2AuqPqdZzvF7ri2A1wLPLJ/3A39bntsXgUc37HtFud83gKdXfS7tPF/g9yiax97S8Di56vNp5+fbcIzXAC+v+lyW8nDGZUmSJEnHWC3NjSRJkiQtkEmCJEmSpGOYJEiSJEk6hkmCJEmSpGOYJEiSJEk6hkmCJEmSpGOYJEiSJEk6hkmCJEmSpGP8/7ERjTkKWELYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start=timer()\n",
    "\n",
    "env_id = \"PongNoFrameskip-v4\"\n",
    "env    = make_atari(env_id)\n",
    "env    = wrap_deepmind(env, frame_stack=False)\n",
    "env    = wrap_pytorch(env)\n",
    "model = Model(env=env, config=config)\n",
    "\n",
    "episode_reward = 0\n",
    "\n",
    "observation = env.reset()\n",
    "for frame_idx in range(1, config.MAX_FRAMES + 1):\n",
    "    epsilon = config.epsilon_by_frame(frame_idx)\n",
    "    \n",
    "    action = model.get_action(observation, epsilon)\n",
    "    prev_observation=observation\n",
    "    observation, reward, done, _ = env.step(action)\n",
    "    observation = None if done else observation\n",
    "\n",
    "    model.update(prev_observation, action, reward, observation, frame_idx)\n",
    "    episode_reward += reward\n",
    "\n",
    "    if done:\n",
    "        model.finish_nstep()\n",
    "        model.reset_hx()\n",
    "        observation = env.reset()\n",
    "        model.save_reward(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "        if np.mean(model.rewards[-10:]) > 19:\n",
    "            plot(frame_idx, model.rewards, model.losses, model.sigma_parameter_mag, timedelta(seconds=int(timer()-start)))\n",
    "            break\n",
    "\n",
    "    if frame_idx % 10000 == 0:\n",
    "        plot(frame_idx, model.rewards, model.losses, model.sigma_parameter_mag, timedelta(seconds=int(timer()-start)))\n",
    "\n",
    "model.save_w()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}