第16章-模型预测控制.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 6698,
     "status": "ok",
     "timestamp": 1649956814219,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "pkDNguALCr-X"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.stats import truncnorm\n",
    "import gym\n",
    "import itertools\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "class CEM:\n",
    "    def __init__(self, n_sequence, elite_ratio, fake_env, upper_bound,\n",
    "                 lower_bound):\n",
    "        self.n_sequence = n_sequence\n",
    "        self.elite_ratio = elite_ratio\n",
    "        self.upper_bound = upper_bound\n",
    "        self.lower_bound = lower_bound\n",
    "        self.fake_env = fake_env\n",
    "\n",
    "    def optimize(self, state, init_mean, init_var):\n",
    "        mean, var = init_mean, init_var\n",
    "        X = truncnorm(-2, 2, loc=np.zeros_like(mean), scale=np.ones_like(var))\n",
    "        state = np.tile(state, (self.n_sequence, 1))\n",
    "\n",
    "        for _ in range(5):\n",
    "            lb_dist, ub_dist = mean - self.lower_bound, self.upper_bound - mean\n",
    "            constrained_var = np.minimum(\n",
    "                np.minimum(np.square(lb_dist / 2), np.square(ub_dist / 2)),\n",
    "                var)\n",
    "            # 生成动作序列\n",
    "            action_sequences = [X.rvs() for _ in range(self.n_sequence)\n",
    "                                ] * np.sqrt(constrained_var) + mean\n",
    "            # 计算每条动作序列的累积奖励\n",
    "            returns = self.fake_env.propagate(state, action_sequences)[:, 0]\n",
    "            # 选取累积奖励最高的若干条动作序列\n",
    "            elites = action_sequences[np.argsort(\n",
    "                returns)][-int(self.elite_ratio * self.n_sequence):]\n",
    "            new_mean = np.mean(elites, axis=0)\n",
    "            new_var = np.var(elites, axis=0)\n",
    "            # 更新动作序列分布\n",
    "            mean = 0.1 * mean + 0.9 * new_mean\n",
    "            var = 0.1 * var + 0.9 * new_var\n",
    "\n",
    "        return mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1649956814220,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "coGG5UOpCr-Z"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\n",
    "    \"cpu\")\n",
    "\n",
    "\n",
    "class Swish(nn.Module):\n",
    "    ''' Swish激活函数 '''\n",
    "    def __init__(self):\n",
    "        super(Swish, self).__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x * torch.sigmoid(x)\n",
    "\n",
    "\n",
    "def init_weights(m):\n",
    "    ''' 初始化模型权重 '''\n",
    "    def truncated_normal_init(t, mean=0.0, std=0.01):\n",
    "        torch.nn.init.normal_(t, mean=mean, std=std)\n",
    "        while True:\n",
    "            cond = (t < mean - 2 * std) | (t > mean + 2 * std)\n",
    "            if not torch.sum(cond):\n",
    "                break\n",
    "            t = torch.where(\n",
    "                cond,\n",
    "                torch.nn.init.normal_(torch.ones(t.shape, device=device),\n",
    "                                      mean=mean,\n",
    "                                      std=std), t)\n",
    "        return t\n",
    "\n",
    "    if type(m) == nn.Linear or isinstance(m, FCLayer):\n",
    "        truncated_normal_init(m.weight, std=1 / (2 * np.sqrt(m._input_dim)))\n",
    "        m.bias.data.fill_(0.0)\n",
    "\n",
    "\n",
    "class FCLayer(nn.Module):\n",
    "    ''' 集成之后的全连接层 '''\n",
    "    def __init__(self, input_dim, output_dim, ensemble_size, activation):\n",
    "        super(FCLayer, self).__init__()\n",
    "        self._input_dim, self._output_dim = input_dim, output_dim\n",
    "        self.weight = nn.Parameter(\n",
    "            torch.Tensor(ensemble_size, input_dim, output_dim).to(device))\n",
    "        self._activation = activation\n",
    "        self.bias = nn.Parameter(\n",
    "            torch.Tensor(ensemble_size, output_dim).to(device))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self._activation(\n",
    "            torch.add(torch.bmm(x, self.weight), self.bias[:, None, :]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1649956814220,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "SNVDgXI2Cr-a"
   },
   "outputs": [],
   "source": [
    "class EnsembleModel(nn.Module):\n",
    "    ''' 环境模型集成 '''\n",
    "    def __init__(self,\n",
    "                 state_dim,\n",
    "                 action_dim,\n",
    "                 ensemble_size=5,\n",
    "                 learning_rate=1e-3):\n",
    "        super(EnsembleModel, self).__init__()\n",
    "        # 输出包括均值和方差,因此是状态与奖励维度之和的两倍\n",
    "        self._output_dim = (state_dim + 1) * 2\n",
    "        self._max_logvar = nn.Parameter((torch.ones(\n",
    "            (1, self._output_dim // 2)).float() / 2).to(device),\n",
    "                                        requires_grad=False)\n",
    "        self._min_logvar = nn.Parameter((-torch.ones(\n",
    "            (1, self._output_dim // 2)).float() * 10).to(device),\n",
    "                                        requires_grad=False)\n",
    "\n",
    "        self.layer1 = FCLayer(state_dim + action_dim, 200, ensemble_size,\n",
    "                              Swish())\n",
    "        self.layer2 = FCLayer(200, 200, ensemble_size, Swish())\n",
    "        self.layer3 = FCLayer(200, 200, ensemble_size, Swish())\n",
    "        self.layer4 = FCLayer(200, 200, ensemble_size, Swish())\n",
    "        self.layer5 = FCLayer(200, self._output_dim, ensemble_size,\n",
    "                              nn.Identity())\n",
    "        self.apply(init_weights)  # 初始化环境模型中的参数\n",
    "        self.optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate)\n",
    "\n",
    "    def forward(self, x, return_log_var=False):\n",
    "        ret = self.layer5(self.layer4(self.layer3(self.layer2(\n",
    "            self.layer1(x)))))\n",
    "        mean = ret[:, :, :self._output_dim // 2]\n",
    "        # 在PETS算法中,将方差控制在最小值和最大值之间\n",
    "        logvar = self._max_logvar - F.softplus(\n",
    "            self._max_logvar - ret[:, :, self._output_dim // 2:])\n",
    "        logvar = self._min_logvar + F.softplus(logvar - self._min_logvar)\n",
    "        return mean, logvar if return_log_var else torch.exp(logvar)\n",
    "\n",
    "    def loss(self, mean, logvar, labels, use_var_loss=True):\n",
    "        inverse_var = torch.exp(-logvar)\n",
    "        if use_var_loss:\n",
    "            mse_loss = torch.mean(torch.mean(torch.pow(mean - labels, 2) *\n",
    "                                             inverse_var,\n",
    "                                             dim=-1),\n",
    "                                  dim=-1)\n",
    "            var_loss = torch.mean(torch.mean(logvar, dim=-1), dim=-1)\n",
    "            total_loss = torch.sum(mse_loss) + torch.sum(var_loss)\n",
    "        else:\n",
    "            mse_loss = torch.mean(torch.pow(mean - labels, 2), dim=(1, 2))\n",
    "            total_loss = torch.sum(mse_loss)\n",
    "        return total_loss, mse_loss\n",
    "\n",
    "    def train(self, loss):\n",
    "        self.optimizer.zero_grad()\n",
    "        loss += 0.01 * torch.sum(self._max_logvar) - 0.01 * torch.sum(\n",
    "            self._min_logvar)\n",
    "        loss.backward()\n",
    "        self.optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1649956814221,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "kVE0nKi6Cr-b"
   },
   "outputs": [],
   "source": [
    "class EnsembleDynamicsModel:\n",
    "    ''' 环境模型集成,加入精细化的训练 '''\n",
    "    def __init__(self, state_dim, action_dim, num_network=5):\n",
    "        self._num_network = num_network\n",
    "        self._state_dim, self._action_dim = state_dim, action_dim\n",
    "        self.model = EnsembleModel(state_dim,\n",
    "                                   action_dim,\n",
    "                                   ensemble_size=num_network)\n",
    "        self._epoch_since_last_update = 0\n",
    "\n",
    "    def train(self,\n",
    "              inputs,\n",
    "              labels,\n",
    "              batch_size=64,\n",
    "              holdout_ratio=0.1,\n",
    "              max_iter=20):\n",
    "        # 设置训练集与验证集\n",
    "        permutation = np.random.permutation(inputs.shape[0])\n",
    "        inputs, labels = inputs[permutation], labels[permutation]\n",
    "        num_holdout = int(inputs.shape[0] * holdout_ratio)\n",
    "        train_inputs, train_labels = inputs[num_holdout:], labels[num_holdout:]\n",
    "        holdout_inputs, holdout_labels = inputs[:\n",
    "                                                num_holdout], labels[:\n",
    "                                                                     num_holdout]\n",
    "        holdout_inputs = torch.from_numpy(holdout_inputs).float().to(device)\n",
    "        holdout_labels = torch.from_numpy(holdout_labels).float().to(device)\n",
    "        holdout_inputs = holdout_inputs[None, :, :].repeat(\n",
    "            [self._num_network, 1, 1])\n",
    "        holdout_labels = holdout_labels[None, :, :].repeat(\n",
    "            [self._num_network, 1, 1])\n",
    "\n",
    "        # 保留最好的结果\n",
    "        self._snapshots = {i: (None, 1e10) for i in range(self._num_network)}\n",
    "\n",
    "        for epoch in itertools.count():\n",
    "            # 定义每一个网络的训练数据\n",
    "            train_index = np.vstack([\n",
    "                np.random.permutation(train_inputs.shape[0])\n",
    "                for _ in range(self._num_network)\n",
    "            ])\n",
    "            # 所有真实数据都用来训练\n",
    "            for batch_start_pos in range(0, train_inputs.shape[0], batch_size):\n",
    "                batch_index = train_index[:, batch_start_pos:batch_start_pos +\n",
    "                                          batch_size]\n",
    "                train_input = torch.from_numpy(\n",
    "                    train_inputs[batch_index]).float().to(device)\n",
    "                train_label = torch.from_numpy(\n",
    "                    train_labels[batch_index]).float().to(device)\n",
    "\n",
    "                mean, logvar = self.model(train_input, return_log_var=True)\n",
    "                loss, _ = self.model.loss(mean, logvar, train_label)\n",
    "                self.model.train(loss)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                mean, logvar = self.model(holdout_inputs, return_log_var=True)\n",
    "                _, holdout_losses = self.model.loss(mean,\n",
    "                                                    logvar,\n",
    "                                                    holdout_labels,\n",
    "                                                    use_var_loss=False)\n",
    "                holdout_losses = holdout_losses.cpu()\n",
    "                break_condition = self._save_best(epoch, holdout_losses)\n",
    "                if break_condition or epoch > max_iter:  # 结束训练\n",
    "                    break\n",
    "\n",
    "    def _save_best(self, epoch, losses, threshold=0.1):\n",
    "        updated = False\n",
    "        for i in range(len(losses)):\n",
    "            current = losses[i]\n",
    "            _, best = self._snapshots[i]\n",
    "            improvement = (best - current) / best\n",
    "            if improvement > threshold:\n",
    "                self._snapshots[i] = (epoch, current)\n",
    "                updated = True\n",
    "        self._epoch_since_last_update = 0 if updated else self._epoch_since_last_update + 1\n",
    "        return self._epoch_since_last_update > 5\n",
    "\n",
    "    def predict(self, inputs, batch_size=64):\n",
    "        mean, var = [], []\n",
    "        for i in range(0, inputs.shape[0], batch_size):\n",
    "            input = torch.from_numpy(\n",
    "                inputs[i:min(i +\n",
    "                             batch_size, inputs.shape[0])]).float().to(device)\n",
    "            cur_mean, cur_var = self.model(input[None, :, :].repeat(\n",
    "                [self._num_network, 1, 1]),\n",
    "                                           return_log_var=False)\n",
    "            mean.append(cur_mean.detach().cpu().numpy())\n",
    "            var.append(cur_var.detach().cpu().numpy())\n",
    "        return np.hstack(mean), np.hstack(var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1649956814221,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "1auD04WgCr-c"
   },
   "outputs": [],
   "source": [
    "class FakeEnv:\n",
    "    def __init__(self, model):\n",
    "        self.model = model\n",
    "\n",
    "    def step(self, obs, act):\n",
    "        inputs = np.concatenate((obs, act), axis=-1)\n",
    "        ensemble_model_means, ensemble_model_vars = self.model.predict(inputs)\n",
    "        ensemble_model_means[:, :, 1:] += obs.numpy()\n",
    "        ensemble_model_stds = np.sqrt(ensemble_model_vars)\n",
    "        ensemble_samples = ensemble_model_means + np.random.normal(\n",
    "            size=ensemble_model_means.shape) * ensemble_model_stds\n",
    "\n",
    "        num_models, batch_size, _ = ensemble_model_means.shape\n",
    "        models_to_use = np.random.choice(\n",
    "            [i for i in range(self.model._num_network)], size=batch_size)\n",
    "        batch_inds = np.arange(0, batch_size)\n",
    "        samples = ensemble_samples[models_to_use, batch_inds]\n",
    "        rewards, next_obs = samples[:, :1], samples[:, 1:]\n",
    "        return rewards, next_obs\n",
    "\n",
    "    def propagate(self, obs, actions):\n",
    "        with torch.no_grad():\n",
    "            obs = np.copy(obs)\n",
    "            total_reward = np.expand_dims(np.zeros(obs.shape[0]), axis=-1)\n",
    "            obs, actions = torch.as_tensor(obs), torch.as_tensor(actions)\n",
    "            for i in range(actions.shape[1]):\n",
    "                action = torch.unsqueeze(actions[:, i], 1)\n",
    "                rewards, next_obs = self.step(obs, action)\n",
    "                total_reward += rewards\n",
    "                obs = torch.as_tensor(next_obs)\n",
    "            return total_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1649956814222,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "Kl3fh7_iCr-c"
   },
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = collections.deque(maxlen=capacity)\n",
    "\n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        self.buffer.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def size(self):\n",
    "        return len(self.buffer)\n",
    "\n",
    "    def return_all_samples(self):\n",
    "        all_transitions = list(self.buffer)\n",
    "        state, action, reward, next_state, done = zip(*all_transitions)\n",
    "        return np.array(state), action, reward, np.array(next_state), done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1649956814723,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "7iPZNkXHCr-d"
   },
   "outputs": [],
   "source": [
    "class PETS:\n",
    "    ''' PETS算法 '''\n",
    "    def __init__(self, env, replay_buffer, n_sequence, elite_ratio,\n",
    "                 plan_horizon, num_episodes):\n",
    "        self._env = env\n",
    "        self._env_pool = ReplayBuffer(buffer_size)\n",
    "\n",
    "        obs_dim = env.observation_space.shape[0]\n",
    "        self._action_dim = env.action_space.shape[0]\n",
    "        self._model = EnsembleDynamicsModel(obs_dim, self._action_dim)\n",
    "        self._fake_env = FakeEnv(self._model)\n",
    "        self.upper_bound = env.action_space.high[0]\n",
    "        self.lower_bound = env.action_space.low[0]\n",
    "\n",
    "        self._cem = CEM(n_sequence, elite_ratio, self._fake_env,\n",
    "                        self.upper_bound, self.lower_bound)\n",
    "        self.plan_horizon = plan_horizon\n",
    "        self.num_episodes = num_episodes\n",
    "\n",
    "    def train_model(self):\n",
    "        env_samples = self._env_pool.return_all_samples()\n",
    "        obs = env_samples[0]\n",
    "        actions = np.array(env_samples[1])\n",
    "        rewards = np.array(env_samples[2]).reshape(-1, 1)\n",
    "        next_obs = env_samples[3]\n",
    "        inputs = np.concatenate((obs, actions), axis=-1)\n",
    "        labels = np.concatenate((rewards, next_obs - obs), axis=-1)\n",
    "        self._model.train(inputs, labels)\n",
    "\n",
    "    def mpc(self):\n",
    "        mean = np.tile((self.upper_bound + self.lower_bound) / 2.0,\n",
    "                       self.plan_horizon)\n",
    "        var = np.tile(\n",
    "            np.square(self.upper_bound - self.lower_bound) / 16,\n",
    "            self.plan_horizon)\n",
    "        obs, done, episode_return = self._env.reset(), False, 0\n",
    "        while not done:\n",
    "            actions = self._cem.optimize(obs, mean, var)\n",
    "            action = actions[:self._action_dim]  # 选取第一个动作\n",
    "            next_obs, reward, done, _ = self._env.step(action)\n",
    "            self._env_pool.add(obs, action, reward, next_obs, done)\n",
    "            obs = next_obs\n",
    "            episode_return += reward\n",
    "            mean = np.concatenate([\n",
    "                np.copy(actions)[self._action_dim:],\n",
    "                np.zeros(self._action_dim)\n",
    "            ])\n",
    "        return episode_return\n",
    "\n",
    "    def explore(self):\n",
    "        obs, done, episode_return = self._env.reset(), False, 0\n",
    "        while not done:\n",
    "            action = self._env.action_space.sample()\n",
    "            next_obs, reward, done, _ = self._env.step(action)\n",
    "            self._env_pool.add(obs, action, reward, next_obs, done)\n",
    "            obs = next_obs\n",
    "            episode_return += reward\n",
    "        return episode_return\n",
    "\n",
    "    def train(self):\n",
    "        return_list = []\n",
    "        explore_return = self.explore()  # 先进行随机策略的探索来收集一条序列的数据\n",
    "        print('episode: 1, return: %d' % explore_return)\n",
    "        return_list.append(explore_return)\n",
    "\n",
    "        for i_episode in range(self.num_episodes - 1):\n",
    "            self.train_model()\n",
    "            episode_return = self.mpc()\n",
    "            return_list.append(episode_return)\n",
    "            print('episode: %d, return: %d' % (i_episode + 2, episode_return))\n",
    "        return return_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 469
    },
    "executionInfo": {
     "elapsed": 641756,
     "status": "ok",
     "timestamp": 1649957456469,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "pfzBBzuECr-d",
    "outputId": "4b8d971d-31e3-47c6-aa29-00f0f4c062a3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 1, return: -985\n",
      "episode: 2, return: -1384\n",
      "episode: 3, return: -1006\n",
      "episode: 4, return: -1853\n",
      "episode: 5, return: -378\n",
      "episode: 6, return: -123\n",
      "episode: 7, return: -124\n",
      "episode: 8, return: -122\n",
      "episode: 9, return: -124\n",
      "episode: 10, return: -125\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEWCAYAAACjYXoKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xc9ZX//9dRsdzlJhfJxgVccJOwTQsxKUAoATv0npDkB6GFZMluFlLYLEk25QtLQkIgpAAJYJKl2bRQAwQSihuS3HDDILnJNh65yFY7vz/myh7bkjXSzOjOWO/n43EfmvncMkdja87cz7n38zF3R0REJBFZYQcgIiKZT8lEREQSpmQiIiIJUzIREZGEKZmIiEjClExERCRhSiYihwgze9XM/r9kbysSDyUTSTtm9oGZ1ZjZdjPbYGb3m1nPYN2rZrYrWNe0PGVml8Y8rzGzxthtgn0/aWb/NLOImW0xszfN7OgO+H1iY95kZo+b2ZBUv24mMLN+ZvaEme0wszVmdknYMUn7KJlIujrL3XsCU4BpwPdi1l3v7j1jlrPc/aGm58DpwNrYbcysN/A08CugH1AE/Dewu4N+n+uD2MYAfYA7Ouh1091dQC0wCLgUuNvMJoQbkrSHkomkNXevBJ4DJiZ4qDHB8Wa5e4O717j7C+5e2tzGZpZnZr8ws7XB8gszywvWfdrMKszsW2a20czWmdmX4/x9tgCPNf0+ZjbOzF4MzpSWmdkFMTHcb2Z3mdkzZrbNzN42s8Nj1p9iZkuDM61fAxaz7gdm9mDM8xFm5maW08zvetBtgzOrHwVndU1ngv3N7CEzqzazd81sRAvv491mdtt+bbPN7EYz6wGcC3zf3be7+xvAHODyeN5LSS9KJpLWzGwYcAawIMFDvQ80mNkDZna6mfVtZfvvAscBJUAxcAz7nh0NBvKJnuF8FbgrjmNiZgOIfoAuCD5MXwQeBgYCFwG/MbPxMbtcRPQMqi+wAvhxzHEeD2IaAKwETmjt9RNwEdEP+SLgcOBfwH1Ez/KWAP/Vwn6zgAvNzIK4+wKfAx4hmuDr3f39mO3fA3RmkoGUTCRdPWlmW4E3gNeA/4lZd6eZbY1Zftjawdy9Gvgk4MDvgCozm2Nmg1rY5VLgVnff6O5VRD/QY78x1wXr69z9WWA7MPYgIdwZ/D7vAeuAG4EzgQ/c/T53r3f3BUTPWs6P2e8Jd3/H3euBh4gmN4gm2EXu/qi71wG/ANa39j4k4D53X+nuEaJniivd/aUgrv8Djmphv38Qfc+nB8/PA/7l7muBnkD1fttHgF5Jj15STslE0tUX3L2Puw9392vdvSZm3Q3Buqbl+/Ec0N2XuPsV7j6UaDdTIdEP4eYUAmtinq8J2ppsDj5Im+wk+uHYkqaYi9z90iBBDQeOjU2MRJPY4Jj9YhNE7GsUAh/F/G4e+zwFNsQ8rmnmedMFEt+JufDhniCuR4CLg20vIZoUIZqAe+/3Or2BbckOXlJPyUQ6JXdfCtxPy7WYtUQ/7JscFrQl00fAa/slxp7ufk0c+64DhjU9CbqRhsWs3wF0j3kem6D215ZtD8rd/yfmwoerg+ZZwHlmNhw4lujZF0S7HnPMbHTMIYqBRe19fQmPkol0CkGh+1tmNjR4Pozot+W3WthlFvA9MysI6hO3AA+2sG17PQ2MMbPLzSw3WI42syPj2PcZYIKZnRMUym9g3ySwEDjRzA4zs3zg5oMcqy3btlnQfbcJ+D3wvLtvDdp3EK373GpmPczsBGAm8Odkvr50DCUTyUS/tn3vM5kXxz7biH4rftvMdhBNIuXAt1rY/kfAXKAUKAPmB21J4+7biBajLyJ61rMe+BmQF8e+m4jWVn4KbAZGA2/GrH8R+EsQ/zyiiaulY8W9bQIeBk4Ofsa6FugGbCSawK9xd52ZZCDT5FgiIpIonZmIiEjClExERCRhSiYiIpIwJRMREUnYAeP0dBYDBgzwESNGhB2GiEhGmTdv3iZ3L9i/vdMmkxEjRjB37tywwxARyShmtqa5dnVziYhIwpRMREQkYUomIiKSMCUTERFJmJKJiIgkTMlEREQSpmQiIiIJ67T3mYgcqrbsqOXVZRtpaNx3RPBgGvbo4/32MWvlecwe+6/bn+23gbvjDk70Z6MHbfusI1gXbSf42djo+6wLVuF4cJy9x409VmPMY4L9zIwsM8wgy6LPo4+NrOAn7H1uMT9jtzP2Po/9aTHHyYrOeL/nedN2TW+jxbxPFvOeNh17z79RC+179t17yGB9c+22zzaGMXxAd3p3zT34P2QbKZmIHEKWb9jGFfe9S+XWmtY3lk7r/i8fzafHDkzqMZVMRA4Rb67YxNUPzqNrbjazrjyOoX27Nbvd/lMYBecCB1kfu85bXNfcvuB7vhnbft/sm77NN30zz2r6Bh1skxWzvulbfdM37j1nA7HHij1jYO/xY2OPPfvZc/YStEWX6C/V9Lyl7WKf791m79lSc9vtObPa533yPY9j25vO3Jre0z3PnGbbfZ/2mH/R/bcJVkwsyt//HyphSiYih4C/vvsR33mijMMLevLHLx9NUZ/mE0lntqe76oBOPkkGJRORDNbY6Nz+4jLu+vtKpo8ewF2XTkl6X7hIPJRMRDLUrroG/uPRUp56by0XHzOMW2dOJDdbF2hKOJRMRDLQlh21XPmnucxb8zE3nT6Or5046oCrqEQ6kpKJSIZZVbWdL9//Lusiu7jrkil8fvKQsEMSUTIRySTvrN7CVX+eS7YZs648jqnD+4YdkgigZCKSMZ5cUMm3Hy1laL9u3H/FMRzWv3vYIYnsoWQikubcnTtfXsEdL73PcaP68dvLppHfXVdsSXpRMhFJY7X1jdz8eBmPza/gnClF/PScyXTJ0RVbkn6UTETSVGRnHV97cC5vrdrCv508hhtOOkJXbEnaSruvOGb2/8xsqZmVmtkTZtYnaB9hZjVmtjBY7onZZ6qZlZnZCjO70/QXJxnuw807OefuN5m/Zit3XFjMN04erUQiaS3tkgnwIjDR3ScD7wM3x6xb6e4lwXJ1TPvdwJXA6GA5rcOiFUmy+R9+zNm/eZPNO2r581eP4eyjhoYdkkir0i6ZuPsL7l4fPH0LOOhfkpkNAXq7+1seHYXuT8AXUhymSEo8U7qOi+99i55dc3j8mk9w7Kj+YYckEpe0Syb7+QrwXMzzkWa2wMxeM7PpQVsRUBGzTUXQdgAzu8rM5prZ3KqqqtRELNIO7s49r63kuofnM7EonyeuPYFRBT3DDkskbqEU4M3sJWBwM6u+6+6zg22+C9QDDwXr1gGHuftmM5sKPGlmE9ryuu5+L3AvwLRp0w4YLFskDHUNjdwyexGz3vmQMycP4bbzi+mamx12WCJtEkoycfeTD7bezK4AzgROCrqucPfdwO7g8TwzWwmMASrZtytsaNAmkvaqd9Vx3UPz+cfyTVz3mcP51iljycpSoV0yT9pdGmxmpwHfBj7l7jtj2guALe7eYGajiBbaV7n7FjOrNrPjgLeBLwK/CiN2kbao3FrDV+57l5VV2/n5uZO54OhhYYck0m5pl0yAXwN5wIvBpZBvBVdunQjcamZ1QCNwtbtvCfa5Frgf6Ea0xvLc/gcVSSdlFRG+8sC77Kpt4P4vH8MnRw8IOySRhKRdMnH3I1pofwx4rIV1c4GJqYxLJFleXLyBG2YtoF+PLjx07bGMGdQr7JBEEpZ2yUTkUHbfm6u59enFTC7K53dfmsbAXl3DDkkkKZRMRDpAQ6Pzw6cXc/8/P+DUCYP4xYVH0a2LrtiSQ4eSiUiK7dhdzw2zFvDy0o1cOX0kN51+JNm6YksOMUomIim0oXoXX7n/XZasq+aHX5jI5ccNDzskkZRQMhFJkSXrqvnK/e9SXVPHH750NJ8ZNzDskERSRslEJAVeXbaR6x6aT6+uufzf1Z9gfGHvsEMSSSklE5Eke+jtNdwyexFjB/Xij1cczeB8XbElhz4lE5EkaWx0fvq3pdz7+io+M7aAX10yhZ55+hOTzkH/00WS5JWlG7n39VVcftxw/uus8eRkp/ug3CLJo//tIkky78OPyc02vnfmkUok0unof7xIkpRXRhgzqBd5OboZUTofJRORJHB3yiojTB6aH3YoIqFQMhFJgoqPa9i6s46JRUom0jkpmYgkQVllBIDJRX1CjkQkHEomIklQWhEhN9sYM1jztkvnpGQikgTllRHGDlbxXTovJRORBDUV3yepi0s6MSUTkQR9tKWGSE0dk1R8l05MyUQkQXuK77osWDqxtEsmZvYDM6s0s4XBckbMupvNbIWZLTOzU2PaTwvaVpjZTeFELp1VaeVWumRnaS536dTSdWyuO9z9ttgGMxsPXARMAAqBl8xsTLD6LuAUoAJ418zmuPvijgxYOq+m4nuXnLT7bibSYTLpf/9M4BF33+3uq4EVwDHBssLdV7l7LfBIsK1Iyrk7ZRURJqmLSzq5dE0m15tZqZn90cz6Bm1FwEcx21QEbS21H8DMrjKzuWY2t6qqKhVxSyfz4ZadVO+qV/FdOr1QkomZvWRm5c0sM4G7gcOBEmAdcHuyXtfd73X3ae4+raCgIFmHlU6sqfiuZCKdXSg1E3c/OZ7tzOx3wNPB00pgWMzqoUEbB2kXSamyioiK7yKkYTeXmQ2JeXo2UB48ngNcZGZ5ZjYSGA28A7wLjDazkWbWhWiRfk5HxiydV1llhHFDVHwXSceruX5uZiWAAx8AXwNw90Vm9ldgMVAPXOfuDQBmdj3wPJAN/NHdF4URuHQuTXe+zyguDDsUkdClXTJx98sPsu7HwI+baX8WeDaVcYnsb83mnWxT8V0ESMNuLpFMsaf4rsuCRZRMRNqrrDJClxwV30VAyUSk3coqIhw5uBe52fozEtFfgUg7NDY65ZW6812kiZKJSDus2bKTbbtVfBdpomQi0g5773zXhFgioGQi0i5lFVvpkpPF6EGa810ElExE2qWsMsKRQ3qr+C4S0F+CSBtFi+/VTFa9RGQPJRORNvpg8w62q/gusg8lE5E20p3vIgdSMhFpo7KKCHk5WYweqOK7SBMlE5E2aiq+56j4LrKH/hpE2qCx0Vm0tprJ6uIS2YeSiUgbrA6K7xNVfBfZh5KJSBuUVUSL7zozEdmXkolIG5RVRuiam8URBSq+i8RSMhFpAxXfRZqnvwiRODU2OosqI7rzXaQZSiYicVq1aQc7ahtUfBdpRtolEzP7i5ktDJYPzGxh0D7CzGpi1t0Ts89UMyszsxVmdqeZWXi/gRyqyiq3AjB5qIadF9lfTtgB7M/dL2x6bGa3A5GY1SvdvaSZ3e4GrgTeBp4FTgOeS2Wc0vmUVVTTNTeLwwt6hB2KSNpJuzOTJsHZxQXArFa2GwL0dve33N2BPwFf6IAQpZMpr4wwXsV3kWal81/FdGCDuy+PaRtpZgvM7DUzmx60FQEVMdtUBG0HMLOrzGyumc2tqqpKTdRySGpodMrXRtTFJdKCULq5zOwlYHAzq77r7rODxxez71nJOuAwd99sZlOBJ81sQlte193vBe4FmDZtmrc9cumsVm/azk4V30VaFEoycfeTD7bezHKAc4CpMfvsBnYHj+eZ2UpgDFAJDI3ZfWjQJpI0pbrzXeSg0rWb62Rgqbvv6b4yswIzyw4ejwJGA6vcfR1QbWbHBXWWLwKzmzuoSHuVVUbolpvN4brzXaRZaXc1V+AiDiy8nwjcamZ1QCNwtbtvCdZdC9wPdCN6FZeu5JKkKq+MML6wN9lZuupcpDlpmUzc/Ypm2h4DHmth+7nAxBSHJZ1UQzDn+4VHDws7FJG0la7dXCJpY1XVdmrqGjTnu8hBKJmItELFd5HWKZmItKKsMkL3LtmMUvFdpEVKJiKtKAvufFfxXaRlSiYiB9HQ6CxeW80kdXGJHFRcycTMvmFmvS3qD2Y238w+l+rgRMK2UsV3kbjEe2byFXevBj4H9AUuB36asqhE0oSK7yLxiTeZNHUWnwH82d0XxbSJHLLKg+L7yAEqvoscTLzJZJ6ZvUA0mTxvZr2I3oUuckgrrdjKxMJ8Fd9FWhFvMvkqcBNwtLvvBLoAX05ZVCJpoL6hkcXrqjVSsEgc4hpOxd0bzWwDMD4Y0VfkkLeyage76hqZNLR32KGIpL24EoOZ/Qy4EFgMNATNDryeorhEQldaEZ3zfVKRJsQSaU28ZxlfAMYGc4qIdArllRF6dMlm1ADN+S7SmnhrJquA3FQGIpJuSisjTCjKJ0vFd5FWxXtmshNYaGYvE8x2CODuN6QkKpGQ1Tc0smRdNZceOzzsUEQyQrzJZE6wiHQKK6q2R4vvupJLJC6tJpNgqtwr3P0zHRCPSFpouvNdY3KJxKfVmom7NwCNZqa/Kuk0yisj9MzLYWR/Fd9F4hFvN9d2oMzMXgR2NDWqZiKHqtKKCBMKe6v4LhKneK/mehz4PtH7SubFLO1mZueb2SIzazSzafutu9nMVpjZMjM7Nab9tKBthZndFNM+0szeDtr/YmZdEontYBobnfWRXak6vKSBuqD4rnqJSPzivQP+gRS8djlwDvDb2EYzGw9cBEwACoGXzGxMsPou4BSgAnjXzOa4+2LgZ8Ad7v6Imd1DdPiXu1MQM999spx/LK/imRumk99NV0sfipZv2M7u+kbVS0TaIN75TFab2ar9l0Re2N2XuPuyZlbNBB5x993uvhpYARwTLCvcfZW71wKPADPNzIDPAo8G+z9A9CbLlDh/2lDWR3Zx02OluHuqXkZCVF4ZFN91ZiISt3i7uaYBRwfLdOBO4MEUxVQEfBTzvCJoa6m9P7DV3ev3az+AmV1lZnPNbG5VVVW7gptyWF/+49SxPFe+ngffWtOuYxxKnlxQyQebdrS+YQYpC4rvI1R8F4lbXMnE3TfHLJXu/gvg863tZ2YvmVl5M8vMhCNvB3e/192nufu0goKCdh/nyumj+PTYAn749BIWrY0kMcLM8sg7H/LNvyzk1qcXhx1KUpVWRphYpOK7SFvE2801JWaZZmZXE0e9xd1PdveJzSyzD7JbJTAs5vnQoK2l9s1An5jRjJvaUyYry7j9/GL69sjl6w8vYPvu+tZ3OsSUVmzlljmLyMvJ4vX3q/h4R23YISWFiu8i7RNvN9ftMctPgCnABSmKaQ5wkZnlmdlIYDTwDvAuMDq4cqsL0SL9HI8WLv4OnBfs/yXgYMkqKfr3zOOXFx3FB5t38L0nyjpV/WTLjlqueXA+BT3z+OMVR1Pf6Dxbvi7ssJLi/Q3bqK1v1BwmIm0U9+RY7v6ZYDnF3a8CEvoqamZnm1kFcDzwjJk9DxBMCfxXosPd/w24zt0bgprI9cDzwBLgr8G2AP8J3GhmK4jWUP6QSGzxOm5Uf75x0hieXLiW/5tX0REvGbqGRueGWQuo2r6buy+bwicO788RA3sye+HasENLiqbi++ShGnZepC3ivWnxUaJnI/u3TW3vC7v7E8ATLaz7MfDjZtqfBZ5tpn0V0au9Otz1nz2Ct1Zt5pbZ5Rw1rA+jB/UKI4wOc/sLy3hjxSZ+du6kPR+4M4oLueOl91m7tYbCPt1CjjAxZZUReuXlMLxf97BDEckoBz0zMbNxZnYukG9m58QsVwBdOyTCNJedZfzyohJ6dMnhuofnU1Pb0PpOGer5Rev5zasrufiYYVx49GF72mcUF+IOT5dm/tlJWUWEiRp2XqTNWuvmGgucCfQBzopZpgBXpja0zDGwd1f+98IS3t+wnVufXtT6DhloVdV2vvXX9ygems8PZkzYZ92IAT0oHprPnPcyO5nU1jeyZP023awo0g4H7eYKrrqabWbHu/u/OiimjPSpMQVc8+nDufvVlRw3qj8zS5q91SUj7dhdz9f+PI8uOVn85rKp5OVkH7DNjJIifvj0YlZWbefwgp4hRJk4Fd9F2i/eAvxmM3vZzMoBzGyymX0vhXFlpBtPGcPU4X35zuNlrD5EbuRzd779WCkrq7bzq4uPoqiFmsiZk4dgBnMyuBC/p/iuZCLSZvEmk98BNwN1AO5eSvTSXImRm53FnRcfRU52Ftc/PJ/d9ZlfP/nDG6t5pnQd/37qWE44YkCL2w3q3ZXjR/VnzntrM/Yy6bLKCL265jC8v4rvIm0VbzLp7u7v7NfW+e7Ui0NRn27cdn4xi9ZW85Nnl4YdTkLeWrWZnzy3lFMnDOKaTx3e6vYzigtZvWkH5ZXVHRBd8pVVRphUlE90uDcRaYt4k8kmMzsccAAzOw84NO5SS4FTxg/iyyeM4P5/fsDzi9aHHU67rI/s4vqH5zO8X3duO784rg/Y0ycOITfbmL0wpQMQpERtfSNL123Tne8i7RRvMrmO6FDx48ysEvgmcHXKojoE3HT6OCYV5fMf//ceFR/vDDucNqmtb+Tah+axs7aB314+lV5d4xtqP797Lp8aM5CnStfS0JhZXV3vb9hGbYOK7yLtFe9Aj6vc/WSgABgHfAr4ZCoDy3R5Odn8+pKjaHT4+qwF1DU0hh1S3H70zGLmf7iV/3decZtvwpxZUsiG6t28s3pLiqJLjbI9d74rmYi0R2s3LfYOZj38tZmdAuwkOvbVClI3NtchY3j/Hvz03Eks+HArt73Q3NQt6efx+RX86V9ruHL6SD4/eUib9z/5yEF075KdcfeclFVG6N01h8N057tIu7R2ZvJnojculhG9SfHvwPnA2e4eyjDymebMyYVccuxh/Pa1Vby6bGPY4RzU4rXVfOeJMo4d2Y//PG1cu47RrUs2nxs/iGfL1lFbnzlnY2UVESYNVfFdpL1aSyaj3P0Kd/8tcDEwHjjV3RemPrRDxy1njmfc4F7c+Nf32FCdnvPHR3bWcfWD8+jTrQu/vmQKOdnxltMONKOkkEhNHf9Y3r4JyDra7voGlq6vVr1EJAGtfWLUNT1w9wagwt3T89MwjXXNjdZPamobuGHWgrQrTjc2Ot/8ywLWRWq469IpFPTKS+h400cX0Ld7bsaMJPz++u3UNbiu5BJJQGvJpNjMqoNlGzC56bGZZebNBCE5YmAvfviFiby9egt3vrw87HD2cecry/n7sipuOXM8U4f3Tfh4udlZnD5pCC8u3sDO2vS/HWlP8b1Iw86LtNdBk4m7Z7t772Dp5e45MY97d1SQh4rzpg7lnClF3PnKcv65clPY4QDw96Ub+eXLyzlnShGXHTc8acedWVxITV0DLy7ekLRjpkpZ5Vbyu+UyrF9mD58vEqb2d4xLu/xw5kRGDujBNx9ZyKbtu0ON5cPNO/nGIws4cnBv/ufsSUktPh89oh9D8rvyVAZc1aU730USp2TSwXrk5XDXJVPYWlPHjX99j8aQ6ic1tQ187cF5mBn3XDaVrrkHjgSciKws46ziQl57v4qtO9N3fvjd9Q0sW79NxXeRBCmZhODIIb255czxvP5+Fb99fVWHv767890nyli6vppfXFTCYSka2HBGcSF1Dc5z5ek7pMyy9dtUfBdJAiWTkFx67GF8ftIQbnthGfPWdOzd4g++tYbHF1TyjZNG85mxA1P2OhMKezOqoEdaj9WlO99FkkPJJCRmxk/OnURhn658/eEFHdYVNG/Nx9z69GI+O24gN3x2dEpfy8yYWVzE26u3sD6SnleUl1VEyO+Wy9C+Kr6LJCKUZGJm55vZIjNrNLNpMe2nmNk8MysLfn42Zt2rZrbMzBYGy8CgPc/M/mJmK8zsbTMb0fG/Ufv07prLry+eQtX23fzHo6Upnwekatturn1oHkPyu3HHBSUdMs/5jJL0nh++rDLCZN35LpKwsM5MyoFzgNf3a98EnOXuk4iOAfbn/dZf6u4lwdI0NslXgY/d/QjgDuBnKYw76YqH9eE/TxvHi4s3cP8/P0jZ69Q3NHL9w/OJ1NRxz2VTye8e30jAiRo5oAeT03R++F11Dby/QcV3kWQIJZm4+xJ3P2DkQ3df4O5NnzqLgG5m1trt2DOBB4LHjwInWYZ9zfzqJ0dy0riB/M+zSyit2JqS1/jpc0t5e/UWfnLOJMYXduwtQjOKCymtiKTdVMYqvoskTzrXTM4F5rt77M0Y9wVdXN+PSRhFwEcA7l4PRID+zR3QzK4ys7lmNreqKn3GjTIzbju/mAE987j+4QVU76prfac2eLp0Lb9/YzVfOn44Zx81NKnHjseZkwvTcn74puK7kolI4lKWTMzsJTMrb2ZpdbRhM5tAtLvqazHNlwbdX9OD5fK2xuTu97r7NHefVlBQ0NbdU6pvjy7cefFRVG6t4TuPlyWtfvL+hm18+9FSpg7vy3c/Pz4px2yrwfldOXZkP2a/V5lW88OXVUTo013Fd5FkSFkycfeT3X1iM8vsg+1nZkOBJ4AvuvvKmONVBj+3AQ8DxwSrKoFhwb45QD6wOfm/UeodPaIfN54yhqdL1/HIux8lfLzqXXVc/ed5dO+Sw28unUKXnPBORGeWFLGqageL1qbPkG66810kedKqm8vM+gDPADe5+5sx7TlmNiB4nAucSbSIDzCHaLEe4DzgFU+nr79tdM2nDmf66AH8YM4ilq5v/wevu/Pvf32PNVt2ctclRzGod9ckRtl2p08cTG62pU0hvqn4ri4ukeQI69Lgs82sAjgeeMbMng9WXQ8cAdyy3yXAecDzZlYKLCR6NvK7YJ8/AP3NbAVwI3BTR/4uyZaVZfzvBSX06prLdQ/Nb/eou3e/tpIXFm/g5tPHceyoZktIHapP9y58akwBT723NrQhZGItXb+N+kYV30WSJayruZ5w96Hunufug9z91KD9R+7eI+by3xJ33+juO9x9qrtPdvcJ7v6NYH4V3H2Xu5/v7ke4+zHu3vHjkyRZQa88fnlRCas27eCW2YvavP8byzdx2/PLOHPyEL76yZEpiLB9ziouZF1kF+9+EP788HuK77rzXSQp0qqbS/Y64YgBfP0zR/DovAoen18R936VW2u44ZEFHDGwJz87d3Ja1QNOGT+IbrnpMT98WcVW+nbPpaiPiu8iyaBkksZuOGk0x4zox/eeLGdl1fZWt99V18A1D86jrr6Rey6bSo+8nA6IMn7du+RwSjA/fF1DuPPDl1VWM2lon7RKtiKZTMkkjeVkZ/HLi0vIy8niunnjcIYAAA9tSURBVIfms6uu4aDb//dTiyitiHD7BcWMKujZQVG2zcySQj7eWccby8ObHGxv8V3zu4kki5JJmhuS343bLyhm6fpt/OiZxS1u95d3P2TWOx9x7acP53MTBndghG0zfXQB+d1yQx1JeMm6ahpUfBdJKiWTDPDZcYO46sRRPPjWhzxTuu6A9aUVW/n+7EVMHz2Ab31ubAgRxq9LThZnTBrCC4s3UFN78DOtVCnfU3zXnO8iyaJkkiH+/XNjKRnWh5seK+XDzTv3tG/ZUcs1D86noGcev7zoKLI7YCTgRM0oLmRnbQMvLQlnfvjSigj9enShMD/ce29EDiVKJhmiS04Wv7r4KDD4+qz51NY30tDo3DBrAVXbd3P3ZVPo16NL2GHG5ZiR/RjcuyuzQxqrS3e+iySfkkkGGdavOz8/dzLvVUT4+d+WcvsLy3hjxSZ+OHMCkzOoyyY7yzhz8hBee38jkZ3JHdSyNbvqGli+cbvqJSJJpmSSYU6fNIQvHj+c37+xmt+8upKLjxnGhUcfFnZYbTazpCiYH/7AGlAqLW4qvutmRZGkUjLJQN8540hKhvVh6vC+/NdZE8IOp10mFvVm5IAeHX4DY7mGnRdJifS6q03i0jU3m8eu+QQGHTL1biqYGTOKC7nzleVsqN7VYQNRllZE6N+jC0NUfBdJKp2ZZKjsLMvYRNJk7/zwHdfVVV4ZYZLmfBdJOiUTCc3hBT2ZWNSbOR10A2NNrYadF0kVJRMJ1YziQt6riPBBB8wPv3hdNY2ueolIKiiZSKjOKg7mh++AQny5hp0XSRklEwnVkPxuHD2iH7MXpn5++NKKCAN6dmFwyLNOihyKlEwkdDNLCllZtYPF61I7P3y57nwXSRklEwndGROHkJOV2vnhd9bWs3yjiu8iqaJkIqHr26MLJ44p4KmFqZsffklT8T2Dhp0RySShJBMzO9/MFplZo5lNi2kfYWY1ZrYwWO6JWTfVzMrMbIWZ3WlBX4WZ9TOzF81sefCzbxi/kyRmRnEhayO7mPfhxyk5flmF7nwXSaWwzkzKgXOA15tZt9LdS4Ll6pj2u4ErgdHBclrQfhPwsruPBl4OnkuGOWX8ILrmZqVs0qzSyggDeuYxqHdeSo4v0tmFkkzcfYm7L4t3ezMbAvR297c8esnPn4AvBKtnAg8Ejx+IaZcM0iMvh5OPHMSzZetTMj98eWWEybrzXSRl0rFmMtLMFpjZa2Y2PWgrAipitqkI2gAGuXvTeBzrgUEdFKck2cySIrbsqOWNFcmdH35nbT0rNm5norq4RFImZQM9mtlLQHOTkX/X3We3sNs64DB332xmU4EnzSzuYXHd3c2sxQqumV0FXAVw2GGZN2z7oe7EMQPo3TWHpxau5TNjBybtuIvXRovvk5VMRFImZcnE3U9uxz67gd3B43lmthIYA1QCQ2M2HRq0AWwwsyHuvi7oDtt4kOPfC9wLMG3atNTeISdtlpeTzRmThvDUe2upqW2gW5fspBy3tEJ3voukWlp1c5lZgZllB49HES20rwq6sarN7LjgKq4vAk1nN3OALwWPvxTTLhloRnEhO2obeGVpi98J2qy8MkJBr7wOG+ZepDMK69Lgs82sAjgeeMbMng9WnQiUmtlC4FHganffEqy7Fvg9sAJYCTwXtP8UOMXMlgMnB88lQx07qj8De+Ul9aqussqIurhEUiyUybHc/QngiWbaHwMea2GfucDEZto3AyclO0YJR3R++EIefGsNkZo68rvlJnS8HbvrWVG1nTMmDUlShCLSnLTq5hKB6FhdtQ2NPF++PuFjLV5XjTtMVr1EJKWUTCTtTB6az/D+3ZMyVlep7nwX6RBKJpJ2zIyZxYX8c+UmNm7bldCxyisjDOyVx0AV30VSSslE0tKMkkIaHZ5JcH74suDOdxFJLSUTSUtHDOzF+CG9mb2w/V1d23fXs7JKd76LdAQlE0lbM0oKWfjRVj7cvLNd+y9eq+K7SEdRMpG0dVZxIQBz3mvfPSelFVsBdGYi0gGUTCRtFfXpxtEj+jJ74dp2zQ9fXhlhUO88BvZS8V0k1ZRMJK3NKCli+cbtLF2/rc37llVGmFSkmRVFOoKSiaS1MyYOJrsd88Nv313Pqk07dH+JSAdRMpG01r9nHtNHD2BOG7u6FlVGVHwX6UBKJpL2ZhQXUrm1hvltmB++rDJ657uK7yIdQ8lE0t7nJgwmLyerTfeclFVGGNy7KwW9NOe7SEdQMpG01zOYH/6Z0nXUxzk/fFllRJNhiXQgJRPJCDNKCtm8o5Y3V25uddttu+pYVaXiu0hHUjKRjPDpsQX06prDnDi6uhatrQY0Ta9IR1IykYyQl5PN6RMH8/yi9eyqazjotmUadl6kwymZSMaYUVzE9t31/L2V+eHLKiMMye/KgJ4qvot0FCUTyRjHH96fAT3zWr2qK3rnu85KRDqSkolkjOj88EN4ZdlGqnfVNbtN9a46VuvOd5EOF0oyMbPzzWyRmTWa2bSY9kvNbGHM0mhmJcG6V81sWcy6gUF7npn9xcxWmNnbZjYijN9JOsbMkkJq61ueH35RpYrvImEI68ykHDgHeD220d0fcvcSdy8BLgdWu/vCmE0ubVrv7k0d518FPnb3I4A7gJ91QPwSkpJhfTisX8vzw5dVRoed15mJSMcKJZm4+xJ3X9bKZhcDj8RxuJnAA8HjR4GTzMwSiU/Sl5kxo7iQN1dsomrb7gPWl1VWU5jflf4qvot0qHSumVwIzNqv7b6gi+v7MQmjCPgIwN3rgQjQv7kDmtlVZjbXzOZWVVWlKm5Jsb3zwx94dlJWsVVdXCIhSFkyMbOXzKy8mWVmHPseC+x09/KY5kvdfRIwPVgub2tM7n6vu09z92kFBQVt3V3SxJhBvRg3uNcBXV3Vu+r4YPNOdXGJhCAnVQd295MT2P0i9jsrcffK4Oc2M3sYOAb4E1AJDAMqzCwHyAdaH3NDMtqMkkJ+/rdlfLRlJ8P6dQeiMysCTBqqCbFEOlradXOZWRZwATH1EjPLMbMBweNc4EyiRXyAOcCXgsfnAa94e+Z4lYxy1uSm+eH3np3ozneR8IR1afDZZlYBHA88Y2bPx6w+EfjI3VfFtOUBz5tZKbCQ6NnI74J1fwD6m9kK4EbgppT/AhK6Yf26M3V4333G6iqrjFDUpxv9enQJMTKRzill3VwH4+5PAE+0sO5V4Lj92nYAU1vYfhdwfpJDlAwws6SQW2YvYun6asYN7q0730VClHbdXCLxOmPSkOj88AvXEqmpY83mnbqSSyQkSiaSsQb0zOOEIwYw5721e4vvOjMRCYWSiWS0GcWFVHxcw5/+9QGgZCISFiUTyWinThhEl5wsnl+0gaI+3eir4rtIKJRMJKP16prLSeMGAjBZ9RKR0CiZSMabWRK952SiurhEQqNkIhnvs+MG8bUTR3H2UUVhhyLSaYVyn4lIMnXJyeLmM44MOwyRTk1nJiIikjAlExERSZiSiYiIJEzJREREEqZkIiIiCVMyERGRhCmZiIhIwpRMREQkYdZZZ7g1sypgTTt3HwBsSmI4mU7vx156L/al92Nfh8L7MdzdC/Zv7LTJJBFmNtfdp4UdR7rQ+7GX3ot96f3Y16H8fqibS0REEqZkIiIiCVMyaZ97ww4gzej92Evvxb70fuzrkH0/VDMREZGE6cxEREQSpmQiIiIJUzJpIzM7zcyWmdkKM7sp7HjCYmbDzOzvZrbYzBaZ2TfCjikdmFm2mS0ws6fDjiVsZtbHzB41s6VmtsTMjg87prCY2b8FfyflZjbLzLqGHVOyKZm0gZllA3cBpwPjgYvNbHy4UYWmHviWu48HjgOu68TvRaxvAEvCDiJN/BL4m7uPA4rppO+LmRUBNwDT3H0ikA1cFG5Uyadk0jbHACvcfZW71wKPADNDjikU7r7O3ecHj7cR/aDo1JOwm9lQ4PPA78OOJWxmlg+cCPwBwN1r3X1ruFGFKgfoZmY5QHdgbcjxJJ2SSdsUAR/FPK+gk3+AApjZCOAo4O1wIwndL4BvA41hB5IGRgJVwH1Bt9/vzaxH2EGFwd0rgduAD4F1QMTdXwg3quRTMpGEmFlP4DHgm+5eHXY8YTGzM4GN7j4v7FjSRA4wBbjb3Y8CdgCdssZoZn2J9mCMBAqBHmZ2WbhRJZ+SSdtUAsNing8N2jolM8slmkgecvfHw44nZCcAM8zsA6Ldn581swfDDSlUFUCFuzedrT5KNLl0RicDq929yt3rgMeBT4QcU9IpmbTNu8BoMxtpZl2IFtHmhBxTKMzMiPaHL3H3/w07nrC5+83uPtTdRxD9f/GKux9y3z7j5e7rgY/MbGzQdBKwOMSQwvQhcJyZdQ/+bk7iELwYISfsADKJu9eb2fXA80SvyPijuy8KOaywnABcDpSZ2cKg7Tvu/myIMUl6+TrwUPDFaxXw5ZDjCYW7v21mjwLziV4FuYBDcFgVDaciIiIJUzeXiIgkTMlEREQSpmQiIiIJUzIREZGEKZmIiEjClExEEmBmDWa2MGY56F3eZna1mX0xCa/7gZkNSPQ4IsmiS4NFEmBm2929Zwiv+wHRUWg3dfRrizRHZyYiKRCcOfzczMrM7B0zOyJo/4GZ/Xvw+IZgPphSM3skaOtnZk8GbW+Z2eSgvb+ZvRDMifF7wGJe67LgNRaa2W+DOVWyzez+YP6MMjP7txDeBulElExEEtNtv26uC2PWRdx9EvBroiMK7+8m4Ch3nwxcHbT9N7AgaPsO8Keg/b+AN9x9AvAEcBiAmR0JXAic4O4lQANwKVACFLn7xCCG+5L4O4scQMOpiCSmJvgQb86smJ93NLO+lOhwI08CTwZtnwTOBXD3V4Izkt5E5wY5J2h/xsw+DrY/CZgKvBsd9oluwEbgKWCUmf0KeAY45IY8l/SiMxOR1PEWHjf5PNGZO6cQTQbt+XJnwAPuXhIsY939B+7+MdHZDV8letbT6SfsktRSMhFJnQtjfv4rdoWZZQHD3P3vwH8C+UBP4B9Eu6kws08Dm4J5Yl4HLgnaTwf6Bod6GTjPzAYG6/qZ2fDgSq8sd38M+B6dd/h36SDq5hJJTLeYUZMhOud50+XBfc2sFNgNXLzfftnAg8H0tgbc6e5bzewHwB+D/XYCXwq2/29glpktAv5JdFhz3H2xmX0PeCFIUHXAdUAN0VkOm74w3py8X1nkQLo0WCQFdOmudDbq5hIRkYTpzERERBKmMxMREUmYkomIiCRMyURERBKmZCIiIglTMhERkYT9//QR9OysykgbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "buffer_size = 100000\n",
    "n_sequence = 50\n",
    "elite_ratio = 0.2\n",
    "plan_horizon = 25\n",
    "num_episodes = 10\n",
    "env_name = 'Pendulum-v0'\n",
    "env = gym.make(env_name)\n",
    "\n",
    "replay_buffer = ReplayBuffer(buffer_size)\n",
    "pets = PETS(env, replay_buffer, n_sequence, elite_ratio, plan_horizon,\n",
    "            num_episodes)\n",
    "return_list = pets.train()\n",
    "\n",
    "episodes_list = list(range(len(return_list)))\n",
    "plt.plot(episodes_list, return_list)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('PETS on {}'.format(env_name))\n",
    "plt.show()\n",
    "\n",
    "# episode: 1, return: -1062\n",
    "# episode: 2, return: -1257\n",
    "# episode: 3, return: -1792\n",
    "# episode: 4, return: -1225\n",
    "# episode: 5, return: -248\n",
    "# episode: 6, return: -124\n",
    "# episode: 7, return: -249\n",
    "# episode: 8, return: -269\n",
    "# episode: 9, return: -245\n",
    "# episode: 10, return: -119"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第16章-模型预测控制.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}