pattern_finding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "\n",
    "import sys\n",
    "sys.path.append('./code')\n",
    "from parsed_args import args\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as Data\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import Dataset\n",
    "import math\n",
    "from tqdm import tqdm\n",
    "import warnings \n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score, roc_auc_score\n",
    "\n",
    "from vae_train.vae_utils import *\n",
    "from code.dataset_iterators import *\n",
    "\n",
    "torch.manual_seed(0)\n",
    "np.random.seed(0)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from vae_train.vae_model import HierachyVAE\n",
    "from lstm_train.lstm_model import AttnContentStrategy\n",
    "from read_data import *\n",
    "from utils import *\n",
    "from collections import defaultdict\n",
    "from editing_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First, we'll check the labeled data distribution!\n",
    "### Table 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.data_path + 'labeled_data.pkl', 'rb') as f:\n",
    "    labeled_data_pkl = pickle.load(f)\n",
    "with open(args.data_path + 'unlabeled_data.pkl', 'rb') as f:\n",
    "    unlabeled_data_pkl = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49855"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unlabeled_data_pkl) + len(labeled_data_pkl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The labeled data pkl is structured as follows:\n",
    "```\n",
    "[[sentences, labels], [sentences, labels], ...]\n",
    "```\n",
    "Here, we extract the labels and count how frequently they appear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_label_distribution(labeled_data_pkl):\n",
    "    sentence_labels = [labeled_data_pkl[k][1] for k in labeled_data_pkl]\n",
    "    freq = defaultdict(int)\n",
    "    for sent in sentence_labels:\n",
    "        for strategy in sent:\n",
    "            freq[strategy] += 1\n",
    "            \n",
    "    tot = sum(freq[strategy] for strategy in freq)\n",
    "    for strategy in freq:\n",
    "        freq[strategy] = freq[strategy] / tot\n",
    "    return freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {3: 0.38982758620689656,\n",
       "             2: 0.17517241379310344,\n",
       "             0: 0.05672413793103448,\n",
       "             1: 0.07879310344827586,\n",
       "             8: 0.11724137931034483,\n",
       "             9: 0.16,\n",
       "             6: 0.0017241379310344827,\n",
       "             5: 0.010344827586206896,\n",
       "             7: 0.01,\n",
       "             4: 0.00017241379310344826})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_label_distribution(labeled_data_pkl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We actually started with a larger set of labels than originally presented in the paper;\n",
    "however, these labels were eventually merged to the subset that appears in our work. This bit does a little merging."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Cred': 0.07879310344827586, 'Recip': 0.17517241379310344, 'Concret': 0.38982758620689656, 'Impac': 0.11724137931034483, 'Polit': 0.16, 'other': 0.0789655172413793}\n"
     ]
    }
   ],
   "source": [
    "# these are all our proposed strategies\n",
    "all_strategies = ['Other', 'Cred', 'Recip', 'Concret', 'Commit', 'Scar', 'Identit', 'Emo', 'Impac', 'Polit']\n",
    "\n",
    "with open(args.data_path + 'label_mapping.pkl', 'rb') as f:\n",
    "    label_mapping = pickle.load(f)\n",
    "\n",
    "def merge_labels(label_mapping, all_strats, freq):\n",
    "    # build an inverse mapping from the label_mapping pkl\n",
    "    inverse_label_mapping = {}\n",
    "    for key in label_mapping:\n",
    "        value = label_mapping[key] \n",
    "        if value in inverse_label_mapping:\n",
    "            inverse_label_mapping[value] = 0\n",
    "        else:\n",
    "            inverse_label_mapping[value] = key\n",
    "            \n",
    "    # select the subset that we use from all_strategies \n",
    "    selected = [1, 2, 3, 8, 9]\n",
    "    merged = {}\n",
    "    for i in selected: merged[all_strats[i]] = 0\n",
    "    merged[\"other\"] = 0\n",
    "\n",
    "    for k in freq:\n",
    "        if k in selected:\n",
    "            merged[all_strats[k]] += freq[k]\n",
    "        else:\n",
    "            merged[\"other\"] += freq[k]\n",
    "    \n",
    "    return merged, inverse_label_mapping\n",
    "\n",
    "merged, inverse_label_mapping = merge_labels(label_mapping, all_strategies, get_label_distribution(labeled_data_pkl))\n",
    "print(merged)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now, we'll extract the content and strategy vectors from our VAE model.\n",
    "#### Section 2, Modeling Part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpu num:  1\n"
     ]
    }
   ],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = \"6\"\n",
    "use_cuda = torch.cuda.is_available()\n",
    "devices = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "print(\"gpu num: \", n_gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/borrow/\n"
     ]
    }
   ],
   "source": [
    "print(args.data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 0, 1: 1, 2: 2, 3: 3, 4: 0, 5: 0, 6: 0, 7: 0, 8: 4, 9: 5}\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "#Labeled: 900, Unlabeled 48155, Val 400, Test 400, N class 6, 2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DataParallel(\n",
       "  (module): HierachyVAE(\n",
       "    (embedding): Embedding(13944, 128)\n",
       "    (encoder): Encoder(\n",
       "      (lstm): LSTM(128, 128, batch_first=True)\n",
       "    )\n",
       "    (hidden_to_mu): Linear(in_features=70, out_features=64, bias=True)\n",
       "    (hidden_to_logvar): Linear(in_features=70, out_features=64, bias=True)\n",
       "    (hidden_linear): Linear(in_features=128, out_features=64, bias=True)\n",
       "    (classifier): Linear(in_features=128, out_features=6, bias=True)\n",
       "    (predictor): Predictor(\n",
       "      (embedding): Linear(in_features=6, out_features=64, bias=False)\n",
       "      (lstm): LSTM(70, 64, batch_first=True)\n",
       "      (predict): Linear(in_features=64, out_features=2, bias=True)\n",
       "    )\n",
       "    (generator): Generator(\n",
       "      (lstm): LSTM(198, 128, batch_first=True)\n",
       "      (linear): Linear(in_features=128, out_features=13944, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labeled_dataset, train_unlabeled_dataset, test_dataset, val_dataset, vocab, n_labels, doc_labels = read_data(\n",
    "    data_path=args.data_path, n_labeled_data=args.n_labeled_data, n_unlabeled_data=args.n_unlabeled_data, max_seq_num=args.max_seq_num, max_seq_len=args.max_seq_len, embedding_size=args.embedding_size)\n",
    "\n",
    "dist = train_labeled_dataset.esit_dist\n",
    "\n",
    "labeled_trainloader = Data.DataLoader(\n",
    "    dataset=train_labeled_dataset, batch_size=args.batch_size, shuffle=True)\n",
    "unlabeled_trainloader = Data.DataLoader(\n",
    "    dataset=train_unlabeled_dataset, batch_size=args.batch_size_u, shuffle=True)\n",
    "val_loader = Data.DataLoader(\n",
    "    dataset=val_dataset, batch_size=16, shuffle=False)\n",
    "test_loader = Data.DataLoader(\n",
    "    dataset=test_dataset, batch_size=16, shuffle=False)\n",
    "\n",
    "model = HierachyVAE(vocab.vocab_size, args.embedding_size, args.n_highway_layers, args.encoder_hidden_size, args.encoder_layers, args.generator_hidden_size, args.generator_layers, args.z_size, n_labels, doc_labels, args.bidirectional, vocab.embed, args.hard).cuda()\n",
    "model = nn.DataParallel(model).cuda()\n",
    "model.load_state_dict(torch.load(args.data_path + 'model_state.pkl'))\n",
    "model.eval()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 120/120 [00:52<00:00,  2.30it/s]\n"
     ]
    }
   ],
   "source": [
    "labeled_train_iter = iter(labeled_trainloader)\n",
    "unlabeled_train_iter = iter(unlabeled_trainloader)\n",
    "tau = args.tau\n",
    "\n",
    "content_vectors = []\n",
    "strat_vectors = []\n",
    "doc_labels = []\n",
    "mappings = []\n",
    "orig_fetch = []\n",
    "for batch_idx in tqdm(range(args.val_iteration), position=0, leave=True):\n",
    "    try:\n",
    "        x, l, y, mask1, mask2, mask3, mask4, mid, sent_len, doc_len = labeled_train_iter.next()\n",
    "\n",
    "    except:\n",
    "        labeled_train_iter = iter(labeled_trainloader)\n",
    "        x, l, y, mask1, mask2, mask3, mask4, mid, sent_len, doc_len = labeled_train_iter.next()\n",
    "\n",
    "    try:\n",
    "        x_u, l_u, y_u, mask1_u, mask2_u, mask3_u, mask4_u, mid_u, sent_len_u, doc_len_u = unlabeled_train_iter.next()\n",
    "\n",
    "    except:\n",
    "        unlabeled_train_iter = iter(unlabeled_trainloader)\n",
    "        x_u, l_u, y_u, mask1_u, mask2_u, mask3_u, mask4_u, mid_u, sent_len_u = unlabeled_train_iter.next()\n",
    "\n",
    "    x = torch.cat([x, x_u], dim = 0)\n",
    "    l = torch.cat([l, l_u], dim = 0)\n",
    "    y = torch.cat([y.long(), y_u.long()], dim = 0)\n",
    "\n",
    "    mask1 = torch.cat([mask1, mask1_u], dim = 0)\n",
    "    mask2 = torch.cat([mask2, mask2_u], dim = 0)\n",
    "    mask3 = torch.cat([mask3, mask3_u], dim = 0)\n",
    "    mask4 = torch.cat([mask4, mask4_u], dim = 0)\n",
    "\n",
    "    doc_len = torch.cat([doc_len, doc_len_u], dim = 0)\n",
    "    sent_len = torch.cat([sent_len, sent_len_u], dim = 0)\n",
    "\n",
    "    batch_size = l.shape[0]\n",
    "    seq_num = x.shape[1]\n",
    "    seq_len = x.shape[2]\n",
    "\n",
    "    mid = mid + mid_u\n",
    "\n",
    "    temp = l.view(-1, 1).long()\n",
    "    l_one_hot = torch.zeros(batch_size*seq_num, n_labels).cuda()\n",
    "\n",
    "    for i in range(0, len(temp)):\n",
    "        if temp[i] != 10:\n",
    "            l_one_hot[i][temp[i]] = 1\n",
    "\n",
    "    l_one_hot = l_one_hot.view(batch_size, seq_num, n_labels)\n",
    "\n",
    "    if batch_idx % 30 == 1:\n",
    "        tau = np.maximum(tau * np.exp(-args.anneal_rate*batch_idx), args.tau_min)\n",
    "\n",
    "    xs, ys = (x.view(batch_size*seq_num, seq_len), l.view(batch_size*seq_num))\n",
    "    prob = create_generator_inputs(xs, vocab, train = False)\n",
    "\n",
    "\n",
    "    x, prob, l_one_hot, y, l = x.cuda(), prob.cuda(), l_one_hot.cuda(), y.cuda(), l.cuda()\n",
    "    mask1, mask2 = mask1.cuda(), mask2.cuda()\n",
    "\n",
    "    logits, kld_z, q_y, q_y_softmax, t, strategy_embedding, y_in2, content_vec = model(x, prob, args.tau, \n",
    "        mask1, mask2, args.hard, l_one_hot, doc_len = doc_len, sent_len = sent_len)\n",
    "    \n",
    "    max_idxs = y_in2.argmax(axis=1)\n",
    "    argmaxed = torch.zeros(y_in2.shape)\n",
    "    argmaxed[torch.arange(y_in2.shape[0]),max_idxs] = 1\n",
    "    y_in2 = (argmaxed.T.cpu() * y_in2.sum(axis=1).cpu()).T\n",
    "    \n",
    "    last_dim = int((content_vec.shape[0] * content_vec.shape[1]) / (batch_size * seq_num))\n",
    "    content_vectors.append(content_vec.reshape((batch_size, seq_num, last_dim)).tolist())\n",
    "    strat_vectors.append(y_in2.reshape(batch_size, seq_num, n_labels).tolist())\n",
    "    doc_labels.append(y.tolist())\n",
    "    orig_fetch.append(mid)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Here is where our LSTM Model would also be trained (on content and strat vec tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttnContentStrategy(\n",
       "  (linearStrategy): Linear(in_features=6, out_features=64, bias=True)\n",
       "  (linearStrategy2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (linearContent): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (linearContent2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (content_proj): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (strat_proj): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (linear1): Linear(in_features=128, out_features=256, bias=True)\n",
       "  (linear2): Linear(in_features=256, out_features=512, bias=True)\n",
       "  (lstm): LSTM(512, 512)\n",
       "  (s_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "  (softmax): Softmax(dim=1)\n",
       "  (sent_linear1): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (sent_linear2): Linear(in_features=256, out_features=128, bias=True)\n",
       "  (sent_linear3): Linear(in_features=128, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_model = AttnContentStrategy(n_labels)\n",
    "test_model.cuda()\n",
    "test_model.load_state_dict(torch.load(\"best_borrow_model.pkl\"))\n",
    "test_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25it [00:02,  9.64it/s]\n",
      "2it [00:00, 10.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6136089923725412 f1\n",
      "0.6141304347826086 p\n",
      "0.6135817307692308 r\n",
      "0.615 acc\n",
      "0.6135817307692308 roc auc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25it [00:02, 10.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.608345258718084 f1\n",
      "0.6086956521739131 p\n",
      "0.6082706766917294 r\n",
      "0.61 acc\n",
      "0.6082706766917294 roc auc\n"
     ]
    }
   ],
   "source": [
    "test_details = get_dataloader_details(test_loader, model, \n",
    "                                       test_model, n_labels, vocab)\n",
    "\n",
    "val_details = get_dataloader_details(val_loader, model, \n",
    "                                       test_model, n_labels, vocab)\n",
    "\n",
    "train_details = get_content_strat_vector_details(content_vectors, strat_vectors, \n",
    "                                        doc_labels, orig_fetch, test_model)\n",
    "\n",
    "# detailers return the following:\n",
    "# f1_score(all_corr, all_out, average=\"macro\"), attns, labels, strategy_orders, all_mids\n",
    "# acc, attn_content_arr, attn_strat_arr, s_score_arr, attn_aud_arr, attn_msg_arr, labels, strategy_orders, orig_fetch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now, we'll look at correlations between success rate and strategy attention\n",
    "#### Section 3, Content-Strategy interplay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# datasets is an array of detailers.\n",
    "def get_strategy_attn_freqs(datasets):\n",
    "    freq_all = defaultdict(int)\n",
    "    freq_success = defaultdict(int)\n",
    "    freq_fail = defaultdict(int)\n",
    "    strat_attns = {}\n",
    "\n",
    "    all_attn_vals = []\n",
    "    \n",
    "    for dataset in datasets:\n",
    "        for i, batch_content in enumerate(dataset[-3]):\n",
    "            for j, label in enumerate(batch_content):\n",
    "                attns = dataset[1]\n",
    "                doc_level_attn = attns[\"document\"][i][j].squeeze()\n",
    "                strategy_level_attn = attns[\"strategy\"][i][j].squeeze()\n",
    "                strategy_orders = dataset[-2][i][j]\n",
    "                \n",
    "                # strat arg will contain the relevant ordering\n",
    "                strat_arg = []\n",
    "                \n",
    "                # sum attn is just for the stat in the paper that shows the model focusing on a single triplet.\n",
    "                sum_attn = 0\n",
    "                # extract the most important sentence to find the \"middle\" of the triplet\n",
    "                important_sent = (doc_level_attn.argmax())\n",
    "                all_attn_vals.append(doc_level_attn[important_sent].tolist())\n",
    "                \n",
    "                # handle the left of the triplet\n",
    "                if important_sent == 0:\n",
    "                    strat_arg.append(\"UNK\")\n",
    "                else:\n",
    "                    strat_arg.append(strategy_orders[important_sent - 1].argmax().tolist())\n",
    "                    sum_attn += strategy_level_attn[important_sent - 1].tolist()\n",
    "\n",
    "                # handle the middle triplet        \n",
    "                strat_arg.append(strategy_orders[important_sent].argmax().tolist())\n",
    "                \n",
    "                # are we exceeding the sentence length? sum(strategy_orders) == 0 will tells us that.\n",
    "                if important_sent >= strategy_orders.shape[0] - 1 or sum(strategy_orders[important_sent + 1]) == 0:\n",
    "                    strat_arg.append(\"UNK\")\n",
    "                else:\n",
    "                    strat_arg.append(strategy_orders[important_sent + 1].argmax().tolist())\n",
    "                    sum_attn += strategy_level_attn[important_sent + 1].tolist()\n",
    "                \n",
    "                curr_attn = strategy_level_attn[important_sent].tolist()\n",
    "                sum_attn += curr_attn\n",
    "\n",
    "                strat_key = \" \".join([str(x) for x in strat_arg])\n",
    "        \n",
    "                if strat_key not in strat_attns:\n",
    "                    strat_attns[strat_key] = {\n",
    "                        \"triplet_strategy_attn\": 0,\n",
    "                        \"freq\": 0,\n",
    "                        \"label\": []\n",
    "                    }\n",
    "                \n",
    "                strat_attns[strat_key][\"triplet_strategy_attn\"] += (sum_attn / 3) #curr_attn\n",
    "                strat_attns[strat_key][\"freq\"] += 1\n",
    "                strat_attns[strat_key][\"label\"].append(dataset[-1][i][j])\n",
    "        \n",
    "                if label: freq_success[strat_key] += 1                    \n",
    "                else: freq_fail[strat_key] += 1\n",
    "                \n",
    "                freq_all[strat_key] += 1  \n",
    "    \n",
    "    return freq_all, freq_success, freq_fail, strat_attns, all_attn_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_all, freq_success, freq_fail, strat_attns, attn_vals = get_strategy_attn_freqs([train_details, val_details, test_details])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "fixed_attns = {strat: (strat_attns[strat][\"triplet_strategy_attn\"] / strat_attns[strat][\"freq\"]) for strat in strat_attns}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all k values here are \"strategies\"\n",
    "sorted_success = {k: v for k, v in sorted(freq_success.items(), key=lambda item: item[1], reverse=True)}\n",
    "sorted_fail = {k: v for k, v in sorted(freq_fail.items(), key=lambda item: item[1], reverse=True)}\n",
    "sorted_strat_attn = {k: v for k, v in sorted(fixed_attns.items(), key=lambda item: item[1], reverse=True)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "throw_out = 0\n",
    "success_rate = {}\n",
    "for k in freq_all:\n",
    "    v = freq_all[k]\n",
    "    if v >= 60 and '0' not in k:\n",
    "        success_rate[k] = freq_success[k] / freq_all[k]\n",
    "    else:\n",
    "        throw_out += v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.30839338702840186"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "throw_out / sum([freq_all[k] for k in freq_all])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_success_rate = {k: v for k, v in sorted(success_rate.items(), key=lambda item: item[1], reverse=True)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_strats = {}\n",
    "for k in sorted_success_rate:\n",
    "    merged_strats[k] = [sorted_success_rate[k], sorted_strat_attn[k], strat_attns[k][\"label\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "rate_success_line = []\n",
    "attn_success_line = []\n",
    "for i in merged_strats:\n",
    "    rate_success_line.append(merged_strats[i][0])\n",
    "    attn_success_line.append(merged_strats[i][1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f359c9b8710>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xU9Z3/8dcn95AQCORCSMJFCMhFAYl4v4KKttW6Vbxtf3Zrl+6qW7t2u71s17ruY7vdXrb3dmtbd2vrjaqtqCCCVdEqSlBUCIQ7SQi5QALkPsnM9/fHGSTEQCbJhMlM3s/HYx4z58zJnM9xzDuH7/l+v8ecc4iISPSLi3QBIiISHgp0EZEYoUAXEYkRCnQRkRihQBcRiREJkdpxVlaWmzRpUqR2LyISlTZs2HDAOZfd03sRC/RJkyZRUlISqd2LiEQlM9t7ovfU5CIiEiMU6CIiMUKBLiISIxToIiIxQoEuIhIjFOgiIjFCgS4iEiMU6CIip0ogAKv+Bep3DcrHK9BFRE6Vt38Jb/4U9rw+KB+vQBcRORXqymDN/TDtapj36UHZRUiBbmaLzazMzHaY2Vd7eH+Cmb1sZu+a2ftmdk34SxURiVL+Dnh6KSSlwbU/BrNB2U2vgW5m8cDPgKuBmcAtZjaz22bfAJY55+YBNwM/D3ehIiJRa+13Yf9G+PgPIT1n0HYTyhn6AmCHc26Xc84HPA5c120bB2QEX48CqsJXoohIFKvcAGu/B3NugZnXDuquQpltMR+o6LJcCZzTbZv7gRfN7B+ANGBRWKoTEYlmvhb441IYmQdX/9eg7y6UM/SeGntct+VbgP9zzhUA1wC/M7OPfLaZLTWzEjMrqaur63u1IiLRZM034eAO+OTPIWXUoO8ulECvBAq7LBfw0SaVO4BlAM65N4EUIKv7BznnHnTOFTvnirOze5yfXUQkNuz8M7z9IJzz93DaJadkl6EE+nqgyMwmm1kS3kXP5d22KQcWApjZDLxA1ym4iAxPrQ3wp7sgaxos+uYp222vge6c6wTuBlYBW/B6s2w2swfM7GgL/5eAvzWz94DHgM8457o3y4iIDA8rvgzNtXD9LyEx9ZTtNqRb0DnnVgAruq27r8vrUuCC8JYmIhKFNj0NH/wBLv065J91SnetkaIiIuHSWA3P3wvjz4KL7j3lu1egi4iEg3PwzN3Q0QZ/9SDEJ57yEkJqchERkV5s+F/YsRqu/i5kFUWkBJ2hi4gMVF2ZNy3uaZfC2Z+LWBkKdBGRgWg+CI8u8Sbeuu7nEBe5WFWTi4hIf3W2wxO3wZH98JnnYVR+RMtRoIuI9Idz8Ow9UP4m3PAQFJ4d6YrU5CIi0i+vfR/ee8zrbz77U5GuBlCgi4j03eY/wZ//Hc5YApf8c6Sr+ZACXUSkL/ZtgD9+HgrPgWt/Mmh3H+oPBbqISKgOV8Jjt3h3HbrpEUhMiXRFx9FFURGRULQ3waM3Q0cr/L9nIH3oTQGuQBcR6U3AD0/dAbWlcNsyyJkR6Yp6pEAXEenN6vtg2wtwzfdg6tC9w6ba0EVETqbkf+HNn8KCz8OCv410NSelM3QRke5a6mH3q7DzZXj39zD1CrjqW5GuqlcKdBGRznYoXwe7XvZCfP97gIPkDJh1PXz8BxA/9ONy6FcoIhJuzkHNJtj1ihfge9+AzlaIS4CCBXDp12DKZd6NKqIgyI+KnkpFRMLBOXj2C/DOw95y1nSYfzucdhlMugCSR0a2vgFQoIvI8PKXH3phfu6dcN7dEZ8hMZwU6CIyfJQuhzX3e5NpXfWtITVsPxzUbVFEhoeqd+HppVBwtncjihgLc1Cgi8hwcHifN2w/LRtufnTIzcESLgp0EYlt7U3w2E3ga4Zbn/Am1opRakMXkdgV8HvNLDWb4dZlkDsz0hUNKgW6iMSuNfdD2fNw9Xeg6IpIVzPo1OQiIrFpw2/hjR/D2Z+DBUsjXc0poUAXkdizey08fy9MuRwW/1dM9mjpiQJdRGLLgR3wxKdh7FS48f+iauj+QA2fIxWR2OYcNNXCozdCXLzXoyVlVKSrOqUU6CIy9NVuhYq3oLUBWuu96W1bG7xHS723rrUB/D6IT4Lbn4XMSZGu+pRToIvI0OYcPHIDHK7wluOTIHUMjBgDqZkwdgqkFgeXx8CkC6GgOLI1R4gCXUSGtur3vTBf/F9w1qchccSwucjZVwp0ERnaylYC5k2olZQW6WqGNPVyEZGhrWwlFC6A9OxIVzLkKdBFZOg6vA/2b4TpV0e6kqigQBeRoWvbC97z9GsiW0eUUKCLyNBVthIyJ0PWtEhXEhUU6CIyNLU3we5XvbNz9WoJiQJdRIamXS97A4XUfh6ykALdzBabWZmZ7TCzr55gmyVmVmpmm83s0fCWKSLDTtlKSBkNE86NdCVRo9d+6GYWD/wMuAKoBNab2XLnXGmXbYqArwEXOOcazCx2bwkiIoMv4PcuiBZdCfGJka4maoRyhr4A2OGc2+Wc8wGPA9d12+ZvgZ855xoAnHO14S1TRIaVyvXQclDNLX0USqDnAxVdliuD67qaBkwzs7+Y2TozW9zTB5nZUjMrMbOSurq6/lUsIrGvbAXEJcDUhZGuJKqEEug9XV523ZYTgCLgUuAW4NdmNvojP+Tcg865YudccXa2Rn2JyAmUveBNsjXMpr8dqFACvRIo7LJcAFT1sM0zzrkO59xuoAwv4EVE+ubgTjhQpsFE/RBKoK8HisxsspklATcDy7tt8yfgMgAzy8JrgtkVzkJFZJgoW+k9T+ux5VZOotdAd851AncDq4AtwDLn3GYze8DMrg1utgo4aGalwMvAl51zBweraBGJYWUrIXc2ZE6MdCVRJ6Tpc51zK4AV3dbd1+W1A+4NPkRE+qelHsrfhIsUJf2hkaIiMnTsWAPOD9PUXbE/FOgiMnSUrYD0XBg/L9KVRCUFuogMDZ0+2L7Guxgap2jqD/1XE5GhYe/r4GtUd8UBUKCLyNBQthISUuG0SyJdSdRSoItI5DnnjQ6dchkkpka6mqilQBeRyKvZDIfLNRnXACnQRSTyylYCptGhA6RAF5HIK1sBBcWQrlspDIQCXUQi68h+qHpHzS1hoEAXkcja9oL3rO6KA6ZAF5HI2vYCjJ4I2adHupKop0AXkcjxNcOuV7yzc+vpXjrSFwp0EYmcXa9AZ5vaz8NEgS4ikVO2ApJHwcTzI11JTAhpPnQRkRNyDloOwpF9cHif99y4H9obwdcCHc3es6/52OuO4HLbIZh1PcQnRvooYoICXUQ8nb6PBm5HS5dQbob2JmisgiNVx8L7SBX424//LIuH5HRITIOkEZCU5r0eMRZGFQaXg+vPuj0yxxuDFOgiQ5lz0LAHqj84/tFcF+b9+CHQGdq2cYmQkQcZ+ZB/Fsz4OGQUQMZ4b92ofEjLhrj48NYovVKgiwwVne1Qu+X44K7ZBO1HvPctDrKmwcTzvPAkjL1CLM47kz56Rv3h84guZ9MjIGmkd5at+cqHJAW6SCS1HoLtq2Hrc97t13xN3vrENBg3G85cAuPO8B7ZM7xQFTkBBbrIqXakCrY+7z32vOY1daTnwhk3wuSLIW8OZE7WWbD0mQJd5FSo2wZbn/VCfN8Gb92YKXDeXXD6xyG/WAEuA6ZAFxksB7bDB0/C5qfhwDZv3fizYOF9XohnTdPoSAkrBbpIOB0qh01Pw6YnvYuaGEy6EBYs9Ya3j8qPdIUSwxToIgPVVAub/+SFeMVb3rr8+XDVf8KsTwZ7pIgMPgW6SF8EAt4oyIY9ULcVtiyH3WvBBSBnJlz+rzD7UzBmcqQrlWFIgS7SXXujF9gNe6Bhb5fXe+DQXvD7jm2bOQkuvNcL8dyZkahW5EMKdJGjfC3w8n/Aul94IyePSh4FYyZ5gT39ai/Ejz7GnKYLmzJkKNBFAPb8BZbfDfW7YN6nYerCY6Gdmhnp6kRCokCX4a29Edb8G6z/lRfetz/rDe4RiUIKdBm+dv4Zlt8Dhyvg3Dvh8m9485aIRCkFugw/rYfgxW/Au7+DsUXw2VUw4ZxIVyUyYAp0GV7KXoDnvuj1Hb/wH+GSr0JiSqSrEgkLBbrEPl+zN2pz/W/gg2WQMwtuftSby1skhijQJbb4WrzwrnoX9m+Eqo1woMwb+BOXAJd+zes3npAU6UpFwk6BLtHt4E5vHvGqjV6IHw1v8KakzZsLM6+D8XO94fjpOZGtV2QQKdAl+vhavCH37/wO9r7urUvL8UJ75rVeiI+f590mTWQYUaBLdHDOOwN/93felLTtR7ybQCy8D2bfAKMnaMSmDHsKdBnaWurh/WVekNdsgoRUrwnlrE/DxAsU4iJdhBToZrYY+BEQD/zaOfftE2x3A/AH4GznXEnYqpThJRCA3a/COw9799r0+7xmlI/9N5xxA6SMinSFIkNSr4FuZvHAz4ArgEpgvZktd86VdttuJPAF4K3BKFSGgcOVsPFR72z8UDmkjIb5f+OdjY87I9LViQx5oZyhLwB2OOd2AZjZ48B1QGm37f4d+A7wT2GtUGJbpw+2veCdje98yeuhMvkSWPhN7zZtGvQjErJQAj0fqOiyXAkcN07azOYBhc6558zshIFuZkuBpQATJkzoe7USO+q2wbsPw8bHoOUAjBwPF30J5t6mm0OI9FMogd7TVSf34ZtmccAPgM/09kHOuQeBBwGKi4tdL5tLtHMOWhu8YfZNNdBcB0eqYOvzULHOG+gzbTGcdbs3XW1cfKQrFolqoQR6JVDYZbkAqOqyPBKYDbxiXo+DccByM7tWF0ZjjHPQdhha673eJy0Hg4/g6+ZaaKrzwrup1gvwQMdHP2fsVLjiAZhziwb6iIRRKIG+Higys8nAPuBm4NajbzrnDgNZR5fN7BXgnxTmUaq1AWq3Qm2p96gr84K5pd4L8kBnzz8XlwBp2V5Ap+VA7mxIz/ZGa6YFn9NzvNepmepuKDIIeg1051ynmd0NrMLrtviQc26zmT0AlDjnlg92kTIIfM1eWNduCYb3Fu/R2OUfX8kZkD0dsopgxNhjj9QxXZbHeI/kDIW0SISF1A/dObcCWNFt3X0n2PbSgZcl/Rbwe80djfuhsbrLc1XwObiu5eCxn0lI8YL7tEsgZ4Z39/qcGZCRr5AWiSIaKRrtGmug/E3vsfcNqNl8/A2OASzOa/IYOc4bIl94jjfPSfbpXnhnTtIFSZEYoECPJs55NzEufxP2vgnlb3jL4A2JLyiGC74Aowq8boAjx8HIPK/dOl5ftUis02/5UNXaAPW7oWG391z9AZSvg6Zq7/3UTJhwnjeScuL5MO5MzfEtMswp0CPF1+J18ztceXxwH31uO3T89qMmeHejn3CuF+BZ0yEuLjK1i8iQpEAPl0DAC+Hj+mQH+2I31R7fR7u5DnxNx/98XAKMKvRGSc4+y3vOnBx8nqS70YtIrxTooWip9y421pZ6vUS6D6hpOej10T56p5zuUscc64Odf1aXvtk5kDHeC+5RhWrnFpEBUYJ01emDg9u98K7ZFHze7HXzOyou4fg+2TmnH798tJ92WtaxEI9PjNwxiciwoUA/VAGvfhv2vQsHth0bqh6fFOybfSnkzvIeObO8kFbfbBEZgoZvoDsH7z8BK77sDcaZdCFMuyoY3rNh7BSdWYtIVBmegd5SD899EUqf8br+ffIXmrJVRKLe8Av07avhmbu8UF90P5z/BY2SFJGYMHwC3dcML34DSh7yhrv/9VO6rZmIxJThEegV6+GPS70BO+f/A1z2Dd3aTERiTmwHur8DXv0OvPY9b+bAzzznXfwUEYlBsRvoDXtg2e2wfyPMuRWu/jakjIp0VSIigyY2A33fBnj0JvD7YMnvYOa1ka5IRGTQxV6gb30enrzDu/3ZZ1ZA9rRIVyQickrE1nR9b/0SHr/Nu9vO515SmIvIsBIbZ+iBAKz+V3jzpzD9GvjUrzU7oYgMO9Ef6B2t8PRS2LIcFnweFv+nBgqJyLAU3YHefAAeuwUq18NV34Jz79TEWSIybEVvoB/cCb//lDe17ZLfwszrIl2RiEhERWegl78Fj93snY3f/iwULoh0RSIiERd9vVxKl8NvPwGpo+GO1QpzEZGg6DtDTxoBBWfDkochbWykqxERGTKiL9CnLoIpC3XxU0Skm+hrcgGFuYhID6Iz0EVE5CMU6CIiMUKBLiISIxToIiIxQoEuIhIjFOgiIjFCgS4iEiMU6CIiMUKBLiISIxToIiIxQoEuIhIjFOgiIjEipEA3s8VmVmZmO8zsqz28f6+ZlZrZ+2b2kplNDH+pIiJyMr0GupnFAz8DrgZmAreY2cxum70LFDvnzgSeBL4T7kJFROTkQjlDXwDscM7tcs75gMeB427g6Zx72TnXElxcBxSEt0wREelNKIGeD1R0Wa4MrjuRO4CVPb1hZkvNrMTMSurq6kKvUkREehVKoPd0NwnX44Zmfw0UA9/t6X3n3IPOuWLnXHF2dnboVYqISK9CuQVdJVDYZbkAqOq+kZktAv4FuMQ51x6e8kREJFShnKGvB4rMbLKZJQE3A8u7bmBm84BfAtc652rDX6aIiPSm10B3znUCdwOrgC3AMufcZjN7wMyuDW72XSAd+IOZbTSz5Sf4OBERGSShNLngnFsBrOi27r4urxeFuS4REekjjRQVEYkRCnQRkRihQBcRiREKdBGRGKFAFxGJEQp0EZEYoUAXEYkRCnQRkRihQBcRiREKdBGRGKFAFxGJEQp0EZEYoUAXEYkRCnQRkRgR0vS5Q8lz71fxxPoK0pMTvEdKAiODz+nJicctj0lL4rSsNMx6uoueiEhsibpA9wccze2d1Bxpo6mtk8a2Tpp8nbge73IK03NHcmNxAdfPy2dsevKpLVZE5BQyd6IkHGTFxcWupKQkLJ8VCDhaOvw0tXXS1N7hhXx7J3sOtvDUhko2VhwiMd5YNCOXJWcXcnFRNvFxg3fWvrX6CGNGJJGTkTJo+xCR4cnMNjjnint8LxYCvTdl1Y0sK6ngj+/uo77Zx7iMFG6YX8CS4kImjB0Rtv10+gP8+KXt/OTlHWSkJPKDm+Zw+em5Yft8EZFhH+hH+ToDvLSlhidKKli7rY6Ag/NOG8uSswv42BnjSUro/zXiyoYW7nl8Ixv2NvBX8/LZWt1I6f4j3HnpFO69YhoJ8br+LCIDp0Dvwf7DrTy1oZJlJZWU17dQOCaVf1w0jevm5ve5OWblB/v5ylPvE3DwH9fP5rq5+bR1+Pm3Zzfz2NsVnDN5DD+5ZZ6aYERkwBToJxEIOF7dVsf3Xixjc9URpuWm86Urp3PlzNxee8e0+vw88Fwpj71dzpzC0fzk5nkfacJ5+p1K/uWPm0hLTuDHt8zl/ClZg3k4IhLjFOghCAQcKzdV8/3VZeyqa2ZO4Wj++arpXDC15wDeWn2Ef3j0XbbXNvF3l0zhS1dOI/EEzSrbahr5+99vYPeBZv5x0TTuumwqcYN4UTZc9h1q5Y0dBwg4R1JCHEnx8SQnxJGUEPfhs/faWz8iKZ70lASSE+IjXbpIzFKg90GnP8DT7+zjh2u2UXW4jfOnjOXLV01n3oRMAJxz/H7dXv79+S2MSk3kv5fM4aKi7F4/t7m9k6//8QOe2VjFJdOy+cFNcxmTljTYh9Mnzjk2Vx1hdWkNq0trKN1/pF+fkxhvH44RSEtKYGRKAmlHxw0kJ5CaFE9KovdH4ETPyQnxTM1JZ9woNVOJdKVA74e2Dj+PvlXOz17ewcFmH1fMzGXpxafxq7W7eLG0hkumZfP9JXPI6kPfduccj7xVzgPPljI2PYmf3noW8ydmDuJR9K6908+bOw+yZksNa0prqT7ShhkUT8xk0YxcLp2eQ3pKAr7OAL7OAO2d/mOv/QHaOwL4/AHaO/y0dvg/7DLa3N7pjRMIPjf7ji23+fy0dfrp8J/8/73EeOO2cyZy12VTyR6pMQQioEAfkOb2Th56fTcPrt1FY3snifHGVxafzmcvmNzvZpMPKg9z56Mb2H+ojY+dmUdCXBwOh3MQcI6A88L/2LIjb1Qqf3PBJCaOTRvwMbX6/KzctJ81W2p4tayOZp+f1MR4Lp6WxaIZuVx+es4pGYTlDzjaO/20dwRo6/bc4vOz/L19LCupJDkhjs9eMJmll5xGRkpin/bR4Q+wurSG36/by9bqRpYUF/K5iyb36Q+xyFCiQA+DQy0+Hl9fwYVTs5idP2rAn3e4tYN//dMm1u+pJ84MM3p8jjMwjN0Hm/EHHNfNGc+dl01has7IPu+ztrGNh9/Yy+/f2suhlg5yRiazaGYuV8zI5bwpY0lJHHpt37vqmvj+6m08//5+RqUmcuelU7j9/Em91lrZ0MLjb1fwREkFdY3t5I9OZUbeSF7aWktyQhy3LpjI0otPU5OORB0FegyoPdLGr17bxe/XldPW6eea2XncedkUZo3v/Y9LWXUjv35tF89srKIjEGDRjFzuuHAyCyaNiYqLswCb9h3mO6vKWLutjtyMZO5ZOI0biwuOuxDtDzhe3VbLI+vKebmsFgdcPj2H286dwCXTcoiPM3bWNfHzl3fyp437iDdjydkF/N0lUyjIDN8AM5HBpECPIfXNPh56fTe/fWMPje2dLJqRw12XTf3wou1Rzjle236AX722i9e2HyAlMY4b5xfy2QsnMzlr4M02kbJu10G+88JW3ik/xKSxI7j3yumcO3kMy0oqeOztCvYdaiUrPZmbzy7k5gWFJwzqivoWfvHqTv5QUoFzcP28fO68bGpU/7eR4UGBHoMOt3bw8Bt7+M1fdnOopYOLirK4+7KpzJ0wmuUbq/jN67vZWt1I9shkPnP+JG5dMIHMIdarpr+cc6zZUsv3VpVRVtP44foLpo7ltnMmcsXM3BN2Ie1u/+FWfvnqLh57u5wOf4BPzBnPXZdNZVpu35u0RE4FBXoMa27v5JG39vLg2t0caGonLSmeZp+f08eN5I4LJ3Pt3PEx2y/cH3A8+14Vuw40c93c8UzJTu/3Z9U1tvPr13bxu3V7afH5Ofe0Mdx0diFXz87r97UFf8Dx5s6DvFhaTZwZE8eOYMIY71E4ZsSgXrNo9fl5r/IQjW2dnD9lLGnJUTexqpyAAn0YaOvw88T6Ct6rOMQn5+VzUVGW5oHvh4ZmH4++Xc4T6ysor29hZEoCn5ybz01nF4Z0MTwQcLxT3sCz71Xx/AfVHGhqZ0SSF9wtPv9x2+ZmJDNxTBqFwZCfOHYEBZmp5GemkjMypU9TUFQfbqNkbz0b9jawYW8DpVVH6Ax4v9tJCXFcODWLq2blsnBGrnr4RDkFukgfBQKOdbsPsmx9BSs3VdPeGWBmXgY3nV3IJ+fmM2rEse6TRwdkPfteFc+9v599h1pJTohj4YwcPnHmeC47PYfkhDgONvsor2+h/GCL99zldfWRtuP2nxBn5I1OIX90KvmjR5CfmUr+6JQPXze3d34Y3hv2NrDvUCsAKYlxzCkYTfGkTOZPzCQlMZ41pbWs2lzNvkOtxBkUTxzDlbNyuXLmuLDONgreuIY/lFTyh5IKZuRlcMP8AuZPzNTJRRgp0EUG4HBLB8+8t48n1lewueoISQlxLJ41jo+fmcemqiM8F2z2SYgzLp6WzSfm5LFoRi4j+9Bnvq3DT2VDCxUNrexraGXfIe+56pD3uuZIG4EeflXHZaQwf1Im8ydkUjwpkxl5GT1eP3DOUbr/CKs21/Di5mq2VnvXHk4fN5KrZo3jylm5zMzL6HfwtnX4WVZSwS9e2cn+w21Mzx1JRUMLLT4/k8aO4Ib5BVx/VgH5o1P79flyjAJdJEw27Tv84dz6jW2dmHlTMH9izngWzxo3aBeeO/wBqg+3URkM+6SEOOZPzGT8qJR+hfDeg82sLq1h1eZqSvY24BxMHDuCq2fncc0Z4zgjf1RIn9vW4eext8v5n1d3UnOkneKJmdyzqIgLp2bR4vOzclM1T26oYN2ueszg/CljuWF+AYtn5ZGaFJvXdgabAl0kzNo6/Ly9u57Tx42M+mmRDzS18+LmGlZu2s8bOw/iDzjyR6dyzRnjuPqMPOYWjP7IeIVWn59H3trLL9fuoq6xnQWTx/DFhUWcN2Vsj38IKupbeOqdSp56p5KK+lbSkxP42Bl53FBcQLGaZPpEgS4iIWlo9rF6Sw0rP9jP6zsO0OF35I1KYfHscVxzRh4z8jJ49K29PLh2FweafJw/ZSxfWFjEuaeNDenzAwHH+j31PLmhkuc/2E+Lz8+cglF8/ZoZnBPiZwx3CnQR6bPDrR28tKWGFR9Us3Z7Hb7OAHEGAQcXFWXxhYVFnD1pTL8/v7m9k+XvVfGjNdupPtLGlTNz+crVpw+o+2m41Ta2saO2iTMLRpM+RLp+KtBFZEAa2zr489ZaNlYc4uNnjg/rLKGtPj8P/WU3P395B22dAW47ZwL3LCw6JRPEdeeco6ymkTWlNazZ4h0veL2O5k/M5OJp2VwyLZuZeRkRmzZDgS4iQ15dYzs/emkbj71dQWpiPH9/6RTuuHDyoE8a5+sM8Pbuem8K6S01VDZ4XUDnFI5m0ek5zByfwfo9DazdVvfhPQKy0pO4qCibi6dlcVFR9int2z/gQDezxcCPgHjg1865b3d7Pxl4GJgPHARucs7tOdlnKtBFpCc7apv49sqtrNlSQ96oFP7pyulcPy8/pDPitg4/bR1+OgOOTr+jMxAIPh//2h8IUF7fwpottawtq6OxvZPkhDguKjo2hXRPF7trG9t4bdsB1m6v47XtB6hv9gEwOz+Di4q8M/ei3HQmZ6UN2gjtAQW6mcUD24ArgEpgPXCLc660yzZ3Amc65/7OzG4GrnfO3XSyz1Wgi8jJrNt1kG+t2ML7lYeZmZfBwhk5NLZ1Bh8dNLUfe310vc8f6NM+stKTWTQjh4UzcrlwalafulIGAo5NVYdZu62OtdsOsKG8AX9wsECcwcSxaUzNSacoJ52i3HSmZo9kSk4aI5IG1hY/0EA/D7jfOXdVcPlrAM65/+yyzargNm+aWQJQDWS7k3y4Al1EehMIOJ59v4rvripj3yGvu2NGSiLpyd6tDb1HIiNTvFseZqQkkpIYT2K8kRAXR0KckRBvxMcZifFxwWcjPuJc1TcAAAUGSURBVC6OMSOSmDU+fG3hbR1+dh9oZnttEztqGtlR18T2miZ2H2j+cBoGgILMVL581XSum5vfr/2cLNBD+VORD1R0Wa4EzjnRNs65TjM7DIwFDnQrZCmwFGDChAkhFS8iw1dcnHHd3HyunTMe5xjS8/enJMYzIy+DGXkZx63v8AfYe7CZ7TVN7KhtYnttE9mD1OYeSqD39F+w+5l3KNvgnHsQeBC8M/QQ9i0iggXv4hWNEuPjmJozsl93GeurUCaNrgQKuywXAFUn2ibY5DIKqA9HgSIiEppQAn09UGRmk80sCbgZWN5tm+XA7cHXNwB/Pln7uYiIhF+vTS7BNvG7gVV43RYfcs5tNrMHgBLn3HLgN8DvzGwH3pn5zYNZtIiIfFRI/WeccyuAFd3W3dfldRtwY3hLExGRvgjtxosiIjLkKdBFRGKEAl1EJEYo0EVEYkTEZls0szpgbz9/PItuo1CjmI5l6ImV4wAdy1A1kGOZ6JzL7umNiAX6QJhZyYnmMog2OpahJ1aOA3QsQ9VgHYuaXEREYoQCXUQkRkRroD8Y6QLCSMcy9MTKcYCOZagalGOJyjZ0ERH5qGg9QxcRkW4U6CIiMSLqAt3MFptZmZntMLOvRrqegTCzPWb2gZltNLOouh+fmT1kZrVmtqnLujFmttrMtgefMyNZYyhOcBz3m9m+4Pey0cyuiWSNoTKzQjN72cy2mNlmM7snuD6qvpeTHEfUfS9mlmJmb5vZe8Fj+bfg+slm9lbwO3kiODX5wPcXTW3oodywOpqY2R6g2DkXdYMlzOxioAl42Dk3O7juO0C9c+7bwT+2mc65r0Syzt6c4DjuB5qcc9+LZG19ZWZ5QJ5z7h0zGwlsAD4JfIYo+l5OchxLiLLvxcwMSHPONZlZIvA6cA9wL/C0c+5xM/sf4D3n3C8Gur9oO0NfAOxwzu1yzvmAx4HrIlzTsOScW8tH70p1HfDb4Ovf4v0SDmknOI6o5Jzb75x7J/i6EdiCd7/fqPpeTnIcUcd5moKLicGHAy4HngyuD9t3Em2B3tMNq6Pyiw5ywItmtiF4A+1ol+uc2w/eLyWQE+F6BuJuM3s/2CQzpJsoemJmk4B5wFtE8ffS7TggCr8XM4s3s41ALbAa2Akccs51BjcJW45FW6CHdDPqKHKBc+4s4GrgruA//yXyfgFMAeYC+4HvR7acvjGzdOAp4IvOuSORrqe/ejiOqPxenHN+59xcvPsxLwBm9LRZOPYVbYEeyg2ro4Zzrir4XAv8Ee/LjmY1wfbPo+2gtRGup1+cczXBX8IA8Cui6HsJttM+BTzinHs6uDrqvpeejiOavxcA59wh4BXgXGC0mR29Y1zYcizaAj2UG1ZHBTNLC17wwczSgCuBTSf/qSGv683CbweeiWAt/XY0/IKuJ0q+l+AFuN8AW5xz/93lraj6Xk50HNH4vZhZtpmNDr5OBRbhXRN4GbghuFnYvpOo6uUCEOyq9EOO3bD6PyJcUr+Y2Wl4Z+Xg3dv10Wg6FjN7DLgUbxrQGuCbwJ+AZcAEoBy40Tk3pC84nuA4LsX7Z70D9gCfP9oGPZSZ2YXAa8AHQCC4+ut47c9R872c5DhuIcq+FzM7E++iZzzeCfQy59wDwd//x4ExwLvAXzvn2ge8v2gLdBER6Vm0NbmIiMgJKNBFRGKEAl1EJEYo0EVEYoQCXUQkRijQRURihAJdRCRG/H8ijebk1f9eTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(0, len(attn_success_line[::-1])), attn_success_line[::-1])\n",
    "plt.plot(range(0, len(attn_success_line[::-1])), rate_success_line[::-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydeXhTVfrHPzfdWwqh7FBA9n0XEGVHW1BZFNEARjKu47iMOuq46+A66qjj6IwozESDEhXUHyCKIuACqIAgssjSspWdQqB7m+T8/jhJSdO0vV2TlPN5njxN7r05920peXvO+b7fVxNCoFAoFApFqGEIdgAKhUKhUARCJSiFQqFQhCQqQSkUCoUiJFEJSqFQKBQhiUpQCoVCoQhJIoMdQG1hMBhEXFxcsMNQKBSKoJKbmyuEEGE5Gam3CSouLo6cnJxgh6FQKBRBRdO0vGDHUFXCMqsqFAqFov6jEpRCoVAoQhKVoBQKhUIRkqgEpVAoFIqQRCUohUKhUIQkKkEpFAqFIiRRCcqPu36G236EdSdAGb0rFIraQuQVkfW3ryj67UiwQwlZVILywylgfjpc/CV0XwzP/wYZqpxKoVDUMAUr9+DaewotPjrYoYQsWn3tB5WQkCCqWqibVQQL94M1Db47DhpwWSuwdIIpbSGu3pY3KxSKukAUODl7/xIi2jemwf2ja/VemqblCiESavUmtYRKUBWQlgXvpcO7abA/BxpGwXXt4eYuMKRpDQSqUCjOO/K/+J38DzfT4NFLiexSux8kKkGFIDWVoLy4BXx7TM6qFu6HPBf8fDlc2KTGbqFQKM4DRKGTs/cvJSK5EQ0eHFPr9wvnBKUWq3Ri0GBMS/n4xyBo9wnM3a0SlEKhqByF36YjzuYTO/niYIcCgMNiHw/8E4gA5hqtphf8zrcD3gWMnmseMlpNy+oiNiWSqAJNY2FqO7DvgzxnsKNRKBThgih0kf/5DiK6NSOyW/Ngh4PDYo8A3gQmAD2B6Q6LvaffZY8BHxmtpgGACfh3XcWnElQV+UMnOFMEnx0MdiQKhSJcKPw+HeHII3Zy72CH4mUIsMdoNaUbraZCwA5M9rtGAA09zxsBh+sqOJWgqsjoltA+Af6XFuxIFApFOCCKXOR/vp2ILk2J7FGns6dITdM2+Dxu9TnXBvD9MzvDc8yXp4DrHRZ7BrAMuKtWo/VB9x6UZmMoMA5ojl9iE2buruG4Qh6DBrM6wdNb4GAOtA3LLUiFQlFXFK7ZiziVR+yNQ9A0rS5v7RRCXFjGuUCB+CvnpgNWo9X0D4fFPgywOSz23karyV2jUQZA1wxKs3E/sA6wAP2BPj6PkJmr1jWzOsp/yffSgx2JQqEIZYTTTcHSHUR0TCKyV8tgh+NLBtDW53UypZfwbgI+AjBaTeuAWKBOimz0zqD+DNwtzLxRm8GEGx0TYVQLKT1/pDfU7R9FCoUiXChctw/3yRwSrh9U17OnilgPdHFY7B2AQ0gRxAy/aw4gV8+sDou9BzJBnaiL4PTuQTVErj0q/PhDJ9iTBWvq5J9LoVCEG8LlpmDJdiLaNyayX6tgh1MCo9XkBO4ElgM7kGq9bQ6LfbbDYp/kuewvwC0Oi/1XYAFgMVpNdVJAq6tQV7PxFrBFmOtOXlhdarpQtyxyiqDlQri2PcwLjbIGhUIRQhSu2UvuOz+RcPdwogYm1/n9z4dC3YPA3zQblwBbgCLfk8LMKzUdWLiQEAXT2sNH++H1wfK1QqFQAAi3m/wl2zG0NRI5wF8cp6gIvQnqZiAbuNjz8EXA+ZugQC7z/S8NFh2AGzrVzj22nobkBDAq42OFImwo+vkg7qNZxN9xSajtPYUFuhKUMNOhtgMJZ4Y3h06JMknVRoIqcsOwL+HmzvDq4JofX6FQ1DzCLchfvA1Dm0ZEDar7pb36QKULdTUbDTQbVVrP1GyM12zs1Gzs0Ww8FOD8SM3GL5oNp2bjGr9zszQbuz2PWVW5f22haWDpCKuPQXpWzY+/zQHZTvg5s+bHVigUtUPRhoO4D58ldlIvNIOaPVUF3QlKs3GHZuMAcAY4q9nYr9n4UyXeX8rzSbPh7/l0AFlr9YHfe5OAJ4GhSGuOJzUbjfXeuy6Y1UlWvNVGTdRGT2L69TS4ar00TqFQVJfi2VOrhkQNVrOnqqK3UPcR4AVgHpDiefwPeCHQTKgMhgB7hJl0YSag55Mws0+Y2QL4fwynAl8LM6eEmdPA18B4nfetE9omwKWtZE2Uu4YFmBtOya85TilpVygUoU3RpkO4M84QO7EnmkE5ylUVvSKJPwK3CjMLfI59o9nYDTyHTF4VEcjzaajO++vxi8LjMXUrQHR03asJLJ1g5g+w+iiMrcFyh42Z0DwWjufDplPQrVHNja1QKGoWUeQi/6NfMbRMJGpou2CHE9boTe3NkRXH/vwMtNA5hh7Pp2q9VwjxthDiQiHEhZGRdd/q6qq20CgKrDW4zFfkhi2nYfoFEGWATadrbmyFQlHzFHzxO+5jWcRdPxAtQs2eqoPen94uSttf4Dm2U+cYejyfauO9dUZcJJgukB13zxbWzJjbHFDghouaQm+jnEEpFIrQxHUim/yl24m6sC1RvUPLNSIc0TvNeAr4SLMxEliDnL0MB0YB03SOsR7ootkoz/OpLJYDz/kII1KAh3W+t06xdII5u2Xh7s1dqj+eVyAxqAkMaAyLM0AI5funUIQieR9sAk0jbnr/YIdSL9A1gxJmPkHuFx0FrgQmeZ4PEWY+0zlGKc8nYWabZmO2ZmMSgGZjsGYjA5n05mg2tnneewp4Gpnk1gOzPcdCjqFNoXtDKZaoCTacgoZRss5qQBKcLIBDuTUztkKhqDmKNh/GuekQsZN7YWgSls5CIYcuL75wpK68+ALx4jb46y+wczJ0bVjx9eUxZBk0iISVKbDmOAxfDotHw8S2Fb5VoVDUEaLQSdajX0BUBImzU9EiI4IdUjHh7MVX5gzKU3tU/Ly8R92EGj5c30E2NHy3mrMor0BiUBP5ul9jqRZRQgmFIrQoWPY77hM5xF0/KKSSU7hT3hLfCc2Gty/xSWT/D/+H97jCh9bxML41vJtevcJar0BikOdPgAZR0KWhEkooFKGE67hHGDG0HVE99YqaFXooTyQxFor3ecaiXxKuQBrITvsOVhyF1NZVG8NXIOFlQGP48WT141MoFNVHCEHe+79ApIE404Bgh1PvKDNBCTPf+jxfXSfR1CMmJkNSNMxPr3qC8hVIeBmQBB/uh1MFkBRTM7EqFIqq4dx8GOevh4k19cfQOC7Y4dQ79FoduXyW+3yPN9FsuGo+rPAnJgImtIHlh6tufbQxUy7v+fpMDvAs921Wy3wKRVARBU5y52/E0KYRMZd2DXY49RK9hbplVd3EADVUklr/SG0NJwqqlkz8BRJevAlKCSUUiuCSv3Q7IjOX+BsGoUUqx4jaoNxCXc3GfZ6nAvijZiPb53QEMAL4vZZiC3tSPIXkyw/DwCblX+uPv0DCS7NYaBOvhBIKRTBxHc2i4IvfiRrWnshupRaXFDVERU4Sd3m+asiuur7LeYXAPqSRrCIALeKgf2NYfgQe7lO59wYSSHgZ0FglKIUiWAghyJu/EaIiiLtOOUbUJuUmKG8nXc3GKuBqT6sLRSVIbQ3/2A5ZRZAYpf99GwMIJLwMSIJlhyHXCfF174mrUJzXFG3IwLn1KHEzB2IwKmFEbaLX6miMSk5VI7U1OAWsOlq5920IIJDwMiBJCi9+U/8iCkWd4T6bT/7yneTZNmBoayR6bOdgh1Tv0f33t2bjOmAcsvVGicQmzNJLT1GaS5pBQqTch5qk057IK5C4q3vg875CiaHNaiZOhUJRGuF0U/TrYQp/2Itzy2FwCSI6JBF/45B600rDYbGPB/6J1BXMNVpNL/idfxUY43kZDzQ3Wk3GuohNV4LSbLwE3AOsQra5UEW7OomOgDEt5D6UXsoSSHhpnwCNo9U+lEJRW7gOOij8Pp3CdfsRWQVojWKJSelG9PAORLSpPx1DHRZ7BPAmcBmyrdF6h8W+2Gg1bfdeY7Sa7vW5/i6gziqS9c6gbgCmCzMLazOY+kpqa1h6CNKyAu8p+VOeQAJkq43+SiihUNQootBJ4bfpFP6wF9f+0xBhIGpAa6KHdySyT8t6M2PyYwiwx2g1pQM4LHY7MBnYXsb104En6yg23QnKAGyuzUDqM14nieWH4U/dKr6+PIGElwFJ8O9d4HSDKsFQKKpP7n/XU/TjfiLaNyZu5kCihrXH0KBe2LVEapq2wef120KItz3P2wAHfc5lIFsrlcJhsbcHOgArayXKAOhNUG8D1yMbFyoqSedE6NBAf4IqTyDhZUAS5Lvg9zPQu3HZ1ykUioop2nGMoh/3E3NlT+Ku6RvscGoapxDiwjLOBfqUKWsLxwQsNFpNdeYepDdBGYEZmo3LgC1Ake9JYebumg6sPqFpchY1Px0KXXJfqiwqEkh48RVKqASlUFQd4XSTZ9uIoWkCsRN7BjucuiYD8JVvJSN1BoEwAXfUekQ+6F0c6olc4isEugN9fB69aye0+kVqK8h2wroKnMgrEkh46dYQYiPUPpRCUV0Kvt6F+/BZ4mYORIs57woL1wNdHBZ7B4fFHo1MQov9L3JY7N2AxsC6ugxO17+GMBdLDBVVZGxLiNTkMt+oclrGVCSQ8BJpgL5GlaAUiurgPpVL/mdbiezXmqgBbYIdTp1jtJqcDov9TmA5Umb+X6PVtM1hsc8GNhitJm+ymg7YjVaTbgW3w2LvA9wGdAJuNFpNRxwW+xRgv9Fq2qRnjEr9uaDZaOq52WZhpqAy7z3faRgNw5rJBPVcOSJNPQIJL97WG0LIZUSFQlE58uybwC2Imzkw2KEEDaPVtAxY5nfsCb/XT1VmTIfFnoKciX2B7CfotdzoBFiAKXrG0dtuI1Gz8TFwHFiLVH6g2XhLsynhhF5SW8Mvp+B4XtnX6BFIeBmQBI5C2Jdd8bUKhaIkRduOUvTzQWKv7EFE8wbBDqe+8TRwn9FquoqSHS9WI6XtutC7B/V3oDUwEPD9eF0KXKX3Zuc7Xrn512UU7ZbVYqMsVOsNhaJqiCKXFEY0b0DMhB7BDqc+0gu/WZmHU0AFO+zn0JugJgH3CDObKSlB3AF01Huz852BSdA0pmxXCb0CCS99jBChhf4+1N+3wtBlcNuP8PYuOUssUG0uFUGkYPlO3EeziLt+IFp5slpFVTmNZ6XNj4FI5aAu9O5BNQYyAxxPBP0ddTUbJTyfhJkX/M7HAO8Bgzz3u06Y2afZiALmIr+5SOA9YeZ5vfcNFQwaXNYKvvJ02fVfxtMrkPASFwndG4Z2gjqWB09tgeax8NF+eHu3PB6pQW+jTNqDmsiv/RrL70mhqE3cmTnkL95G1MA2RPVtHexw6isfAC85LPZrkZOaSIfFPgp4Gfif3kH0zqDWQwlDWO8s6jbknlSFaDa8nk8TkLL16ZoN/6KDm4DTwkxn4FXk0iLANCBGmOmDTF63aTYu0Bl7SJHaGo7ly6U8fyojkPAyICm0E9SrO6DQDSsuhVPXQtoU+Hgk3N9TJq3/y4A7foZhX0JDO0xeBYv2qxmWovbI+0AKyOJmnL/CiDrgMWAvsB9ogLROWgn8ADyrdxC9f68+AizXbPTyvOc+z/MhwEidYwwB9ggz6QCajUCeT5M551axEHhDs6EhE2KCZiMSqQYpBM7qvG9I4dtlt7/fUt7GSggkvAxIgvl7pfCieYi1pjldIO2Yrm0PXRrKYx0T5eOa9vK1EHAwF37JhO+Pw4J9sDhDmuFe2x5u6CjVj0qlqKgJirYcoWhjBrFT+2BomhDscOotRqupCJjpsNgfR658GYBNRqtpd2XG0VsHtVazcTFwP5CGbLvxCzBMmPlN5730eD4VXyPMODUbZ4AmyGQ1GTiCtHu/V5gpNW/QNO1W4FaA6OhonWHVLa3ioW9jmaD+6lPiXOSGX3U4SPjjK5RIDbEE9eZO2ajx4XJKuTUN2iXIx5R28OJA+OYovJcuH3N2yxnlDR3h+g4yuSkUVUEUush7fyOGlonEjK/kfzRFpXBY7E8AL3tMaNN9jscBDxitptl6xtG94u9JRLMqG6gPejyfyrpmCHKvqzVyP+x7zcYK72ys+EJpgPg2QEJCQsi2BEltBa/9DtlF0MDTZbeyAgkv/T02R5tOnVMJhgI5RfJ7vLKNTMh6iTBASmv5yCqCRQfgvTR46ld48lcY0Ryubif7bPVPgihllKvQScGXv+M+lk3C/aPQopQwopZ5EngLyPU7Hu85pytB6a2Dcmk2mgc43kSz6RZJ6PF8Kr7Gs5zXCClLnAF8KcwUCTPHgTVAWeaHIU9qazljWn3s3LHKCiS8NI6BCxJCbx/q7d2QWQCP9Kn6GIlRYOkEK1Ng39XwXH84WQD3boAhX0AjO4xaDo9sgqUZ8n4KRSBcJ7LJX7KdqAvbEtW7VbDDOR/wbs34MwBKr36Vhd4ZVFk7ADGULMIqj/VAF81GB+AQ0vNpht81i5GztHXANcBKYUZoNg4AYzUb85EZ+CLgNZ33DTmGN4f4CLnMd2WyPFYVgYSXUBNKFLjg5e0wuoXcP6oJ2iXAw33kIyNHehquOQ5rT8BL28Dp+a/QvSFc3AwmtDm3z6U4fxBCIHKLEI483Kfz5FdHHkW/ZIBBI256/2CHWK9xWOxZyMQkgHSHxe6bpCKAWOTMShflJijNxn2epwL4o2bD17MgAhgB/K7nRp49pRKeT8LMNs3GbGCDMLMYmAfYNBt7kFnW5Hn7m0hp4lZksvyfMLNFz31DkZgIGN1SJigvVRFIeBmQBJ8elEtiiVE1F2dVeS8dDufBu5fUzvjJCTAtAaZ5ElCuE9ZnwlpPwvrsIPw3Db5LgRHl+B4qwh/X8WzyP9mC+1Qu4nQebkc+FJVe1NESoombMQBDEyWMqGXuRH5G/xd4FDjjc64Q2Ge0mnQbzmpClL1Vo9nY63naHrn85vsvXwjsA54QZn7Se8O6IiEhQeTk5AQ7jDJ5fQf8eQOkT5EfuIkLpEDipUGVH2tpBkxcBd+nytlZMHG6oftiSIqGnyYER313thCafAQP9oJn66w5tSIY5C3aQsHSHUR2a4ZmjMNgjENrLL8ajLHFz7Xo87fATtO0XCFEnWZmT83TWo+ar8qU+68mzHQA0GysAq4WZpSpTg1R3GX3CFzUtGoCCS/FSr5TwU9QH++Xre1fHhU8abivMa9KUPUbV3omEW0b0eChscEOReGD0Wr61vvcYbG3BKL9zh/QM47ePytWQWn3cs1GHPCAMOtTZCjO0bUhtE+QH6JRng/yygokvLSOg2Yxwd+Hcgt4biv0bAST2lZ8fW2S0gqe+BVO5EOz2ODGoqgdhFvgTD9F9NB2wQ5F4YfDYm8I/Au4Fr/k5EGXjFKvSPdJZDWwP17JoKKSeLvsfnMUfjpZdYGEd6xQEEoszYCtDln3VJW9tJoktbXcOF1Rhu+hIvxxHz0LeUVEdKriX3aK2uQfQD9kW418pCDuAeRW0XV6B9GboGpEMqgoSaqn1ueDfVUXSHgZkATbzsiW8sFACHj2N+jQAEwXBCcGXwYmyX2wr1SCqrc402RtRqRKUKHIBOAuo9W0HKld2Gi0ml4BHkJa5OmiIhVfCcmgZqNakkFFSca1lG7kOc6qL+95GZAka6u2nTm3J1WXrDoKP2fCW0Nlt99gE2E4Z8yrGjrWT1xpmRAXhaFlw2CHoiiNEenDBxQ7Au1BlhDN1TtIRXtQFUoGhblue9TXJxpFS4HEmhNVF0h48RVKBCNBPbcVWsXBrE51f++ySGktOw5vdUCfSrhZ+HIyX3oD/qGTSnKhhjMtk8iOSWjBXk9WBCIN2YrpALItk8lhsf8MXE1NFeoKM+9Csdx8jTDj9L9Gs5EozGRVInCFD+NbywR1YTVnUJ0ToUFkcPahfjoh99JeHgSxIeQg4zXm/epI1RPU81vhlR2y99bgpjUXm6J6iAIn7owzRE30b4igCBGsQF9kB90XkM1t70RuK/1Z7yB6zWK/9T+m2RgO3AJMJbCAQqGDe3rIGU/naq5SGDTZT2nTKbmkleuEbO+jqOTzfJd0sGhUQ366z22V7uO3damZ8WqK5ASpKFx+GP5Shc8xl1u6qwMsPaQSVCjh2it/0SM7qv2nUMRoNb3q83ylw2LvjrSn2220mvQajOs3iwXw+PHNQvZtugDZ3+OuyoyhKEmDKLgiuWbGGpAEb+yEiPmBFS2+PNwbnquBGqHfTsslsKf6njO+DSVSW8O/d0Kes/LNEFcdgyN5EBcBSzLgb/1qJ0ZF5XGmS4GEUvCFB566pwMADou9rdFqOljBWwAdCcrTj2kCcrY0AdgAdAaGCjMbqxyxosa5u7v8MI2LlMt9DSJl0vB//qefYfHBmklQL2yVY1a2TUhdkdJKNk387njl3d7f3yvl//f1kF2BM3LkrEwRfFxpmRiaN8CQGBPsUBQ68RTsPg7ciOzrVyEVqfieBixIHft84D5hZq9mowjIq1a0ihqnS0N4UYdV0tVt4b6NsDcLOlSjv9LBHLDvlx/gSSH6OTGyBcQYpJqvMgkqzylbfVzTTnr+PbUFPj8Et3WtvVgV+hBC4Ew7SWQPZbQYajgsdiPSOzUFKELuP/0LeAL4K7ANmaB0UdEM6mHgeeApYdbdVkMR4kxMlglqSQbc3aPq4yw6IN0jbg2xvSdf4iNlD6nlh2XloF6WZMgates7QI9G0LGBPKYSVPARp3IRjny1/1RDOCz28cA/kaVDc41W0wsBrrkW2e1cAL8arSb/ThRenkN2WX8XGA+8ClwGJAATfC2Q9FBRxcqDwFVAhmbjVc2GcjarB3RuKNtSLMmo3jiLDkh1W5cQL0NJbS3rww75t04rh/l7pYXUqBZSXn5lslQq5pbSsSrqGme6lKqq/afq47DYI5AznglAT2C6w2Lv6XdNF+Rk5RKj1dQLuKecIa8A/mC0mu4HJiHLlNKMVtPYyiYnqCBBCTOvCDO9kdr1ROBbzcY2z03V/DqMmZgM3x6Xzt9V4Wie7McUDj2XUjxLe1/5t8csg5P58MUhmNFBFvyC/Hnlu5R1UijgSjsJkQYi2hmDHUp9YAiwx2g1pRutpkLADkz2u+YW4E2j1XQawGg1HS9nvNbAds916cjtoXeqGpyumn9hZp0wczPQCjllWw98o9nYoNn4a1VvrggeE5Ol88TyKn7gfnpAzvWnhoFPZx8jtIzTb3v08X7ZAHFmh3PHRjaXgoml1Zx1KqqPMz2TiAsao0WGUNFdaBOpadoGn8etPufaAL6KugzPMV+6Al0dFvsah8X+o2dJsCwMyL0nLy5Kt33XH3hlLhZmcpA2FXM1G72Am4H7gL9XNQBFcBjWTHrVLck41/ivMiw6AN0ayjqjUEfTpJrv80Oytimigj/L3t8LvRrJujIv0RFyqXDpIbnvpswLgoNwunHtPU3MmBCyLAl9nEKIC8s4F+g32b9KJRLoAowGkoHvHRZ7b6PV5ChjvPkOi93b/SIWeMdhsZdIUkaraZKewKvsmibMbBNm7vUErAgzIg1weRtY5vnQrgwn82H1MTl7Chf7n9TWkFkAmyroaLY3Szp7zOxQ+nu7so2si/pF2SMHDVeGA4pcRHRSVdM1RAbg2xwnGfBfDM8A/s9oNRUZraa9wE5kwgrEu573Z3oe85EztEy/hy6q3WZSmKlWx0RF8JiYLMUA605WrtHh/x0ElwiP5T0vl7aUX5cfLt9W6oN98uuMDqXPXd5GzpyWZFTfmkpRNVweB/OIjkEwnKyfrAe6OCz2DsAhwIRsjeHLZ8B0wOqw2Jsil/zSAw1mtJr+UJPBhYDvtCJYpLaGSK3yar5FB+CChOCY0laV5nGyBUd5QgkhYH66lKW3D2De1TQWhjVV+1DBxJmeidYwFkNTVTFdExitJifSI2850tT1I6PVtM1hsc92WOzeZbjlQKbDYt+ObF77gNFq0j0Lqg6aEBWZ4oQnCQkJIicnJ9hhhDyXfg2H82C7rhVhcBRC84/hz93hJR1FwaHEw7/Ay9sh81rZFt6fXzJh0DKYMxRuLaPe6e9b4aFNkDEV2sTXbryK0px96HMMrRrS4M8jgh1K2KBpWq4QIiwzuppBnedMTIYdZyBNpx/90gyp/gun5T0vqa2lOm/1scDn5++FKEP50vkrPTuuahZV97izC3AfzVINCs8jdCUozcZnmo0rNVv1EppmY7xmY6dmY49m46EA52M0Gx96zv+k2bjA51xfzcY6zcY2zcZvmo3Y6sSikEz0fODqXeZbdEDOHIaE4R71xc0gIVLuQ/njdS6/ok35tk09G8muwdUtclZUHpe3QFc5SJw36E04OcCHSEeJ5zRbmQqOMtFslKpY1mz4N0G4CTgtzHRG1lv93fPeSKQa5I/CTC+k3FGJM2qAjonyQ1fPB252EXx5WHr5haPMOjoCxrQIXA+18qgsPp4ZQBzhi6bJpK5cJeoeV3omaBDZIYw2PxXVQm+h7kxkke7TwKXATs3Gd5qNGzSbPldaPBXLwky6MFNWxfJkpEwRYCEwzuOmngJsEWZ+9cSTqbwBa46JyfDdMThTgavEskPSTWFqGLhHlEVKa9iTBel+S5pe5/IrdRRNeF0lvlGuEnWKMy0TQ5tGaHEh2NdFUQKHxT7KYbEP9XltcVjsPzgs9jkOi113/0DdS3bCzFlh5j/CzBCgD7ARmAMc1WzM0WxUZDuqp2K5+BpP915vL/uugNBsLNds/KLZeFBv3IqKmZgs92a+rMAKaNEBaB4Lw5vVTVy1gdfR3HcWlevjXK6nI/DI5pAYpZb56hIhBK70TGUQGz68BrQEcFjs3ZC5YgswDHhJ7yCV3lPSbLRGznSuBJzImU5bYItm4/7y3hrgmL+EsKxrIoHhwEzP16s0G+NK3UDTbvXaeTidav1FLxc1haYx5X/g5jmlE8OUthU7MYQyXRKhfULJfaglGbLb8PUd9Y0RHQGprc65SihqH/exLEROoTKIDR86Ad7OuVOBr41W05+Qvn4T9Q6iVyQRpdm4RrOxDNgPTAFeBFoJMzcJM5cjk8dj5Qyjt2K5rYM6BxoAACAASURBVOeekUAj4JTn+LfCzElhJhdYBgz0v4EQ4m0hxIVCiAsjI6tdg3zeEOHjKuEsw1XiqyOQ45SzjHDGa3u08qhUI4KsfWoTL53L9TIxWblK1CXeAl2l4AsbBLJ9B8A44EvP86PIVTFd6P1b+AhyipYGDBJmhggz7wgz2T7XfA2UZySzHuii2eig2YhGViwv9rtmMbKlPMA1wEphRiALxfpqNuI9iWsUHsdcRc0wMRlOF8LaE4HPLzoAjaNhdMu6jas2SG0NZ4vgp5PStunLwzDjgsoJPy5vI6f7Sm5eNzjTMiE2EkPrEO/tovCyHnjcYbGbgRHAF57jFyCTlC70Jqh7gTbCzF3CzJZAFwgzp4WZMjVQnj2lEhXLwsw2zcZszYa3THQe0ESzsQdpQvuQd2zgFeQ3vRn4RZj5XGfsCh2ktJI1QIGW+QpdskX85LbymnBnbEuZjL46DB8FcC7XQ9NYabir9qHqBld6JpEdktAM9eAX8PzgHqA/8AbwrNFqSvMcnwas1TuIcpJQFJOyAg7kwO9+2sovDsHlK2HJGH0qt3Dg4i9l7VOkQc6mtlxZeePbF7bCw8pVotYRhU7O3L6ImAndibumX7DDCTtCyUnCYbHHAi6j1aSrTEjXRo1mYxWlBQ14juUDe4B3hZlf9AaqCD0mJsPd62H32ZJdchcdkKq1y1oFL7aaJqUVzN4if4GfH1A1V/aJyTJBLVWt4GsV1/7T4BJEKgfzsMFhsRsAjFaT2/O6JVJYt91oNemeQemdL+9AihJaIQULGZ7nA4HjSGXdT4GUdYrwIZCrhNMNnx2UrSZi6lF/uNTW5/7iCuRcrgevq4Tah6pdnMUO5kogEUZ8DtwF4Kl72oCUl3/rsNhv0DuI3gSVD1iFmR7CzA2eRw/gv0CmMDMI+DfwTGW+A0VocUED6G0smaC+Oyb7KIVDa/fKMLgJGKNlTVO7Ki5+aJpM3CuUq0St4krLxNA0AUMj5W4WRgwCVnqeXw2cBZojZebllSOVQG+CmoW0KfJnDuDt//E2lLIuUoQZE5Ph++Nw2tMPc9EBiI+A8a2DG1dNE2mAz8fAvGHVG0e5StQ+zvRMNXsKPxIBb8fdFOBTz77TSmSNlC70JigN6BXgeE/OFdcWAZXszaoINSYmy2aEXx6WRaifHIQJbSC+HpaVXdwcOldTtTyqhXKVqE3cjjxEZq4q0A0/DgCXOCz2BCAVWYYEkATklvkuP/R+7LwLzPOYxK5HLt8PAf4KWD3XjAK26r2xIjQZ0gSaeVwl2iZIA9VwbK1RV/i6SghRNbGFomycqkA3XHkFsAHZSHOH7zzHR3LOYaJC9Cao+4FjyHoob6nmUeSm18ue18s5V4ylCFMiDHBFshRGNImBaINsQaEomyuTYeEB6SoxSH2O1iiutEyIMBDRrnGwQ1FUAqPVNMdhsW9EOgN97VXzIc0eHtc7TqXroDQbDUGax1bqjXWMqoOqOp8cgKnfyqLc1FawZGywIwptTuRDi4/hsT4wu3+wo6lfZL+wElHgJPHJlGCHEraEUh1UZalUWbZm40JkPyeX53WCx3qoXvLpp5+iaRq///57sEOpkNzcXGbOnEmfPn3o3bs3w4cPJzs7u+I3BiCllZw5Fblrr7XGBRdcQJ8+fejbty+jRo1i//79Fb7nueeeq51gqkmzWBjXUnbkVeaxNYdwu3HuPaX2n8IUh8X+J4fFvs1hsec6LPaOnmN/dVjs1+odQ69ZbAvNxk/Az8AHgNdW8xXgH5WMO2xYsGABw4cPx26318h4LlfttbD65z//SYsWLfjtt9/YunUr8+bNIyqqan1zGkTBmJYQqcGkWnSOWLVqFVu2bGH06NE880zFFQqhmqAALJ1gb7ZUQFaXY3nS5eJ8x33oLBQ4VYuNMMRhsd+DNA9/m5JdKg4jLe90oXcG9SrnXGh9FRgfIyWE9Y7s7GzWrFnDvHnzSiSo6667jmXLlhW/tlgsLFq0CJfLxQMPPMDgwYPp27cvc+bMAWD16tWMGTOGGTNm0KdPHwCmTJnCoEGD6NWrF2+//XbxWPPmzaNr166MHj2aW265hTvvlP+OJ06cYOrUqQwePJjBgwezZs2aUvEeOXKENm3ObRZ169aNmJgY9u3bR+/evYuPv/zyyzz11FMA7Nmzh0svvZR+/foxcOBA0tKkXdaLL75I2l19aP1sP1588iEA0tLSGD9+PIMGDWLEiBHFs8qPP/6Y3r17069fP0aOHAnAtm3bGDJkCP3796dv377s3r273J/1sGHDOHToUPHrQD+fhx56iLy8PPr378/MmTMBmD9/fvF9brvttlr9A6Airmon1XzWtIqvLY+DOdDhUxj/jexgfD7jTDsJQERnlaDCkD8Ctxitpn8i2zJ5+YXAivDACCEqfPCeOMZ7orfneRbviY6e5x14T+ToGaOuH/Hx8aI62Gw2ceONNwohhBg2bJjYuHGjEEKITz75RNxwww1CCCEKCgpEcnKyyM3NFXPmzBFPP/20EEKI/Px8MWjQIJGeni5WrVol4uPjRXp6evHYmZmZQgghcnNzRa9evcTJkyfFoUOHRPv27UVmZqYoLCwUw4cPF3fccYcQQojp06eL77//XgghxP79+0X37t1Lxbtp0ybRrFkzcdFFF4lHH31U7Nq1SwghxN69e0WvXr2Kr3vppZfEk08+KYQQYsiQIeKTTz4RQgiRl5cncnJyxLJly8SwYcNETk5OiVjHjh1bPOaPP/4oxowZI4QQonfv3iIjI0MIIcTp06eFEELceeedYv78+cU/o9zc3FLxtm/fXpw4cUIIIcSf//xnMWfOnHJ/PkIIkZCQUHzN9u3bxZVXXikKCwuFEELcfvvt4t133y11n7rk5rVCJHwgRFZh1cd49BchtPeEiLAJMeRzIU7m11x8ZfHrKSG+PVr796ksOXN/FI47PxFutzvYoYQ1QJ1/Rp+etSDv9KwF7T3Ps07PWtDR87zr6VkLcvWOo3f/KA4I1BC8GdJlot6xYMEC7rnnHgBMJhMLFixg4MCBTJgwgbvvvpuCggK+/PJLRo4cSVxcHF999RVbtmxh4cKFAJw5c4bdu3cTHR3NkCFD6NDhnJ/O66+/zqeffgrAwYMH2b17N0ePHmXUqFEkJSUBMG3aNHbt2gXAihUr2L79XHeRs2fPkpWVRWJiYvGx/v37k56ezldffcWKFSsYPHgw69atIy4uLuD3l5WVxaFDh7jqqqsAiI2NLb7XH/7wB+LjpftpUlIS2dnZrF27lmnTphW/v6BAVvJecsklWCwWrr32Wq6++mpAzoieffZZMjIyuPrqq+nSpUvAGMaMGcOxY8do3rx5iSW+QD+fJk1K/hX9zTffsHHjRgYPHgxAXl4ezZs3D3ifusLSCebukYo+i+5SxHMUuODt3bIW7abOcO13MGI5fDUOkmtpizuzAC79GvJccOBqaBxTO/cBEE4X7uM5uI6cxX08G5FbiMgtOvc1z/taHiPfSWS/1mhKu1+rOCz28cA/kf2b5hqtphf8zluQim3vMscbRqtpbgXDpiOt8Pw3ly+nEq2S9Cao7wAL8IjntdBsRCDroL7Re7NwITMzk5UrV7J161Y0TcPlcqFpGi+++CKxsbGMHj2a5cuX8+GHHzJ9+nRAzkT/9a9/kZqaWmKs1atXk5CQUOL1ihUrWLduHfHx8YwePZr8/Hw5Uy0Dt9tdbrLx0qBBA66++mquvvpqDAYDy5Yt47rrrsPtPrehkZ+fXxxvIIQQpT4Q3G43RqORzZs3l7r+rbfe4qeffuLzzz+nf//+bN68mRkzZjB06FA+//xzUlNTmTt3LmPHlpYCrlq1ioSEBCwWC0888QSvvPJKmT+fQHHOmjWL559/vtyfSV1ycTPZsdeaVrUE9fF+OFEAd3aDy1rD8kth0iq4ZDl8fSl0rYVWSPdvhFOFsjj7P7vgkT7VH9OdXYD7SJZMREfO4jqahduTlEqoSDQNLT7K84hGi4vC0LwBxHlex0cRPUQV4dUmDos9AukSdBnSY3W9w2JfbLSa/JPIh0arSffeEbL86A2HxR6P3IMa5ukN9SBwo95B9CaoB4FvNRuDgRikMKIXsuPtJZUIOixYuHAhN9xwQ/E+EsCoUaP44YcfGDFiBCaTiblz57JhwwasVisAqamp/Oc//2Hs2LFERUWxa9euEntCXs6cOUPjxo2Jj4/n999/58cffwRgyJAh3HvvvZw+fZrExEQWLVpUvGeVkpLCG2+8wQMPPADA5s2b6d+/pJ55zZo19OzZk8aNG1NYWMj27dsZPXo0LVq04Pjx42RmZtKgQQOWLl3K+PHjadiwIcnJyXz22WdMmTKFgoICXC4XKSkpzJ49mxkzZhAfH8+pU6dISkqiQ4cOfPzxx0ybNg0hBFu2bKFfv36kpaUxdOhQhg4dypIlSzh48CBnzpyhY8eO3H333aSnp7Nly5aACQogLi6O1157jT59+vDYY4+V+fMBiIqKoqioiKioKMaNG8fkyZO59957ad68OadOnSIrK4v27YNnGqhpMjE9uhnSs6BjYsXv8eWNndCtIYzzuMaPagGrUyB1BQz/Er4cBwN1bscIIXAfz8a15ySiMPDe3I4zwA54v43c+zqwHLKPl9PzSwhEnrPMGY88VgQFPlsOkQYMLRKJSG5E1OC2RLRMxNCqIYYWiTIxqdlRsBkC7DFaTekADovdDkymmg1hjVbT/xwWeyTwHBCPLNo9BNxttJo+1DuOrgQlzGzXbPQBbgcKgFikQOJNYabeuZAtWLCAhx56qMSxqVOn8sEHHzBixAhSUlK44YYbmDRpEtHR0QDcfPPN7Nu3j4EDByKEoFmzZnz22Welxh4/fjxvvfUWffv2pVu3blx00UUAtGnThkceeYShQ4fSunVrevbsSaNGjQC55HXHHXfQt29fnE4nI0eO5K233ioxblpaGrfffrv8YHK7ueKKK5g6dSqapvHEE08wdOhQOnToQPfu3YvfY7PZuO2223jiiSeIiori448/Zvz48WzevJkLL7yQ6OhoLr/8cp577jnef/99br/9dp555hmKioowmUz069ePBx54gN27dyOEYNy4cfTr148XXniB+fPnExUVRcuWLXniiSfK/Xm3atWK6dOn8+abb/Lggw8G/PkA3HrrrfTt25eBAwfy/vvv88wzz5CSkoLb7SYqKoo333wzqAkKwNwRHtsM76XDU5VoXbT+pOzw+/rgkp19ByTBD+PhshUw+mvZk6us1vRuRx7OHcdwbj9G0fZjiMzyHWUuQKqf2HnumHNryR3tgERoxTOc4pmPsWHxc80YR0SrhhhaJ2JomqCaDAafSE3TNvi8flsI4VVntQEO+pzLAIYGGGOqw2IfCewC7jVaTQcDXFMCo9X0DvCOw2JvChiMVlOlNa66CnU1G+2Ag57266XOCTMHKnvj2iYcC3Wzs7Np0KABTqeTq666ihtvvLF4j0gRPqSskD210q7S30beskYa8x6aCg2jS5/PyIHUbyAtCz4aCZPagsgtxLnzBEXbZVJyHzoDgJYQTWT35kT2bEFk9+ZoCaUHfOY3+PdOWDhSehIKAVesBEcRfJ8iHUUCocVFQXSEmvmEEeUV6jos9mlAqtFqutnz2gwMMVpNd/lc0wTINlpNBQ6L/Y/AtUarqdzyfYfF3guIMFpNW/yO9wWcAZYQA6J3iW8vsv9TiQyo2WjiOVePOgUFj6eeeooVK1aQn59PSkoKU6ZMCXZIiipg6QQzf4Bvj8l6soo4kQ/2fXBzl8DJCaRI4rtUuHyF4LVPjtE1exct047IPZ2oCCK7NiX64guI7NmCiPbGcmctm07B7P1g6QXDfRot3nwhTPsO/i+r/rVXUZRJBtKOyEsyslapGKPVlOnz8h3g7zrGfRu5t7XF73hPZB3UcD3B6U1QGoE76jagnqr4gsHLL79c8UWKkGdKW2joqYnSk6Dm7oYCN9zRrexrRKGLBj/tZ/nanXDoDMejYvh1cHcuHtWSyM5N0aL1/Y3odMMt66BpDLw0sOS5q9pC50T4+zZpEKwmSecF64EuDou9A3KPyATM8L3AYbG3MlpN3q2cScgGthXRF2nsEOh+uqU45SYozcbrnqcCeF6zlSjSjUBusJWWdoU5o0ePLnXs2muv5U9/+hO5ublcfvnlpc5bLBYsFgsnT57kmmuuKXX+9ttv57rrruPgwYOYzeZS5//yl78wceJEdu7cyW233Vbq/GOPPcall17K5s2bi+Xvvjz33HNcfPHFrF27lkceeaTU+ddee43+/fuzYsWKgK4Nc+bMoVu3bixZsoR//KO0OYjNZqNt27Z8+OGH/Oc//yl1fuHChTRt2hSr1VosHPFl2bJlxMfH8+9//5uPPvqo1PnVq1cDMkkvXbq0xLm4uDi++EL6ED/99NN8801J4WiTJk1YtGgRAA8//DDr1q0rcT45OZn58+cDcM8995RSI3bt2rW4IPjWW28tlvd76d+/P6+99hoA119/PRkZJXtrDBs2rFhNOHXqVDIzM0k4C/PzIb0ZpFw6jscfl/6YEyZMIC8vr/i9Avi13ZWMM99Pj0alf/caGeK4/6Jr6ZtlRGQVsLfwJIvP/Mbbh/dw+AcXrd6B2XdauPlGfb97Y6aaScuS3YCv8nR48/7u7dm9E+3529hwFgY8Ixs6gvrdC5XfvdrAaDU5HRb7nUiz7wjgv0araZvDYp8NbDBaTYuBux0W+yTk9uQppKK7IlxIEZ0/jSnpLFEuFc2gvJlOA3pQshaqEFkVrP7sVyj8aBkHR/Lk8l15ZBbAmSIpLfflgqgmTGrUlxEJnYk8bCCifxMY1Z57750FQJdGEBkBB3LgpW0wOb/i//UHsqUdU1KM9A8MGHcs7MuW4xrLWG5U1C+MVtMyYJnfsSd8nj8MPFzJYb8FHnVY7NOMVpMLwKPqe5RzrTcqRK9I4n/An0PdwdyXcBRJKOoPQkD3xfID/9vUsq8b9zXsyYK0KRChCZw7jlGwZDvOHcchOoLoER2JuawrES0Da9Y/2As3roVWcbB4DPQpoyuFENI+ae0J2D5J9voqi+d/g0c2w+YroF9SJb5pnew6C3f8BB+MKDtRKmqOYLiZOyz2bsAPyH5QP3gOD0duC400Wk16lgkr326jOmg2SlQsCzMv+J2PAd5D9rPPBK4TZvb5nG+H1Oc/Jczlz9xUglIEG+8H/Z4p0ClAftnugF5L4Pn+gr+4jpC/ZBuutEw0YxwxKV2JHtUJQwAFnj8/n4QpqyGrCOYPh8ltS18zPx3Ma+Bfg+HO7qXP+3K6ANp9Io2C3x+h73utDI9sgue3wgsD4K+9K75eUT2C1W7DYbG3Qgoi+iMn+L8A/zZaTYfLfaMPuhOUZmMMMB1oB5T4XyPMVNgxyOM8sQufimVgujCfKwjTbPwJ6CvM/FGzYQKuEmau8zm/CNlW/ieVoBShTkaO/KAvq0/UnT8KDq/NYN7p7WgHT6M1iSf2ip5Ej+iAFlU5YeyhXJmkNmbCs/3hod7nRA4n86HHYimA+CG1bAm5L/dvhNd2yOR6QYNKhVIhfZfAbw7purFzshJj1Db1vh+UZsOC7JabCIwGTiA3uwaiv+J4CLBHmEkXZgoBb8WyL5OR7eUBFgLjNJtcWtdsTEH6O23TeT+FIqgkJ8BlreDd9JIOP8LlxvH9Psy2L/jvljVEFDqJu2kIDf9+JTFjO1c6OQG0iYfvUuC6C+Ss7fofIM9TcXvfRrnP9c5F+pITwD3dZQ3XK9XyEyjNgRyZnAYmwe6smmlPogg9HBb7nQ6L/foAx693WOx/0juO3hLv+4E7hZnpQBHwsDAzAJiPXGPUQ6CKZX8voOJrhBkncAZootlIQPr+/a28G2iadqumaRs0TdvgdFZYD69Q1Dp/6CQ/lFcflYW1Bd+lk/XIMpj3I240jl4/jMTnJxAzoiNaZPUcF+Ii4YPhcgb1wT4Y9RW8mwa2dPhrL+hdia7pyQkws4M0vz1Zg4Ukn3sEkO9cJKX4c/fU3NiKkOIeSn7ee9kH3Kt3EL11UB2BFZ7nBciNLoA3gNXAQwHe40+gibz/+mJZ1/wNeFWYydZsZd/AY9/xNsglPh0xKRQ1jhAC98kcXAccXLHvNPbfHLT72cGZLLnkbGjfmAeHDue39m1Yd2nNrm9pmjR87dFI7jlZ1kp/v0erYAL7QE9Zy/XmTniyErZN5fH5IejYQFo4zeggE+jrg5VisB6STGknc5ATE91tUPUmqEzk8h7IYq7eyArhJshWHHqosGLZ55oMTyv5Rkjd/VDgGs3Gi4ARcGs28oWZN3TeW6GoVYo2HaJo61FcBx24Djogz9NtUIOBDRNZG5/E5HEdie/WlNUNm/POSg1bBWKF6nBVO1ibCA/+An/rC7FV8HrpaZStP/61Ex7oBfF6Py3KINcJ3xyFW7vIRHpzZ3hrFyzYC7eXU6SsCEuOIsUR+/yODwRO6h1E76/c98jOub8BHwGvazYuA8YBX+scYz3QRbNRZsUysBiYBawDrgFWevz/irVEmo2ngGyVnBShgBCC/M+2UvB/2yAmkoi2jYge1p6Itsbix+9nI5n1JczrDTd2hjdWQbMYmFbLdkJ9G0sH9OrwYC/Zk+q/eypW/1XEqqOQ74IrPAv7A5OgX2OYt0clqHrIB8DrDos9B7nKBjAGeA14X+8gehPUnUgHc4DnkRXFlyCTVenS8AAIM07NRomKZWFmm2ZjNrBBmFkMzANsmo09yJmTSe83olDUNcLtJm/+LxSu3EP0iA7EWQajBVAhDG0ql9n+lwZjW8KSDHikN8SEgYPl8Oayz9U/tsMfu0J1tsmWHoKEyHNu7N5Z1F3rYfMp6F8LNVeKoPEk0AH5ee/t92JAdsF4XO8gFcrMPUtttwKfCXOpJbmQRcnMFbWJcLrIffsnin4+QMyE7sRe269ch+8XtsLDm6TH3WcHYe9V5RfLhhKLD8Lk1VKAMb1DhZcHRAho/wkMagKfjj53/HQBtFoIt3SBfw2piWgV/gRTZu6w2LvgUwdltJoqJYvR6ySRA/QU5oCbXiGJSlCK2kIUOMn51w84tx4l9tp+xF7eo8L3HMqVNVFuIZPUwlF1EGgN4RbQewnEGOCXK6pWt7TlNPRbCnMvgpu6lDw383tYdhgOT5VKREXNEs51UHp/HX5EujuETYJSKGoDd3YBOa98h2vvKeJuHELMyI663tcmHlJawZeH4a5aFEfUBgZNKvpuXAcrjsh29JXFKy+/vHSTaW7qLGXxnx6Uyj5F+OOw2F8v77zRarpbzzh6E9Q7wMseq6GNQImpiTDzi85xFIqwxX06l+yXVuM+nk38nZcQPUi3WhaA2f2kvHpk81oKsBaZ0UEWAL+yo2oJaukhGJQEreJLnxvdUkrP5+5WCaoe4V/YEAV0R+Yc3flCb4L6wPP1lQDnBKphoaKe4zp6luyXViNyCkn4yyiiepTRd70cBjeVj3AkJgLu6AqP/yo9BHsa9b83swB+PCktnwJh0KS68bHNsmNwIN9CRXhhtJrG+B9zWOyxSCHc93rH0avJ6VDOQ98ah0IRpjj3nSL72W+g0EWDh8ZWKTnVB/7YVdZTvabLh/ocXx6S+1hXBlje82LpJBPVf5WzRL3FaDXlA88iW27oQu8Mqj2w1mM/VIxH4Xcxam9KUc8QLjeutEyKfjtCwde70BKiafDAaCJaNgx2aEGjaSzc0BHeS4dnB+hvlbH0ELSIlQq+smgTDxNaS+eKv/WrnpxdEdI045wTUYXoTVCrgFaAv7VjI885tcSnCHvcmTkUbT2K87cjFG07Jt0gDBqRPZoTf/NQDI0DbKCcZ9zTA97eLR0gHu9b8fVOtxSGTGkrZ0jlcXNnuOpbef2VldveU4QYDov9Pr9DGjKHzMSvOWJ56E1QGqV980BaHSkttyIsEUUunLtOyIT021Hch84AoCXFET24LZF9WhHZs4WunkznCz0ayZnOmzuly0RFxcZrT4CjsPzlPS9XJMuZ1rw9KkHVA+7ye+1GdsH4H9LsQRflJijNxmLPUwHM12wU+JyOQHryrdV7M4UiFBBON4Wr08hfvBVxtgAiDUR2a0b08A5E9W2FoXXDcotuz3fu6wmXrYAF++TeUXl8fgiiDLLtSEVEGWBWJ+lacTQPWup1+VSEHEarqUb0mBXNoDI9XzXgNJDnc64Q2cr3nZoIRKGobYQQFK0/SP6iLbiPZRPRrRmxf+hOZM8WaDGqQlQv41pCHyO8ugNmdSy/cHdphpTVN9Q5Cb2pM7y4Te5zPdirZuJVBB+HxR4JxBqtJr3tmQD9ThJPAi8Lc/gs5yknCYUvzt+Pk/fRZlzppzAkNyLumn5E9mulZkpV5H97PIW7l8K4MmZHe7Og42fwyiC4t6f+sUcuh2P58Psk1W23JqhLJwmHxT4OaGK0mj7yOfYQ8BRyQrQCMBmtJoee8XRpZYSZv/kmJ83GSM3G5ZqNSlRDKBR1jyvDQfar35H9wkrcjjzibhpC4uxUovq3VsmpGkzvAM1j5SyqLD4/JL9Wdj/p5s6w6yz8oLrthiMP4dPvyWGxDwGeA2zAg0A/KiEzLzdBaTbu1GwlB9NsLEXapy8Fdmg2KjYiUyjqGPepXHLn/UTW48tx7jpB7LR+NHzhCtm51qA0zNUlNgLu6CaT0O9nAl/z+SHokghdKqnMv6a96rYbxvQBvvV5PQ1Ya7SabjFaTa8AdwOT9A5W0f/UWcAB7wvNxlVAKmAGLgT2UgnrdIWitnHnFJL38a+c/evnFK7bT0xqVxq+dCWxV/RAi1b7TDXJH7tIA9lAhbs5RbL/U1XUePGR0vLo4/1wprD6cSrKx2Gxj3dY7DsdFvsez3JcWddd47DYhcNiv7Cc4YyULEe6BPjS5/V6QIemU1JRguoEbPJ5fTmwWJh53+O/94gnAIUi9E6FMAAAIABJREFUqIgiF/nLd5L14FIKlu0g6sJkEp+/nDjTAAwNYoIdXr2keRyYPYW7J/NLnvvmKBS4zzUnrCw3dYY8l1QKKmoPh8UeAbwJTAB6AtMdFnupHUOHxZ6InP38VMGQR5B5A4fFHgMMQDag9ZIIJdTg5VJRgooBfFUXw4DvfF6nA2FofamoLwi3oPDH/WQ9soz8BZuIuKAxDZ5KJeG2YUQ0012wrqgi9/SQiWTO7pLHl2ZAYhSMqOKnw6Ak6NkIFiqPmtpmCLDHaDWlG62mQsAOTA5w3dPAi0B+gHO+fAG86LDYxwJ/R9bJ+nrv9QV0L95WlKD2A4MBNBvNgR5IabmXloAuNYZCUdMU7ThG9uyvyH1rHcRGkXD/KBo8MIbI9o2DHdp5Qy8jpLaGN3ZCgadvqhBy/ymlFURX0WNG0+S4PxyHPGfF1yvKJVLTtA0+j1t9zrUBDvq8zsBvCc5hsQ8A2hqtpqU67vUEMomtAG4EbvEkPi83Al/rDryC8+8Cb2o2+gCjgR3CzEaf8xcDv+m9mUJRE7gOOsj7+FecW46gJcUTf8tQooa1V+KHIHFfD0j9Bj7cBzd0gs2n4XBe1Zf3vIxrKVWCa0+ULWVX6MIphChr3yiQlLW49shhsRuAVwGLnhsZraaTwEiHxd4IyDZaTS6/S6ZRclWuXCpKUC8BCcBE4Chwm9/5S4AP9d5ModCLEAKRXYj7ZA7uE9merzm4j2fh3H4MYqOIvbYfMZd2Ravqn+mKGuGyVtCrkUwm5o7lNyesDCNbQKQm97NUgqo1MoC2Pq+TgcM+rxORjkGrHRY7yFWzxQ6LfZLRatpQ1qBGqymgttNoNZ2qTHC6CnXDEVWoG9oIpwv36TzE6Tzcp3Jxe7/6JCTyS67taAnRGJolENmjBTFX9FDihxBi3m64+UdYeRk8vEm21/j58uqPO/xLKHTXzFjnK+UV6nocHnYB44BDSJXdDKPVtK2M61cD95eXnGoSpbtV1BrCLXAfOoNz5wlch8/gPpUrE9LpXOmB509MJIamCRiaJhDZvTmGZg0wNE0golkChmYN0OKi6v6bUOhiZkeZmB7fDD+fhKf61cy4l7aCp3+D0wXQWP09UuMYrSanw2K/E1iO9Ff9r9Fq2uaw2GcDG4xW0+LyR6hd1AxKUWMIpxvXgdM4d57AufM4rt0nETlyf1RLiEZLisfQOA6D56vWuORr4qKUu0MY89Sv8Lct8vmGy8vv/6SX74/ByK/gk1FwVbvqj3c+UpdWRzVNnc6gNBvjgX8iM/VcYeYFv/MxwHvAIKRR7XXCzD7NxmXAC0D0/7d35uFRllf//3yTACKoQWW3ICguuOJC3SpVULu4vLTWYutYtFXbam1dX619rT+v+rq+rW1dCmpFp1pqXbHWFdxaFXGhKLgBgogoIotC2JKc3x/3M8lkmMATmMxM8pzPdc2VZzlzzyFk5sx93+ecL6FJ7QWWYlIxfXeaYmbY0lXULficuvc+DbIV7y2CNWFPtKLXFnTYdzuqdu5O5U7dqdi2iwefds5PdoIr34RtOsGQrQsz5pe3hS5VYR/KA1TyKFqAUppMQdgRhI25KUozwVLMyDL7IbDEUuyoNKMIefTfBRYBx1iKj5Rmd8J0dBO3YJ0NYWbYslXUf/IFdZ8sp/6TLxqPFy6H1dEekaBiu2o6HjqQqp26U7VTdyqqXSshafTsDL/bDzpXblicMC4dK0M39IkLCjOe07Yo5gxqKDDTUswGULqhICw7QB1H6HoLcC9wg9LIUk26WUwHNlOaTpaKX5HsxKN++WrWvj6ftS/Po/a9T5smKlQq7Av13IKqXXpQ2XMLKnp1pXLANi7q5wDw050LP+aI3nDeqzC/JkjDO8khVoBSmkubuWWEoqyZwGOWaqIXlUu+grAvN2djKWqVZhlBtXdRls23gdfzBaeoAO10gI4d/QMzLvXLV7P2tfmsnTKP2hkfQ51R0b0LHQ8eQGXvLajouQUVPbtSsU0XVOm1Rk5xGd4r/Jy4INRZlZp5K6BPZ/C3QusTdwb1HaAfoSYqkyPfh9DG4lNCHv1CpRmWmSHlYb0FYXFslGY3wrLfkflewMzGAmMhJEk044dD80Gp01E702FoPyr7d/M9I6cs2KMbbNsJnvq49AHq/S9glwmhOPnKfUrrSxKIG6D+DzgJGG0pPgRQmu2APwN/AR4B7iFUHOfr4wQbLgjLtvlQaaqArYDFWa/3AHCypZgV02+HKN370+XUzV0SHu8vpvadhR6UnDZBhcIsauKC0EaplH+iV08PdVm/fxt+tgv08SXHViVugPo1cFwmOAFYig+V5kLgQUtxZ6Qb9dB6xpgCDFKaAYSCsFHA93JsJhAkPl4EjgcmWQqLhBEfAS62FP+O6XMisdp66hd8Tu2cxdR9sDQEpA+WNO4lVYqKPlt5UHLaFMN7w9/mwtufw65blcaH+TVw+6zQIeOJj+CKN+DG3E0Kp6DEDVA9gc3yXO9EYzfzT4Bmv09Ee0pNCsIsxXSluRx4xVJMAG4D0kozkzBzGhU9/SxgR+B/lG7QnzrSUiROc9PMsBVrQtuf7DZAmcfC5VBbH4w7VlLZr5qOB21PZf9u4dF3K9TBWwM5bYsRWftQpQpQ/zcD6gxu2B+umQFj34PzBsPALUrjTxKIVairNBMIS2+nQ0Oz2H2BMcA8S3Gc0hwL/MZS7NlazraEtlqoa6trqV9SQ/3ildiSGuo/y2oD9NmK5lsARR0YKnp0DYGoXzcqenX1BqpOu2HgA7BnNTx4WPFfe9Eq6H8/fLs/3HlwmE3t+CCc0B/uKHNFvCQU6v6IUEA7Gch0p60AngBOi86/AM4vqHftGFtTR+2sRdS+vZC69xc3tAHKdF7IRl07hq4L3btStWvPxvY/UVDS5p6x6LR/RvSCe+aGBYKqIn/vuv6toHt18e7hvO/mcNbOYVZ14W5BdsQpPC1qdaQ0OwM7E7Lt3rIU77aWY5tKuc2gbG0ddbM+o/btheExc1F4p0lU9N0y1Bdl2v5svTnKbgnkUuWOw9/mwKjnYfLXYei2xXvdZWvC7GlEb7h3WOP1Ratg4IOhm/t9w5p/fqlp9zMopTkOeMRSvAO807outR/q5i9j7SvzooD0GaytA0Flv250Gj4oFLvu1N2LXB0nBodH+1BPLShugLrpHVi2Fi7Zo+n1bTeD83aFy6bBlEWwfxF9Sgpx96BqgBrg70DaUrzQ2o5tKqWeQa2d9hEr/vAvqKunsl83qnbp4QHJcTaRvf8Rev1NPKI4r1dTC9vfD/ttA/8cvu79z9eEWdS+W8PjI4rjU0tp9zMoQhbf8YS08OeU5gPgLuAv0azKyWLtax+y4qYXqOy7FV3OOdT70jlOgRjeC258J8jAdy7Cyvct78Gnq+GXe+S/v2XHsC91/qvw7CcwrGfr+5QkWiy3oTS9gRMJwWoI8KqlGNoKvm0SpZpBrXn5A2rGvEhl/63pct4wny05TgF5dD58YxI8OSLsCbUmq+tghwdhh67w7FHN262sDRl9A7rC80eVtpA4H215BtXiXBhLsQC4AbgSmEZIN3eANS/OoebmF6ncYRu6XvBVD06OU2C+0iPIwD9VhO7md84O6eS5e0+5dK6CS/eEf38Kj+b2xnE2iRYFKKU5TGluJRTl3gq8DpTpymtxWf38bGrGvkTVLt3peu4wV391nFagawc4sHvQh2pNauvhqjfD3tMRMWZqp+4IA7vCJZHcvVMYYgUopblWaeYBjxE6R5wB9LIUp1qKp1vTwbbA6qdnsvK2l6narRddfnEo2syDk+O0FsN7waufweJWFNv52xyYvRwu2T3ekl2HCrh8L5i6BO6d23p+JY24WXwvEJrCjrdUaN5a7hRrD2r1k++y8q7XqNqrD13OPBh19DZCjtOa/HshHPJ4qD36Viuo7NYb7PFwKPacdkx88cW6etj7kdBMdvoxxS8mbo52vwdlKQ6yFDflC05KJ3eJb9Wjb7PyrtfosO92dPmZByfHKQZDt4WuVa2nsvvQPJixLGTutUQZuLICfrM3vPt52L9yNp2NStRUmr7AKQSJ9n6E5q+JYtWE6ay6/w06DO3H5qcfgMrl65LjtHM6VMChPYM+VKExC13KB3YNffZayrHbwdBt4LL/wPcHQKfEfTIWltifqkpTqTQjleYRYA4wEriZ0GU8Uax+6t0QnA7ans3P8ODkOMVmRK8wU5lX4FX8JxbAq4vhot03bolOgv8dAvNqYEzZNoJrO2xwBhX13/sRcDJBQfdugqJtylLMaF33yo+1Mz5m5d2vUzWkL5v/aKh3C3ecEjA8yqyb+DGMLqDK7hVvhEawJw/c+DGG9w5tmS6bBkf2gV1KJA8Sl6Wjx38N+D1hJezW6nGjrsq5/2PgTEKj8OXA6dXjRhXls3+9n65K8zzwElANnGApBlqKXxXDsXKkbuEX1Nz4AhW9t6TL6Qd4cHKcErF7NfTYbNP2oczgk5Whj969c+HSqfD8Qrhg8KYvzd1yQFiK/NpEWFCzaWO1JktHj68EbgS+DgwGTlw6evzgHLO7q8eN2qN63Ki9gWuA3xbLvw3NoA4kOH+LpXizCP6ULbZyLSuufx6ALj//itc5OU4JqVCYpTz18YZl4FfVwT8+hDeXwgcrwmPuirA8uLq+qe0e1XDaoE33b+AW8M/DYdgT8PVJ8NyRoS1SGTIUmFk9btRsgKWjx48HjoPG1bHqcaM+z7LvAhSt0mtDAWo/gt7T80ozh6AJ9dfWdqrcsHpjxZgXqf/4C7qc/1Uqe3QttUuOk3hG9ILxc+CtZTA4jx7T28tCL707ZsNnq0PaeO/O0L9raO468kvQrwv07xJ+9usC1R0L16po321CKvzRk+Bbz4aAVaJE3ypJr2SdjzWzsdFxX2Be1r0PgXWE7JeOHn8mcC7QETi8tRzNZb0BylJMBc5UmvOA7xCy9q4hLA1+U2kWWIolre9maVn1wBvUTv2IziftQ4fB3g3SccqB7H2oTIBaVQf3zQ1y7M8tDG2RRvaD03YMjVyLHSCO6gO3HQg/eAFOeQHSh7Qsdb1A1JrZfs3cy+fNOjOk6nGjbgRuXDp6/PeAXwE/KKB/zRIrzdxSrALSQFppdiQkTZwD/EZpJlmKr7eijyVlzeQPWP3wDDoOG0jH4QWY+zuOUxC27wo7bBH68o3oHc2WZsHiNeH6VUNCAkXPEosJnLxD6On3y6nQZ3O4try6l34IfCnrfDtgfR0FxxOyt4tCi+ugLMVM4CKluQQ4Gji14F6VCbVzFlNz22Qqd9qWzql9Ubm1KXachDO8F9w6EyZ8GJISRn4JTh8Eh/UqyUylWS7aHeavhOtmhCzBX+xaao8amAIMWjp6/ABgPjCKoFTRwNLR4wdVjxv1XnT6TeA9isRGK6pYijrgoejR7qhftooVf3gede1El7MOQVVecec45cboHWDakrCMN3og9ChT6TUJfr8ffFQD574CfTrDCduX2iuoHjeqduno8WcBjxPSzP9cPW7U9KWjx18OvFI9btQE4Kylo8ePANYCSyjS8h5shB5UW2FTevHZ2jqWX/M0dXOX0PWSEVT171Zg7xzHSSIra+GIp2DKZ/D4cPhqr9Z/zbbci6+oAUppmhSEWYqrcu53ImQK7gt8BnzXUsyJ7l1MSNKoA862FI+v77U2NkCZGStvn8Ka52az+U8PouPQVuhG6ThOYlm8OjS7/agmCBzu0crff9tygCpapanSrFMQpjS5BWE/BJZYih2B3wFXR88dTFgb3Q34GnBTNF7BWfPUe6x5bjadjhnswclxnIKzdSd4bDh0qQqFvB8UX/i7zVDMVghDgZmWYralWEPIBjkux+Y44I7o+F5guNIouj7eUqy2FO8DM6PxCorVG2unzqdqSF82G7kBGU3HcZyNpF8XeHQ4rKyDFxaW2pvyZaOTJDaCOAVhDTaWolZplgHbRNdfynlu39wXkHQ6cDpAx44tL9tWhehyzjCoq0fllALkOE67Y89uMOu/oFunUntSvhRzBhWnIKw5m1jFZGY21sz2M7P9qqo2LvaqqgJ1KmbcdhwnqXhwWj/FDFBxCsIabJSmCtgKWBzzuY7jOE47opgBagowSGkGKE1HQtLDhBybCTTm2B8PTLIUFl0fpTSdlGYAMAh4uUh+O47jOCWgaAHKUtQCmYKwt4B7LMV0pblcaY6NzG4DtlGamYTGhBdFz50O3EPosPsYcGZUKOw4juO0U7xQ13Ecpx3jdVCO4ziOU2A8QDmO4zhlSbtd4pNUD6zcyKdXAbUFsGmJXTnSln0vJP57aMR/F4G29BnR2cza5GSk3QaoTUHSK+sR+Ipt0xK7cqQt+15I/PfQiP8uAv4ZURzaZFR1HMdx2j8eoBzHcZyyxANUfsYWyKYlduVIW/a9kPjvoRH/XQT8M6II+B6U4ziOU5b4DMpxHMcpSzxAOY7jOGWJ60q0AEk9gf2j05fNrN1IjUmqAI43s3tK7Uu5IGlLst4jZra4hO44TuLwGRQgaYCk30q6X9KEzCPH5gRCB/XvACcAkyUdn2es7SQ9IOlTSZ9Iuk/SdsX5l2w8ZlZPaOabeCSdIekTYBrwavR4pbRelQ5Jh0g6JTruLmlAqX0qFpKOkdQ/6/xSSf+JPiMGRNdOkzQoOpak2yV9LmmapH2ynhvLzmnEkyQASf8hdFJ/A6jPXDezZ3NsjsjMmiR1B54ys71yxnoSuBtIR5dOAr5vZke06j+iAEj6H0L3jb8BDZ12kzZzkPQecKCZLSq1L6VG0q+B/YCdzWwnSX2Av5vZwSV2rShImgYcYGY1ko4GfgucCAwBvmNmR0l6ExhiZmslfQ84Dzgysvm1mX0lGiuWndOIL/EFVpnZHzZgU5GzpPcZ+Weg3c3s9qzzcZJ+sckeFodTo59nZl0zYGAJfCkls4CaUjtRJowkfIC+BmBmH0naorQuFRUzs8zfwreA28zsVeBVST+Nrtea2dro+GjgTjP7DHhK0jVZY8W1cyI8QAV+H31TfAJYnbloZq9l2Twm6XHgr9H5d4FH84y1SNJJWXYnEoJZ2WNmiVm62QAXAy9ImkzTv4ezS+dSyVhjZibJACS1SdmGTUCSuhK+sAwHbsq6t1n0s15Sb2BJZHNFlk3nrOO4dk6EB6jAHkAKOJzGJT6LzsOJ2QWSvgUcAggYa2YP5BnrVOAG4HfRGC/QODMpS6KAKjNL51w/DVhhZneXxrOSMQaYRM6Sb0K5R9IYoDr6ezgVuLXEPhWT64GpwOfAW2b2CoCkIcCCyOZSwh5lJTDBzKZHNsOA2VljxbVzInwPCpD0NrCnma3Jc29HoKeZ/Tvn+qHAfDObVSQ3Ww1JrwOHmtkXOde3BJ42s31L41lpkPSCmR1Uaj/KBUlHEPZKBDxuZk+W2KWiIqkv0AOYatEHZjQT6mBmH0TnVcAWZrYk63ldCJ+xy7OuxbJzAj6DCvwHqAbypY1fD/wyz/Wa6N4xANEa8mwz+1O2kaRzgF5m9t8F9biwVOYGJwAz+1xSh1I4VGKelnQ68DBNl/gSlSySIQpITwJIqpT0fTO7q8RuFZNPCQH6+9FS5wzgbjNbnWWzNXCmpN0IKyczgJvM7JOcseLaOXiaeYaewNuSHs+TZr69mU3LfUI01d8+69LR5O+p9Xvgm4V2uMB0yLe3EG2GdyyBP6Xme0T7UCQ0zVzSlpIulnSDpCOjtOizCEtRJ5Tav2IhaTAhiHwV+AD4MDqeHt1D0sHAlOgpdwJ/iY4nR/doiZ3TiC/x0bAGvA5m9qykmWa2YzPPa7gnabqZ7daMXbP3ygFJ5xM2bX9iZnOia9sDNwLPmNm1JXPOKQmSHiJs5r9I+NvoRviy8nMzm1pK34qJpInAVbnLmpJGAJeY2WGSXiK8d17PsdkbGGNmX47OY9k5jfgSH03rnfIwRdJpZnZL9kVJPyR8s85QI2mQmb2XYzeIjVf2LQpmdp2k5cCzUcaSEeqgrjKzm0vrXfGIkmCaxczuL5YvZcBAM9sDQNKtwCKgX76l4HZO33x7bmb2lKQ/Rqdb5gadyGZqTkp+XDsnItEBStIXhA/jdW4R6h+2BH4BPCDp+zQGpP0I3yZHZj3nUuBRSb/Jsbs4GqOsifbO/hQFKCXwgwii/cRmMCBJASpTr4OZ1Ul6P6F/ExWSOuXsNyFpMxo/PyWpW3biQ3Rxa5puo8S1cyJ8iS8mkg4Ddo9Op5vZpDw2uwMXZNm9CVxnZm8Ux0vHKQyS6mjsJiJCnU4NTb+8tXsk/Qo4ADgrZ/n7D8ArZnZ5lFBzGnA+UUEzsC9wNfBnMxsTPS+WndOIByjHcZz1ECWHXAhsHl1aQfji+ccsm6Mjm+zsvGvN7OGcsWLZOQEPUI7jODHI7BMldKmzJPi6p9OApA6SzpZ0b/T4WULroBwHAEnXZ52emh2cJI2Lft6Tde3qnOc/kXUcy85pxANUhKSeko6OHj021qaNczNhTfym6LFPdC1ReKB2sjg06/gHOff2jH4OyrqWq1rQPes4rp0TkegsvgwKWk/XAs8QNoH/KOkCM7u3JTaR3QDgZ4Qi3myxu2Nb919REPbPkQ+ZpCAzkjRuBjrQ2Bg0FV37Uck8KjFKrnijmjnOZn37JNbMcUvGSCweoAKXED6cm2g9Afe20AbgQYK21MO0vUajdZJ2yPQXlDQQqCuxT6XAA3WEpDOAywm1fJkP0SRJsFRI6kZYbcocZwJVZfRzc4XmsRVA5+hYNGY/0kI7J8IDVCCO1lNcPag42lLlygWEPnSzCW+a/sAppXWpJHigbuR8YDdLrnjjVoS6xkxQypbgyQTsBQQhQ4CPs44z57TQzonwABWIo/UUVw8qjrZUWWJmE6POFzsT3pBv5xYoJgQP1I0kWrzRzLaPYXNYzLFi2TmNeJp5hJpqPT2XT+spps2VhD2LWWRpS5nZ4bm25YJcD2odJHXCA3VG9+h2wMUbnaKT6AClGFpPcWxyrjerLVWuyPWgAA/U+ZD0MvAvcsQbzeyOkjnlJIakp5lfD+QrustoPcW1ySajLdWWaFYPipDNlhTOIyS55PK36F4SqTWzc83sdjO7I/MotVNOMkj6HlSzWk9Rv624NtlktKWm0HRJpJzTzDtI6mJmK7IvKnl6UC7cuC4u3hghaS/gK9Hp82a2TmanpGNprJ16trkWRnHtkk7SA9Rm67nXuQU22fx6490pGbcB90rKpwd1W+ncKjoeqNfle9HPi7OuJSnNHABJPyc0es10tP+LpLE5/fiuBIYCGbXhsyUdZGYX54wVy87xPai/ApOa0Xo60sy+G8emeB63HpJ+TPgQSrIelAs3OnmRNA04MPPlRUGB+kUz2zPHZm8zq4/OK4HXs21aYuf4DCqO1lMsPSjF05YqW1wPyoUbs5GLN+YimtbC1ZG/s0Q1kFn+3Go948W1SzSJnkFliKn1tEEbp/2Q5EANIOn29dw2Mzu1aM6UAZLOJfTiy5SW/Bcwzsyuz7I5EbgKeJoQvA4FLjaz8TljxbJzPEA5juPEQtI+NK2DXEe+XVJvYP/IZrKZ5e0QEdcu6XiAchzHaQYFafcfAzsSasFuM7PaHJsewC+zbK6MSjRyx4pl5zSS9DooB5C0v6ReWecnS3pI0h8kbV1K3xynxNxB2HN+A/g6cF0emzsJe5V/JCQZNdeLM66dE+EzqAhJPQlTboCXcxrDxrZpiV25IOk1YISZLY46ZIwnSIbsDexqZseX1MEiIWl/YF5muUXSycC3gbnAZUms/Uk6kt4wsz2i4yrC+3mfHJupZrZ31vlruTYtsXMa8RkUDVpPLwPfAU4AJks6vqU2LbErMyqzPny/C4w1s/vM7H8IyxFJYQywBhpaWV1F+Na7DBhbQr+Kjs+qG1ibOchd2stCkrpJ2jr63VTmnLfUzonwGRSgoPVzRK7WU7YmUBybltiVE5LeJNRl1Ea9BE83s+cy98xs9/WP0D6Q9J/M/5OkG4FPzeyy6LzJt9/2js+qA5LqCMty0KjbVENW+YikOYQ+hfnSzs3MBkZjxbJzGkl6HVSGQupBxbUrJ/5KqP1ZRBCmex4amukuK6VjRaZSUlX0TXk4cHrWvaS9V/LOqoH7JE0toV9FxcwqY9hsH3OsWHZOI0l70zVHIfWg4tqVDWZ2haSJQG/gCWucVlcQvjUnBQ/UjXiwdkqOL/FFqEB6UC2xKzeiYuTdCB0UZpjZ0yV2qehIOoDGQJ1pa7MT0NXagOhkoZB0CfANYBHQD9jHzCwK1neY2cElddBJBIkOUCqgHlRcu3JEUl9CE8xVNMpb70NYbx9pZvNL6F7R8UAd8GDtlJpy3xtpbQqpB9VS3ahy4gbgZjMbZkH75xwzGxZdv6nEvhUNSX0lTQYuI3Tr3hG4TNLLURBPFGb2ErAUOEXSmZIOM7N3kxqcJPWUdHT06NGMzSGSTomOu0sa0IzdXpLOih5lm0BVapI+g2o2Qy1T/xDHJu5YhfO8sEh6x8x2bum99oakB4CHzGxczvWTgW+b2XElcawE+Ky6KVH5yLXAM4TfxVeAC8zs3iybXxOKenc2s50k9QH+nrscqnWlO0YSklD+iNOEpG92FlIPqqW6UeVE3kwlSRXN3WunDDazkbkXzezOaE8mSWRm1eOyL0bB+iYgMcE64hJg/9zyEeDeLJuRwBDgNQAz+0hBSyyXHwJfzlo2vRp4kdBhwski6Ut8UySdlntRQevp1RbYtMSuHHlY0i0KGjdAg97Nn4B/ls6touOBupHBucEJQrAGdim+OyUnTvnImigD1qDhPZSPuNIdiSfpM6iC6UG1wK4cuRC4EpgraW50rR+hD9kvS+ZV8XlY0i3AL6ypMN3vSFagBg/WucQpH7lH0higOvqyeipwa56xbid0mMmW7kiScnVsEr0HlUEF1IOKa1eOSOpMSAwQMNPMakrsUlGR1IHqGIwNAAAFbElEQVQQqEcT+u9BVqA2szUlcq3oSPodoaFpvmC9yszOLqV/pSBmKcoRwJGRzeNm9mQzY21QusPxAOXgTVJzSXqgBg/WGTalfERByn2Umd0VnW9QusNpStL3oJyAN0mlsUGqma00szcIfef+quQ1SMXM1prZ+cCXCEHqFKC/mZ2flOAUscHyEUlbSrpY0g2SjlTgLGA2oWF0hjjSHU4WPoNyvElqhLxBagM+qw7ELEV5CFhCyMQbDnQj7D3/3Mym5tpHx3mlO5ymJD1Jwgl437WAN0htZAwwAprMqjPBeiyQlGAdp3xkYFbguZWoPZSZ5c68mkh3SJ64tyGS9OHjNI83SQ14oG7Eg3VgiqTTzOyW7Is55SPZgadO0vt5ghPAXpIyEu8COkfnDdIdreB/myZpbzonD97NvAEP1I14sA7EKR+JFXjiSHc4TfE9KMfJQt4gFfBu5rm05fKRtowHKMdx8uLB2ik1HqAcx3GcssTroBzHcZyyxAOU4ziOU5Z4gHIcx3HKEg9QTmKIFE5vkjRH0mpJn0iaGDX4JLp+fgFfzyQlpaDVcQpOkuoZHOc+YHOCYNxMoAcwDNgm7gCR3ITMrG6Dxo7jbBI+g3ISgaRqgkz3RWY20czmmtkUM7vOzMZLegboD1wbzXwyonOjJS2X9A1JbxKa6u4aNZZ9QtIiSZ9L+pekA7Neb050+PdovDlZ946R9KqkVZLel3SFpI5Z93tKmiBppaS5kk6R9Kaky6L7f5b0j5x/X4WkDySd2wq/PscpCR6gnKSwPHocG8ke5PIt4EPgckLtT++se5sBvwLOAAYTGqZuAaQJQW8oMBX4p6Rto+fsH/08LRprfwBJRwF3ESTVdyOI2h0P/G/W691BCJaHE6TVT4rOM9wCfE1Sto9HAL0inxynXeABykkEUcue0YQP+6WSXpR0naQvR/cXE6S3vzCzjzNdvCMqgZ+Z2b/N7F0z+8LMJplZ2szeMrO3CS2hVgFfi8b7NHru0mi8zPklwLVmdruZzTKzp4H/Bn4cyTTsDBwFnGFmL0bdsEcTliYz/5YXgbeBH2T5eCowIet1HKfN4wHKSQxRs9M+wDEEue6DgJckbUjWvpYwQ2pAUg9JYyS9K2kZQTOoB6Et0PrYF7gkWjZcLmk5cDfQhTAD2gWoB17J8nse8FHOOLcQNJqItKqOw2XDnXaGJ0k4icLMVgFPRo/LI3mEyyStTzxudZ6kiDuAnsA5wBxgNTCR0ER0fVQA/w/4e557nxIajMYhDVwt6RBgCKFn3hMxn+s4bQIPUE7SmUF4H2xGSICI23H6EOBsM3sEQmIDTfetIMgw5I73GrCLmc3MN6iktwhBbF9gcnRtO8LMr4FIVPF+wtLeEGCcZxY67Q0PUE4ikLQNYdbyZ2AaYUluP+BCYKKZfR5l2n1F0l8Is6ZF6xnyXeAkSZMJy3PXEAJcNnOA4ZKejcZbQkjC+IekucA9hOXD3YGhZnahmb0j6XHgT5J+QtjXupYgMZ7bOPMW4DGgA8kREHQShO9BOUlhOfAS8HPgWWA6IXPuboIgH8ClwJeAWYTltvVxKtCVoBE0nhD45uTYnAccBswDXgcws8eBb0bXX44eFwEfZD1vNCGj8BlgAiHrbyEhWGXzTMbOzGZtwF/HaXN4N3PHKXOi1PWPgBOjRI/M9c7AfEKG4V2l8s9xWgtf4nOcMkPS4YQ6qzcImYFXEJIgHovuV9CYoLGS/AkXjtPm8QDlOOVHB+A3wEDC3tNk4NCMaCAhlf19wvLeKWaWu/flOO0CX+JzHMdxyhJPknAcx3HKEg9QjuM4TlniAcpxHMcpSzxAOY7jOGWJByjHcRynLPn/pkpB9gpXE3YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels = []\n",
    "from random import sample \n",
    "for k in merged_strats:\n",
    "    samp_keys = merged_strats[k][-1]\n",
    "    converted_samp = \"\"\n",
    "    for idx, curr_strat in enumerate(k.split(\" \")):\n",
    "        if curr_strat == \"UNK\":\n",
    "            if idx == 0: converted_samp += \"SOS\" + \" \"\n",
    "            else: converted_samp += \"EOS\" + \" \"\n",
    "        else:\n",
    "            converted_samp += all_strategies[inverse_label_mapping[int(curr_strat)]][:2] + \" \"\n",
    "    labels.append(converted_samp[:-1])\n",
    "\n",
    "str_labels = labels[:][::-1]\n",
    "\n",
    "xvals = [0, 1, 2, 7, 14, 21, 28, 29, 30]\n",
    "\n",
    "selected_labels = [str_labels[x] for x in xvals]\n",
    "\n",
    "fig, ax1 = plt.subplots()\n",
    "plt.xticks(xvals, selected_labels, rotation=90)\n",
    "\n",
    "t = range(0, len(attn_success_line[:][::-1]))\n",
    "color = '#00A2FF'\n",
    "ax1.set_xlabel('Strategy', fontsize=14)\n",
    "ax1.set_ylabel('Avg Strategy Attention', color=color, fontsize=14)\n",
    "ax1.plot(t, attn_success_line[:][::-1], color=color)\n",
    "ax1.tick_params(axis='y', labelcolor=color)\n",
    "\n",
    "ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\n",
    "\n",
    "color = '#EF5FA7'\n",
    "ax2.set_ylabel('Success Rate', color=color, fontsize=14)  # we already handled the x-label with ax1\n",
    "ax2.plot(t, rate_success_line[:][::-1], color=color)\n",
    "ax2.tick_params(axis='y', labelcolor=color)\n",
    "ax2.hlines(.485, xmin=0, xmax=31, linestyles='dashed', label=\"baseline\")\n",
    "fig.tight_layout()  # otherwise the right y-label is slightly clipped\n",
    "# plt.show()\n",
    "\n",
    "plt.text(0,.5,'Average Success Rate',rotation=0)\n",
    "plt.savefig(\"color.png\",  bbox_inches='tight')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.9020708231627488, 4.222950873877163e-12)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scipy\n",
    "scipy.stats.pearsonr(rate_success_line[:][::-1], attn_success_line[:][::-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Printing all our strategies -- this is in the appendix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 & Po Po EOS & 0.0 & 0.82 \\\\\n",
      "2 & Re Po EOS & 0.01 & 0.76 \\\\\n",
      "3 & Co Po EOS & 0.02 & 0.71 \\\\\n",
      "4 & Cr Po EOS & 0.01 & 0.7 \\\\\n",
      "5 & SOS Re EOS & 0.01 & 0.65 \\\\\n",
      "6 & SOS Po EOS & 0.0 & 0.65 \\\\\n",
      "7 & SOS Im EOS & 0.03 & 0.51 \\\\\n",
      "8 & Re Co EOS & 0.04 & 0.51 \\\\\n",
      "9 & SOS Co EOS & 0.02 & 0.5 \\\\\n",
      "10 & SOS Im Re & 0.04 & 0.5 \\\\\n",
      "11 & Co Re EOS & 0.05 & 0.5 \\\\\n",
      "12 & SOS Co Po & 0.04 & 0.5 \\\\\n",
      "13 & SOS Co Re & 0.04 & 0.49 \\\\\n",
      "14 & SOS Im Co & 0.05 & 0.48 \\\\\n",
      "15 & SOS Im Po & 0.04 & 0.46 \\\\\n",
      "16 & Re Co Po & 0.05 & 0.46 \\\\\n",
      "17 & SOS Co Im & 0.07 & 0.43 \\\\\n",
      "18 & Co Co EOS & 0.05 & 0.42 \\\\\n",
      "19 & Re Co Co & 0.08 & 0.41 \\\\\n",
      "20 & Po Co Co & 0.08 & 0.39 \\\\\n",
      "21 & Co Co Re & 0.08 & 0.39 \\\\\n",
      "22 & Im Co Re & 0.08 & 0.38 \\\\\n",
      "23 & SOS Co Co & 0.07 & 0.38 \\\\\n",
      "24 & SOS Co Cr & 0.05 & 0.37 \\\\\n",
      "25 & Co Im Re & 0.07 & 0.36 \\\\\n",
      "26 & Co Co Po & 0.07 & 0.36 \\\\\n",
      "27 & Im Co Co & 0.1 & 0.35 \\\\\n",
      "28 & Co Co Cr & 0.07 & 0.35 \\\\\n",
      "29 & Co Im Co & 0.09 & 0.34 \\\\\n",
      "30 & Co Co Co & 0.11 & 0.31 \\\\\n",
      "31 & Co Co Im & 0.11 & 0.27 \\\\\n"
     ]
    }
   ],
   "source": [
    "q = 1\n",
    "for k in merged_strats:\n",
    "    converted_samp = \"\"\n",
    "    for idx, curr_strat in enumerate(k.split(\" \")):\n",
    "        if curr_strat == \"UNK\":\n",
    "            if idx == 0:\n",
    "                converted_samp += \"SOS\" + \" \"\n",
    "            else: converted_samp += \"EOS\" + \" \"\n",
    "        else:\n",
    "            converted_samp += all_strategies[inverse_label_mapping[int(curr_strat)]][:2] + \" \"\n",
    "    print(str(q) + \" & \" + converted_samp + \"& \" + str(round(merged_strats[k][:][1], 2)) + \" & \" + str(round(merged_strats[k][:][0], 2)) + \" \\\\\\\\\")\n",
    "    q += 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finally, we'll edit bad strategies and see what we can do to improve them\n",
    "#### Section 5, Editing Strategy Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "good_strats = [\"5 5 UNK\", \"2 5 UNK\", \"3 5 UNK\"]\n",
    "bad_strats = [\"3 4 3\", \"3 3 3\", \"3 3 4\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this method takes a map of strings and returns the attn and success rate. \n",
    "def read_all_unlabeled(limited_unlabeled_data, data_path):\n",
    "\n",
    "    max_seq_num=args.max_seq_num\n",
    "    max_seq_len=args.max_seq_len\n",
    "    \n",
    "    with open(data_path + 'labeled_data.pkl', 'rb') as f:\n",
    "        labeled_data = pickle.load(f)\n",
    "        # {mid: sentences, labels}\n",
    "    with open(data_path + 'unlabeled_data.pkl', 'rb') as f:\n",
    "        unlabeled_data = pickle.load(f)\n",
    "        # {mid: message}\n",
    "    with open(data_path + 'mid2target.pkl', 'rb') as f:\n",
    "        mid2target = pickle.load(f)\n",
    "        # {mid: target, team_size}\n",
    "    \n",
    "    with open(data_path + 'label_mapping.pkl', 'rb') as f:\n",
    "        label_mapping = pickle.load(f)\n",
    "\n",
    "    try:\n",
    "        with open(data_path + 'vocab_2.pkl', 'rb') as f:\n",
    "            vocab = pickle.load(f)\n",
    "\n",
    "        print('unk words: ', vocab.unk_count)\n",
    "        print('vocab size: ', vocab.vocab_size)\n",
    "    except:\n",
    "        vocab = Vocab(unlabeled_data=unlabeled_data,\n",
    "                      labeled_data=labeled_data, embedding_size=embedding_size)\n",
    "\n",
    "        with open(data_path + 'vocab_2.pkl', 'wb') as f:\n",
    "            pickle.dump(vocab, f)\n",
    "\n",
    "        print('unk words: ', vocab.unk_count)\n",
    "        print('vocab size: ', vocab.vocab_size)\n",
    "\n",
    "    all_ids = [k for k in limited_unlabeled_data]\n",
    "    \n",
    "    train_unlabeled_dataset = Loader_unlabeled(\n",
    "        vocab, limited_unlabeled_data, all_ids, mid2target, max_seq_num, max_seq_len)\n",
    "\n",
    "    train_limited_loader = Data.DataLoader(\n",
    "        dataset=train_unlabeled_dataset, batch_size=16, shuffle=False)\n",
    "    \n",
    "    unlabeled_train_iter = iter(train_limited_loader)\n",
    "\n",
    "    content_vectors, strat_vectors, doc_labels, orig_mids = full_iterator(unlabeled_train_iter, \n",
    "        unlabeled_trainloader, vocab, model, n_labels)\n",
    "        \n",
    "    subset_details = get_content_strat_vector_details(content_vectors, strat_vectors, doc_labels,\n",
    "                                     None, test_model, return_rate=True)\n",
    "\n",
    "    flatten_doc_attn = [j for i in subset_details[1][\"document\"] for j in i]\n",
    "    flatten_sent_attn = [j for i in subset_details[1][\"strategy\"] for j in i]\n",
    "    flatten_mid = [j for i in orig_mids for j in i]\n",
    "    all_sent_attn = []\n",
    "\n",
    "    sent_map = {}\n",
    "    target_sent = []\n",
    "    for i in range(0, len(flatten_doc_attn)):\n",
    "        tot = 0\n",
    "        sent_map[flatten_mid[i]] = flatten_doc_attn[i].argmax().tolist()\n",
    "        if flatten_doc_attn[i].argmax().tolist() > 0:\n",
    "            tot += flatten_sent_attn[i][flatten_doc_attn[i].argmax() - 1].squeeze().tolist()\n",
    "        \n",
    "        if flatten_doc_attn[i].argmax().tolist() < 6:\n",
    "            tot += flatten_sent_attn[i][flatten_doc_attn[i].argmax() + 1].squeeze().tolist()\n",
    "        \n",
    "        tot += flatten_sent_attn[i][flatten_doc_attn[i].argmax()].squeeze().tolist()\n",
    "        all_sent_attn.append(tot/3)\n",
    "    \n",
    "    print(\"average msg attn: \" + str(sum(all_sent_attn) / len(all_sent_attn)))\n",
    "    print(\"success rate: \" + str(subset_details[-1]))\n",
    "    return sent_map, subset_details[-1], (sum(all_sent_attn) / len(all_sent_attn))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n"
     ]
    }
   ],
   "source": [
    "bad, good = get_samples(good_strats, bad_strats, merged_strats, \n",
    "            unlabeled_data_pkl, labeled_data_pkl, all_strategies, inverse_label_mapping)\n",
    "target_sents, _, _ = read_all_unlabeled(bad, args.data_path)\n",
    "bad_copy = fix_bad(bad, good, target_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "insertion_length = []\n",
    "for k in good:\n",
    "    e = sum([len(nltk.word_tokenize(x)) for x in nltk.sent_tokenize(good[k])[-2:]])\n",
    "    insertion_length.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_len = []\n",
    "for k in bad:\n",
    "    total_len.append(len(nltk.word_tokenize(bad[k])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "delete_len = []\n",
    "for k in bad:\n",
    "    sents_bad = nltk.sent_tokenize(bad[k])        \n",
    "    target = target_sents[k]\n",
    "    e = sum([len(nltk.word_tokenize(x)) for x in sents_bad[target-1:target+2]])\n",
    "    delete_len.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    974.000000\n",
       "mean      58.796715\n",
       "std       29.682731\n",
       "min        0.000000\n",
       "25%       43.000000\n",
       "50%       59.000000\n",
       "75%       75.000000\n",
       "max      262.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(delete_len).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     974.000000\n",
       "mean      174.388090\n",
       "std       128.870974\n",
       "min        23.000000\n",
       "25%        93.000000\n",
       "50%       138.500000\n",
       "75%       214.750000\n",
       "max      1613.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(total_len).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    775.000000\n",
       "mean      12.476129\n",
       "std       13.414256\n",
       "min        0.000000\n",
       "25%        3.000000\n",
       "50%        6.000000\n",
       "75%       19.000000\n",
       "max      117.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(insertion_length).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fix_bad takes two optional params: delete=False and insert=False, as defined in the paper\n",
    "### Here, we explore deletion, insertion, then deletion + insertion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n"
     ]
    }
   ],
   "source": [
    "bad_copy = fix_bad(bad, good, target_sents)\n",
    "_, rate, attn = read_all_unlabeled(bad_copy, args.data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n"
     ]
    }
   ],
   "source": [
    "bad_copy = fix_bad(bad, good, target_sents, True)\n",
    "_, rate, attn = read_all_unlabeled(bad_copy, args.data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10443263241609096\n",
      "success rate: 0.10677618069815195\n"
     ]
    }
   ],
   "source": [
    "bad_copy = fix_bad(bad, good, target_sents, False, True)\n",
    "_, rate, attn = read_all_unlabeled(bad_copy, args.data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05593976073684503\n",
      "success rate: 0.41478439425051333\n"
     ]
    }
   ],
   "source": [
    "bad_copy = fix_bad(bad, good, target_sents, True, True)\n",
    "_, rate, attn = read_all_unlabeled(bad_copy, args.data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10461854308646473\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 1/10 [00:28<04:18, 28.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.056197923900931486\n",
      "success rate: 0.4240246406570842\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10442779890192538\n",
      "success rate: 0.11190965092402463\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 2/10 [00:57<03:48, 28.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.056144793185587225\n",
      "success rate: 0.4137577002053388\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10452551064056297\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 3/10 [01:25<03:20, 28.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.05581704872661796\n",
      "success rate: 0.4240246406570842\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10454180615406897\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 4/10 [01:54<02:51, 28.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.05581811352352291\n",
      "success rate: 0.41581108829568786\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10470947515003581\n",
      "success rate: 0.10677618069815195\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 5/10 [02:22<02:22, 28.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.0557364977147501\n",
      "success rate: 0.4219712525667351\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479174559208841\n",
      "success rate: 0.10369609856262833\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 6/10 [02:51<01:53, 28.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.05573215856250455\n",
      "success rate: 0.41786447638603696\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10464056900696656\n",
      "success rate: 0.11088295687885011\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 7/10 [03:19<01:25, 28.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.056092568109071185\n",
      "success rate: 0.41683778234086244\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10460239188637376\n",
      "success rate: 0.10985626283367557\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 8/10 [03:47<00:56, 28.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.05576516908451678\n",
      "success rate: 0.41786447638603696\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10468280395950212\n",
      "success rate: 0.10677618069815195\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 9/10 [04:16<00:28, 28.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.055784642970722075\n",
      "success rate: 0.419917864476386\n",
      "###################################\n",
      "\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.10479660402196406\n",
      "success rate: 0.10882956878850103\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.05553688770502318\n",
      "success rate: 0.43121149897330596\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n",
      "average msg attn: 0.1047780463650295\n",
      "success rate: 0.11396303901437371\n",
      "\n",
      "unk words:  9516\n",
      "vocab size:  13944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [04:44<00:00, 28.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average msg attn: 0.056053411871956504\n",
      "success rate: 0.41786447638603696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "case1_rate = []\n",
    "case1_attn = []\n",
    "\n",
    "case2_rate = []\n",
    "case2_attn = []\n",
    "\n",
    "case3_rate = []\n",
    "case3_attn = []\n",
    "\n",
    "case4_rate = []\n",
    "case4_attn = []\n",
    "\n",
    "for runs in tqdm(range(10)): \n",
    "\n",
    "    print(\"###################################\")\n",
    "    print(\"\")\n",
    "    print(\"\")\n",
    "        \n",
    "    bad_copy = fix_bad(bad, good, target_sents)\n",
    "    _, rate, attn = read_all_unlabeled(bad_copy, args.data_path)\n",
    "    case1_rate.append(rate)\n",
    "    case1_attn.append(attn)  \n",
    "    \n",
    "    print()\n",
    "    \n",
    "    bad_copy = fix_bad(bad, good, target_sents, True)\n",
    "    _, rate, attn = read_all_unlabeled(bad_copy, args.data_path)\n",
    "    case2_rate.append(rate)\n",
    "    case2_attn.append(attn)  \n",
    "    \n",
    "    print()\n",
    "    \n",
    "    bad_copy = fix_bad(bad, good, target_sents, False, True)\n",
    "    _, rate, attn = read_all_unlabeled(bad_copy, args.data_path)\n",
    "    case3_rate.append(rate)\n",
    "    case3_attn.append(attn)  \n",
    "\n",
    "    print()\n",
    "    \n",
    "    bad_copy = fix_bad(bad, good, target_sents, True, True)\n",
    "    _, rate, attn = read_all_unlabeled(bad_copy, args.data_path)\n",
    "    case4_rate.append(rate)\n",
    "    case4_attn.append(attn)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg(lst):\n",
    "    return sum(lst) / len(lst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Original"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10882956878850103"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg(case1_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4312114989733059"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg(case2_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10903490759753594"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg(case3_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Swap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4189938398357289"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg(case4_rate)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}