metric.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from llama_index.llms.azure_openai import AzureOpenAI\n",
    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
    "\n",
    "\n",
    "api_key=os.getenv(\"AZURE_OPENAI_KEY\")\n",
    "api_version=\"2024-02-15-preview\"\n",
    "azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "model = \"gpt-35-turbo\"\n",
    "deployment_name = \"gpt-35-turbo\"\n",
    "\n",
    "LOCAL_EMBED_MODEL = \"BAAI/bge-small-en-v1.5\"\n",
    "\n",
    "class Embedder(HuggingFaceEmbedding):\n",
    "    def __init__(self, model_name: str = LOCAL_EMBED_MODEL, device: str = \"cuda\"):\n",
    "        super().__init__(model_name, device)\n",
    "\n",
    "class LLM(AzureOpenAI):\n",
    "    def __init__(self):\n",
    "        super().__init__(\n",
    "            model=deployment_name,\n",
    "            deployment_name=deployment_name,\n",
    "            api_key=api_key,\n",
    "            azure_endpoint=azure_endpoint,\n",
    "            api_version=api_version,\n",
    "            temperature=0.0\n",
    "        )\n",
    "llm = LLM()\n",
    "embedder = Embedder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lavague.action_engine import ActionEngine\n",
    "\n",
    "action_engine = ActionEngine(llm, embedder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lavague.defaults import evaluation_get_selenium_driver\n",
    "\n",
    "get_driver = evaluation_get_selenium_driver\n",
    "lavague_driver = get_driver()\n",
    "_, driver = lavague_driver.getDriver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>url</th>\n",
       "      <th>html</th>\n",
       "      <th>selenium_ground_truth</th>\n",
       "      <th>ground_truth_outer_html</th>\n",
       "      <th>ground_truth_highlighted_screenshot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>click on create account</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Main_Page</td>\n",
       "      <td>&lt;html class=\"client-js vector-feature-language...</td>\n",
       "      <td>create_account_link = driver.find_element(By.X...</td>\n",
       "      <td>&lt;li id=\"pt-createaccount-2\" class=\"user-links-...</td>\n",
       "      <td>iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>click on past</td>\n",
       "      <td>https://news.ycombinator.com/</td>\n",
       "      <td>&lt;html lang=\"en\" op=\"news\"&gt;&lt;head&gt;&lt;meta name=\"re...</td>\n",
       "      <td>link_to_click = driver.find_element(By.LINK_TE...</td>\n",
       "      <td>&lt;a href=\"front\"&gt;past&lt;/a&gt;</td>\n",
       "      <td>iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>click on pricing</td>\n",
       "      <td>https://www.perfagents.com/</td>\n",
       "      <td>&lt;html lang=\"en\"&gt;&lt;head&gt;\\n    &lt;meta charset=\"utf...</td>\n",
       "      <td>pricing_link = driver.find_element(By.XPATH, \"...</td>\n",
       "      <td>&lt;a class=\"font-medium px-3 lg:px-5 py-2 items-...</td>\n",
       "      <td>iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>click on SPF Justice</td>\n",
       "      <td>https://justice.belgium.be/fr</td>\n",
       "      <td>&lt;html lang=\"fr\" dir=\"ltr\" prefix=\"content: htt...</td>\n",
       "      <td>spf_justice_link = driver.find_element(By.XPAT...</td>\n",
       "      <td>&lt;span&gt;SPF Justice&lt;/span&gt;</td>\n",
       "      <td>iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>click on Search</td>\n",
       "      <td>https://www.cancer.gov/publications/dictionari...</td>\n",
       "      <td>&lt;html lang=\"en\" dir=\"ltr\" prefix=\"og: https://...</td>\n",
       "      <td>search_button = driver.find_element(By.XPATH, ...</td>\n",
       "      <td>&lt;input type=\"submit\" class=\"submit button post...</td>\n",
       "      <td>iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     query                                                url  \\\n",
       "0  click on create account            https://en.wikipedia.org/wiki/Main_Page   \n",
       "1            click on past                      https://news.ycombinator.com/   \n",
       "2         click on pricing                        https://www.perfagents.com/   \n",
       "3     click on SPF Justice                      https://justice.belgium.be/fr   \n",
       "4          click on Search  https://www.cancer.gov/publications/dictionari...   \n",
       "\n",
       "                                                html  \\\n",
       "0  <html class=\"client-js vector-feature-language...   \n",
       "1  <html lang=\"en\" op=\"news\"><head><meta name=\"re...   \n",
       "2  <html lang=\"en\"><head>\\n    <meta charset=\"utf...   \n",
       "3  <html lang=\"fr\" dir=\"ltr\" prefix=\"content: htt...   \n",
       "4  <html lang=\"en\" dir=\"ltr\" prefix=\"og: https://...   \n",
       "\n",
       "                               selenium_ground_truth  \\\n",
       "0  create_account_link = driver.find_element(By.X...   \n",
       "1  link_to_click = driver.find_element(By.LINK_TE...   \n",
       "2  pricing_link = driver.find_element(By.XPATH, \"...   \n",
       "3  spf_justice_link = driver.find_element(By.XPAT...   \n",
       "4  search_button = driver.find_element(By.XPATH, ...   \n",
       "\n",
       "                             ground_truth_outer_html  \\\n",
       "0  <li id=\"pt-createaccount-2\" class=\"user-links-...   \n",
       "1                           <a href=\"front\">past</a>   \n",
       "2  <a class=\"font-medium px-3 lg:px-5 py-2 items-...   \n",
       "3                           <span>SPF Justice</span>   \n",
       "4  <input type=\"submit\" class=\"submit button post...   \n",
       "\n",
       "                 ground_truth_highlighted_screenshot  \n",
       "0  iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...  \n",
       "1  iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...  \n",
       "2  iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...  \n",
       "3  iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...  \n",
       "4  iVBORw0KGgoAAAANSUhEUgAABkAAAAOECAYAAAD5Tf2iAA...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"BigAction/the-wave-clean\")\n",
    "\n",
    "df = dataset[\"test\"].to_pandas()\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of retriever only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "1it [00:01,  1.31s/it]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "2it [00:01,  1.23it/s]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "3it [00:03,  1.45s/it]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "4it [00:05,  1.40s/it]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "5it [00:05,  1.17s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recall_retriever</th>\n",
       "      <th>precision_retriever</th>\n",
       "      <th>retrieved_context</th>\n",
       "      <th>ground_truth_outer_html</th>\n",
       "      <th>retriever_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>&lt;div backend_node_id=\"139\" class=\"vector-menu ...</td>\n",
       "      <td>&lt;li backend_node_id=\"142\" class=\"user-links-co...</td>\n",
       "      <td>1.202136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>&lt;tbody backend_node_id=\"12\"&gt;&lt;tr backend_node_i...</td>\n",
       "      <td>&lt;a backend_node_id=\"26\" href=\"front\"&gt;past&lt;/a&gt;</td>\n",
       "      <td>0.396611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>&lt;div backend_node_id=\"78\" class=\"termly-styles...</td>\n",
       "      <td>&lt;a backend_node_id=\"103\" class=\"font-medium px...</td>\n",
       "      <td>2.135186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>_id=\"205\"&gt;Depuis le 01/01/2024, la Belgique as...</td>\n",
       "      <td>&lt;span backend_node_id=\"170\"&gt;SPF Justice&lt;/span&gt;</td>\n",
       "      <td>1.048980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>&lt;form action=\"/search/results\" backend_node_id...</td>\n",
       "      <td>&lt;input backend_node_id=\"184\" class=\"submit but...</td>\n",
       "      <td>0.491387</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   recall_retriever  precision_retriever  \\\n",
       "0               1.0             0.083333   \n",
       "1               1.0             0.030303   \n",
       "2               1.0             0.050000   \n",
       "3               1.0             0.041667   \n",
       "4               0.0             0.000000   \n",
       "\n",
       "                                   retrieved_context  \\\n",
       "0  <div backend_node_id=\"139\" class=\"vector-menu ...   \n",
       "1  <tbody backend_node_id=\"12\"><tr backend_node_i...   \n",
       "2  <div backend_node_id=\"78\" class=\"termly-styles...   \n",
       "3  _id=\"205\">Depuis le 01/01/2024, la Belgique as...   \n",
       "4  <form action=\"/search/results\" backend_node_id...   \n",
       "\n",
       "                             ground_truth_outer_html  retriever_time  \n",
       "0  <li backend_node_id=\"142\" class=\"user-links-co...        1.202136  \n",
       "1      <a backend_node_id=\"26\" href=\"front\">past</a>        0.396611  \n",
       "2  <a backend_node_id=\"103\" class=\"font-medium px...        2.135186  \n",
       "3     <span backend_node_id=\"170\">SPF Justice</span>        1.048980  \n",
       "4  <input backend_node_id=\"184\" class=\"submit but...        0.491387  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lavague.evaluator import SeleniumActionEvaluator\n",
    "\n",
    "evaluator = SeleniumActionEvaluator(driver, action_engine)\n",
    "\n",
    "sub_df = df.head(5)\n",
    "queries = sub_df[\"query\"].tolist()\n",
    "htmls = sub_df[\"html\"].tolist()\n",
    "ground_truths = sub_df[\"selenium_ground_truth\"].tolist()\n",
    "\n",
    "retriever_results = evaluator.batch_evaluate_retriever(queries, htmls, ground_truths, return_context=True)\n",
    "retriever_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of LLM only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [00:05,  1.11s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recall_llm</th>\n",
       "      <th>precision_llm</th>\n",
       "      <th>llm_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.421249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.874844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.774487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.821771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.694846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   recall_llm  precision_llm  llm_time\n",
       "0         0.0            0.0  1.421249\n",
       "1         1.0            1.0  0.874844\n",
       "2         1.0            1.0  0.774487\n",
       "3         0.0            0.0  0.821771\n",
       "4         1.0            1.0  0.694846"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ground_truth_outer_htmls = retriever_results[\"ground_truth_outer_html\"].tolist()\n",
    "retrieved_contexts = retriever_results[\"retrieved_context\"].tolist()\n",
    "\n",
    "llm_results = evaluator.batch_evaluate_llm(queries, htmls, ground_truth_outer_htmls, retrieved_contexts)\n",
    "llm_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recall_retriever</th>\n",
       "      <th>precision_retriever</th>\n",
       "      <th>retrieved_context</th>\n",
       "      <th>ground_truth_outer_html</th>\n",
       "      <th>retriever_time</th>\n",
       "      <th>recall_llm</th>\n",
       "      <th>precision_llm</th>\n",
       "      <th>llm_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>&lt;div backend_node_id=\"139\" class=\"vector-menu ...</td>\n",
       "      <td>&lt;li backend_node_id=\"142\" class=\"user-links-co...</td>\n",
       "      <td>1.202136</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.421249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>&lt;tbody backend_node_id=\"12\"&gt;&lt;tr backend_node_i...</td>\n",
       "      <td>&lt;a backend_node_id=\"26\" href=\"front\"&gt;past&lt;/a&gt;</td>\n",
       "      <td>0.396611</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.874844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>&lt;div backend_node_id=\"78\" class=\"termly-styles...</td>\n",
       "      <td>&lt;a backend_node_id=\"103\" class=\"font-medium px...</td>\n",
       "      <td>2.135186</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.774487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>_id=\"205\"&gt;Depuis le 01/01/2024, la Belgique as...</td>\n",
       "      <td>&lt;span backend_node_id=\"170\"&gt;SPF Justice&lt;/span&gt;</td>\n",
       "      <td>1.048980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.821771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>&lt;form action=\"/search/results\" backend_node_id...</td>\n",
       "      <td>&lt;input backend_node_id=\"184\" class=\"submit but...</td>\n",
       "      <td>0.491387</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.694846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   recall_retriever  precision_retriever  \\\n",
       "0               1.0             0.083333   \n",
       "1               1.0             0.030303   \n",
       "2               1.0             0.050000   \n",
       "3               1.0             0.041667   \n",
       "4               0.0             0.000000   \n",
       "\n",
       "                                   retrieved_context  \\\n",
       "0  <div backend_node_id=\"139\" class=\"vector-menu ...   \n",
       "1  <tbody backend_node_id=\"12\"><tr backend_node_i...   \n",
       "2  <div backend_node_id=\"78\" class=\"termly-styles...   \n",
       "3  _id=\"205\">Depuis le 01/01/2024, la Belgique as...   \n",
       "4  <form action=\"/search/results\" backend_node_id...   \n",
       "\n",
       "                             ground_truth_outer_html  retriever_time  \\\n",
       "0  <li backend_node_id=\"142\" class=\"user-links-co...        1.202136   \n",
       "1      <a backend_node_id=\"26\" href=\"front\">past</a>        0.396611   \n",
       "2  <a backend_node_id=\"103\" class=\"font-medium px...        2.135186   \n",
       "3     <span backend_node_id=\"170\">SPF Justice</span>        1.048980   \n",
       "4  <input backend_node_id=\"184\" class=\"submit but...        0.491387   \n",
       "\n",
       "   recall_llm  precision_llm  llm_time  \n",
       "0         0.0            0.0  1.421249  \n",
       "1         1.0            1.0  0.874844  \n",
       "2         1.0            1.0  0.774487  \n",
       "3         0.0            0.0  0.821771  \n",
       "4         1.0            1.0  0.694846  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = pd.concat([retriever_results, llm_results], axis=1)\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of full pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:01,  1.32s/it]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "2it [00:01,  1.23it/s]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "3it [00:04,  1.46s/it]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "4it [00:05,  1.41s/it]/home/daniel/miniconda3/envs/lavague/lib/python3.10/site-packages/tree_sitter/__init__.py:36: FutureWarning: Language(path, name) is deprecated. Use Language(ptr, name) instead.\n",
      "  warn(\"{} is deprecated. Use {} instead.\".format(old, new), FutureWarning)\n",
      "5it [00:05,  1.17s/it]\n",
      "5it [00:05,  1.07s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recall_retriever</th>\n",
       "      <th>precision_retriever</th>\n",
       "      <th>retriever_time</th>\n",
       "      <th>recall_llm</th>\n",
       "      <th>precision_llm</th>\n",
       "      <th>llm_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>1.213714</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.891550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>0.390518</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.922016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>2.161623</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.076999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>1.043746</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.903095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.489357</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.838961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   recall_retriever  precision_retriever  retriever_time  recall_llm  \\\n",
       "0               1.0             0.083333        1.213714         0.0   \n",
       "1               1.0             0.030303        0.390518         1.0   \n",
       "2               1.0             0.050000        2.161623         1.0   \n",
       "3               1.0             0.041667        1.043746         0.0   \n",
       "4               0.0             0.000000        0.489357         1.0   \n",
       "\n",
       "   precision_llm  llm_time  \n",
       "0            0.0  0.891550  \n",
       "1            1.0  0.922016  \n",
       "2            1.0  1.076999  \n",
       "3            0.0  0.903095  \n",
       "4            1.0  0.838961  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = evaluator.batch_evaluate(queries, htmls, ground_truths, return_context=False)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(results.recall_retriever == 1.0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAno0lEQVR4nO3df3DU9Z3H8dcmbDbkzCKWIeFHUBQtP4MECi5eC/YIERk0/1AHHEMdoNdrmEFzA9eolcRUwiiITEEQ0cbrNIeFKemNIrCNFxhKqILJDKBycnDEq9lQWslCUpc1+70/nKRu84N8l+z30w3Px0z+2G8+38++98VqXvPdTdZlWZYlAAAAQ5JMDwAAAG5slBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARg0wPUBvRCIRffbZZ0pPT5fL5TI9DgAA6AXLsnT58mUNHz5cSUndX/9IiDLy2WefKSsry/QYAAAgBp9++qlGjhzZ7fcTooykp6dL+urBeL3ePts3HA7rwIEDmjt3rtxud5/ti2jk7ByydgY5O4OcnRHPnIPBoLKysjp+jncnIcpI+0szXq+3z8tIWlqavF4vT/Q4ImfnkLUzyNkZ5OwMJ3K+1lsseAMrAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjLquMrJu3Tq5XC49/vjjPa7btWuXxo4dq9TUVE2aNEl79+69nrsFAAD9SMxl5P3339crr7yi7OzsHtcdOXJEixYt0tKlS1VXV6f8/Hzl5+fr5MmTsd41AADoR2IqI1euXNEjjzyiV199VYMHD+5x7aZNm3T//fdr1apVGjdunMrKypSTk6PNmzfHNDAAAOhfYvrU3sLCQs2fP19z5szRT3/60x7X1tbWqqioKOpYXl6eqqqquj0nFAopFAp13A4Gg5K++mTBcDgcy8hdat+rL/dEZ+TsHLJ2Bjk7g5ydEc+ce7un7TKyc+dOffDBB3r//fd7tT4QCCgjIyPqWEZGhgKBQLfnlJeXq7S0tNPxAwcOKC0tzd7AveD3+/t8T3RGzs4ha2eQszPI2RnxyLm1tbVX62yVkU8//VQrV66U3+9XampqTIP1RnFxcdTVlGAwqKysLM2dO1der7fP7iccDsvv9ys3N1dut7vP9kU0cnYOWTuDnJ2RqDlPLNlvegRbPEmWyqZF4pJz+ysb12KrjBw/flwXLlxQTk5Ox7G2tjYdOnRImzdvVigUUnJyctQ5mZmZampqijrW1NSkzMzMbu/H4/HI4/F0Ou52u+PyhIzXvohGzs4ha2eQszMSLedQm8v0CDGJR8693c/WG1j/6Z/+SSdOnFB9fX3H17Rp0/TII4+ovr6+UxGRJJ/Pp+rq6qhjfr9fPp/Pzl0DAIB+ytaVkfT0dE2cODHq2D/8wz/oG9/4RsfxgoICjRgxQuXl5ZKklStXatasWdqwYYPmz5+vnTt36tixY9q+fXsfPQQAAJDI+vwvsDY0NKixsbHj9syZM1VZWant27dr8uTJ2r17t6qqqjqVGgAAcGOK6Vd7v66mpqbH25K0cOFCLVy48HrvCgAA9EN8Ng0AADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwylYZ2bp1q7Kzs+X1euX1euXz+fTOO+90u76iokIulyvqKzU19bqHBgAA/ccAO4tHjhypdevW6c4775RlWXrjjTf00EMPqa6uThMmTOjyHK/Xq9OnT3fcdrlc1zcxAADoV2yVkQULFkTdfu6557R161YdPXq02zLicrmUmZkZ+4QAAKBfs1VGvq6trU27du1SS0uLfD5ft+uuXLmiW2+9VZFIRDk5OVq7dm23xaVdKBRSKBTquB0MBiVJ4XBY4XA41pE7ad+rL/dEZ+TsHLJ2Bjk7I1Fz9iRbpkewxZP01bzxyLm3e7osy7KV2okTJ+Tz+fTFF1/opptuUmVlpR544IEu19bW1uqTTz5Rdna2mpubtX79eh06dEinTp3SyJEju72PkpISlZaWdjpeWVmptLQ0O+MCAABDWltbtXjxYjU3N8vr9Xa7znYZuXr1qhoaGtTc3Kzdu3drx44dOnjwoMaPH3/Nc8PhsMaNG6dFixaprKys23VdXRnJysrSxYsXe3wwdoXDYfn9fuXm5srtdvfZvohGzs4ha2eQszMSNeeJJftNj2CLJ8lS2bRIXHIOBoMaMmTINcuI7ZdpUlJSNGbMGEnS1KlT9f7772vTpk165ZVXrnmu2+3WlClTdObMmR7XeTweeTyeLs+PxxMyXvsiGjk7h6ydQc7OSLScQ22J+Ysa8ci5t/td998ZiUQiUVcxetLW1qYTJ05o2LBh13u3AACgn7B1ZaS4uFjz5s3TqFGjdPnyZVVWVqqmpkb79391SaqgoEAjRoxQeXm5JOnZZ5/VPffcozFjxujSpUt64YUXdP78eS1btqzvHwkAAEhItsrIhQsXVFBQoMbGRg0aNEjZ2dnav3+/cnNzJUkNDQ1KSvrrxZbPP/9cy5cvVyAQ0ODBgzV16lQdOXKkV+8vAQAANwZbZeS1117r8fs1NTVRtzdu3KiNGzfaHgoAANw4+GwaAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYJStMrJ161ZlZ2fL6/XK6/XK5/PpnXfe6fGcXbt2aezYsUpNTdWkSZO0d+/e6xoYAAD0L7bKyMiRI7Vu3TodP35cx44d03e/+1099NBDOnXqVJfrjxw5okWLFmnp0qWqq6tTfn6+8vPzdfLkyT4ZHgAAJD5bZWTBggV64IEHdOedd+quu+7Sc889p5tuuklHjx7tcv2mTZt0//33a9WqVRo3bpzKysqUk5OjzZs398nwAAAg8Q2I9cS2tjbt2rVLLS0t8vl8Xa6pra1VUVFR1LG8vDxVVVX1uHcoFFIoFOq4HQwGJUnhcFjhcDjWkTtp36sv90Rn5OwcsnYGOTsjUXP2JFumR7DFk/TVvPHIubd72i4jJ06ckM/n0xdffKGbbrpJe/bs0fjx47tcGwgElJGREXUsIyNDgUCgx/soLy9XaWlpp+MHDhxQWlqa3ZGvye/39/me6IycnUPWziBnZyRazs9PNz1BbOKRc2tra6/W2S4j3/zmN1VfX6/m5mbt3r1bS5Ys0cGDB7stJLEoLi6OuqISDAaVlZWluXPnyuv19tn9hMNh+f1+5ebmyu1299m+iEbOziFrZ5CzMxI154kl+02PYIsnyVLZtEhccm5/ZeNabJeRlJQUjRkzRpI0depUvf/++9q0aZNeeeWVTmszMzPV1NQUdaypqUmZmZk93ofH45HH4+l03O12x+UJGa99EY2cnUPWziBnZyRazqE2l+kRYhKPnHu733X/nZFIJBL1/o6v8/l8qq6ujjrm9/u7fY8JAAC48di6MlJcXKx58+Zp1KhRunz5siorK1VTU6P9+7+6JFVQUKARI0aovLxckrRy5UrNmjVLGzZs0Pz587Vz504dO3ZM27dv7/tHAgAAEpKtMnLhwgUVFBSosbFRgwYNUnZ2tvbv36/c3FxJUkNDg5KS/nqxZebMmaqsrNTTTz+tJ598Unfeeaeqqqo0ceLEvn0UAAAgYdkqI6+99lqP36+pqel0bOHChVq4cKGtoQAAwI2Dz6YBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARtkqI+Xl5frWt76l9PR0DR06VPn5+Tp9+nSP51RUVMjlckV9paamXtfQAACg/7BVRg4ePKjCwkIdPXpUfr9f4XBYc+fOVUtLS4/neb1eNTY2dnydP3/+uoYGAAD9xwA7i/ft2xd1u6KiQkOHDtXx48f1ne98p9vzXC6XMjMzY5sQAAD0a7bKyN9qbm6WJN1yyy09rrty5YpuvfVWRSIR5eTkaO3atZowYUK360OhkEKhUMftYDAoSQqHwwqHw9czcpT2vfpyT3RGzs4ha2eQszMSNWdPsmV6BFs8SV/NG4+ce7uny7KsmFKLRCJ68MEHdenSJR0+fLjbdbW1tfrkk0+UnZ2t5uZmrV+/XocOHdKpU6c0cuTILs8pKSlRaWlpp+OVlZVKS0uLZVwAAOCw1tZWLV68WM3NzfJ6vd2ui7mM/Mu//IveeecdHT58uNtS0ZVwOKxx48Zp0aJFKisr63JNV1dGsrKydPHixR4fjF3hcFh+v1+5ublyu919ti+ikbNzyNoZ5OyMRM15Ysl+0yPY4kmyVDYtEpecg8GghgwZcs0yEtPLNCtWrNBbb72lQ4cO2SoikuR2uzVlyhSdOXOm2zUej0cej6fLc+PxhIzXvohGzs4ha2eQszMSLedQm8v0CDGJR8693c/Wb9NYlqUVK1Zoz549evfddzV69Gjbg7W1tenEiRMaNmyY7XMBAED/Y+vKSGFhoSorK/Wb3/xG6enpCgQCkqRBgwZp4MCBkqSCggKNGDFC5eXlkqRnn31W99xzj8aMGaNLly7phRde0Pnz57Vs2bI+figAACAR2SojW7dulSTNnj076vjPf/5zff/735ckNTQ0KCnprxdcPv/8cy1fvlyBQECDBw/W1KlTdeTIEY0fP/76JgcAAP2CrTLSm/e61tTURN3euHGjNm7caGsoAABw4+CzaQAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRtspIeXm5vvWtbyk9PV1Dhw5Vfn6+Tp8+fc3zdu3apbFjxyo1NVWTJk3S3r17Yx4YAAD0L7bKyMGDB1VYWKijR4/K7/crHA5r7ty5amlp6facI0eOaNGiRVq6dKnq6uqUn5+v/Px8nTx58rqHBwAAiW+AncX79u2Lul1RUaGhQ4fq+PHj+s53vtPlOZs2bdL999+vVatWSZLKysrk9/u1efNmbdu2LcaxAQBAf2GrjPyt5uZmSdItt9zS7Zra2loVFRVFHcvLy1NVVVW354RCIYVCoY7bwWBQkhQOhxUOh69j4mjte/XlnuiMnJ1D1s4gZ2ckas6eZMv0CLZ4kr6aNx4593ZPl2VZMaUWiUT04IMP6tKlSzp8+HC361JSUvTGG29o0aJFHcdefvlllZaWqqmpqctzSkpKVFpa2ul4ZWWl0tLSYhkXAAA4rLW1VYsXL1Zzc7O8Xm+362K+MlJYWKiTJ0/2WERiVVxcHHU1JRgMKisrS3Pnzu3xwdgVDofl9/uVm5srt9vdZ/siGjk7h6ydQc7OSNScJ5bsNz2CLZ4kS2XTInHJuf2VjWuJqYysWLFCb731lg4dOqSRI0f2uDYzM7PTFZCmpiZlZmZ2e47H45HH4+l03O12x+UJGa99EY2cnUPWziBnZyRazqE2l+kRYhKPnHu7n63fprEsSytWrNCePXv07rvvavTo0dc8x+fzqbq6OuqY3++Xz+ezc9cAAKCfsnVlpLCwUJWVlfrNb36j9PR0BQIBSdKgQYM0cOBASVJBQYFGjBih8vJySdLKlSs1a9YsbdiwQfPnz9fOnTt17Ngxbd++vY8fCgAASES2roxs3bpVzc3Nmj17toYNG9bx9eabb3asaWhoUGNjY8ftmTNnqrKyUtu3b9fkyZO1e/duVVVVaeLEiX33KAAAQMKydWWkN794U1NT0+nYwoULtXDhQjt3BQAAbhB8Ng0AADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwynYZOXTokBYsWKDhw4fL5XKpqqqqx/U1NTVyuVydvgKBQKwzAwCAfsR2GWlpadHkyZO1ZcsWW+edPn1ajY2NHV9Dhw61e9cAAKAfGmD3hHnz5mnevHm272jo0KG6+eabbZ8HAAD6N9tlJFZ33323QqGQJk6cqJKSEt17773drg2FQgqFQh23g8GgJCkcDiscDvfZTO179eWe6IycnUPWziBnZyRqzp5ky/QItniSvpo3Hjn3dk+XZVkxp+ZyubRnzx7l5+d3u+b06dOqqanRtGnTFAqFtGPHDv3iF7/Q73//e+Xk5HR5TklJiUpLSzsdr6ysVFpaWqzjAgAAB7W2tmrx4sVqbm6W1+vtdl3cy0hXZs2apVGjRukXv/hFl9/v6spIVlaWLl682OODsSscDsvv9ys3N1dut7vP9kU0cnYOWTuDnJ2RqDlPLNlvegRbPEmWyqZF4pJzMBjUkCFDrllGHHuZ5uumT5+uw4cPd/t9j8cjj8fT6bjb7Y7LEzJe+yIaOTuHrJ1Bzs5ItJxDbS7TI8QkHjn3dj8jf2ekvr5ew4YNM3HXAADg74ztKyNXrlzRmTNnOm6fO3dO9fX1uuWWWzRq1CgVFxfrD3/4g/793/9dkvTSSy9p9OjRmjBhgr744gvt2LFD7777rg4cONB3jwIAACQs22Xk2LFjuu+++zpuFxUVSZKWLFmiiooKNTY2qqGhoeP7V69e1b/+67/qD3/4g9LS0pSdna3f/va3UXsAAIAbl+0yMnv2bPX0nteKioqo26tXr9bq1attDwYAAG4MfDYNAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMMp2GTl06JAWLFig4cOHy+Vyqaqq6prn1NTUKCcnRx6PR2PGjFFFRUUMowIAgP7IdhlpaWnR5MmTtWXLll6tP3funObPn6/77rtP9fX1evzxx7Vs2TLt37/f9rAAAKD/GWD3hHnz5mnevHm9Xr9t2zaNHj1aGzZskCSNGzdOhw8f1saNG5WXl2f37gEAQD9ju4zYVVtbqzlz5kQdy8vL0+OPP97tOaFQSKFQqON2MBiUJIXDYYXD4T6brX2vvtwTnZGzc8jaGeTsjETN2ZNsmR7BFk/SV/PGI+fe7hn3MhIIBJSRkRF1LCMjQ8FgUH/5y180cODATueUl5ertLS00/EDBw4oLS2tz2f0+/19vic6I2fnkLUzyNkZiZbz89NNTxCbeOTc2traq3VxLyOxKC4uVlFRUcftYDCorKwszZ07V16vt8/uJxwOy+/36yfHkhSKuPps33g7WZJYL2+155ybmyu32216nH6NrJ1Bzs5I1JwnliTWeyI9SZbKpkXiknP7KxvXEvcykpmZqaampqhjTU1N8nq9XV4VkSSPxyOPx9PpuNvtjssTMhRxKdSWOGUkkf6j/Lp4/fuhM7J2Bjk7I9FyTqSfJ18Xj5x7u1/c/86Iz+dTdXV11DG/3y+fzxfvuwYAAAnAdhm5cuWK6uvrVV9fL+mrX92tr69XQ0ODpK9eYikoKOhY/8Mf/lBnz57V6tWr9fHHH+vll1/Wr371Kz3xxBN98wgAAEBCs11Gjh07pilTpmjKlCmSpKKiIk2ZMkXPPPOMJKmxsbGjmEjS6NGj9fbbb8vv92vy5MnasGGDduzYwa/1AgAASTG8Z2T27NmyrO5/bamrv646e/Zs1dXV2b0rAABwA+CzaQAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAYRRkBAABGUUYAAIBRMZWRLVu26LbbblNqaqpmzJih9957r9u1FRUVcrlcUV+pqakxDwwAAPoX22XkzTffVFFRkdasWaMPPvhAkydPVl5eni5cuNDtOV6vV42NjR1f58+fv66hAQBA/2G7jLz44otavny5HnvsMY0fP17btm1TWlqaXn/99W7PcblcyszM7PjKyMi4rqEBAED/McDO4qtXr+r48eMqLi7uOJaUlKQ5c+aotra22/OuXLmiW2+9VZFIRDk5OVq7dq0mTJjQ7fpQKKRQKNRxOxgMSpLC4bDC4bCdkXvUvpcnyeqzPZ3Qlxk4oX3eRJs7EZG1M8jZGYmasyc5sX6mtP8MjEfOvd3TZVlWr1P77LPPNGLECB05ckQ+n6/j+OrVq3Xw4EH9/ve/73RObW2tPvnkE2VnZ6u5uVnr16/XoUOHdOrUKY0cObLL+ykpKVFpaWmn45WVlUpLS+vtuAAAwKDW1lYtXrxYzc3N8nq93a6zdWUkFj6fL6q4zJw5U+PGjdMrr7yisrKyLs8pLi5WUVFRx+1gMKisrCzNnTu3xwdjVzgclt/v10+OJSkUcfXZvvF2siTP9Ai2tOecm5srt9ttepx+jaydQc7OSNScJ5bsNz2CLZ4kS2XTInHJuf2VjWuxVUaGDBmi5ORkNTU1RR1vampSZmZmr/Zwu92aMmWKzpw50+0aj8cjj8fT5bnxeEKGIi6F2hKnjCTSf5RfF69/P3RG1s4gZ2ckWs6J9PPk6+KRc2/3s/UG1pSUFE2dOlXV1dUdxyKRiKqrq6OufvSkra1NJ06c0LBhw+zcNQAA6Kdsv0xTVFSkJUuWaNq0aZo+fbpeeukltbS06LHHHpMkFRQUaMSIESovL5ckPfvss7rnnns0ZswYXbp0SS+88ILOnz+vZcuW9e0jAQAACcl2GXn44Yf1xz/+Uc8884wCgYDuvvtu7du3r+PXdRsaGpSU9NcLLp9//rmWL1+uQCCgwYMHa+rUqTpy5IjGjx/fd48CAAAkrJjewLpixQqtWLGiy+/V1NRE3d64caM2btwYy90AAIAbAJ9NAwAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyijAAAAKMoIwAAwCjKCAAAMIoyAgAAjKKMAAAAoygjAADAKMoIAAAwijICAACMoowAAACjKCMAAMAoyggAADCKMgIAAIyKqYxs2bJFt912m1JTUzVjxgy99957Pa7ftWuXxo4dq9TUVE2aNEl79+6NaVgAAND/2C4jb775poqKirRmzRp98MEHmjx5svLy8nThwoUu1x85ckSLFi3S0qVLVVdXp/z8fOXn5+vkyZPXPTwAAEh8tsvIiy++qOXLl+uxxx7T+PHjtW3bNqWlpen111/vcv2mTZt0//33a9WqVRo3bpzKysqUk5OjzZs3X/fwAAAg8Q2ws/jq1as6fvy4iouLO44lJSVpzpw5qq2t7fKc2tpaFRUVRR3Ly8tTVVVVt/cTCoUUCoU6bjc3N0uS/vznPyscDtsZuUfhcFitra0aEE5SW8TVZ/vG25/+9CfTI9jSnvOf/vQnud1u0+P0a2TtDHJ2RqLmPODLFtMj2DIgYqm1NRKXnC9fvixJsiyr5xnsbHrx4kW1tbUpIyMj6nhGRoY+/vjjLs8JBAJdrg8EAt3eT3l5uUpLSzsdHz16tJ1x+60hG0xPAADoTxbHef/Lly9r0KBB3X7fVhlxSnFxcdTVlEgkoj//+c/6xje+IZer765gBINBZWVl6dNPP5XX6+2zfRGNnJ1D1s4gZ2eQszPimbNlWbp8+bKGDx/e4zpbZWTIkCFKTk5WU1NT1PGmpiZlZmZ2eU5mZqat9ZLk8Xjk8Xiijt188812RrXF6/XyRHcAOTuHrJ1Bzs4gZ2fEK+eeroi0s/UG1pSUFE2dOlXV1dUdxyKRiKqrq+Xz+bo8x+fzRa2XJL/f3+16AABwY7H9Mk1RUZGWLFmiadOmafr06XrppZfU0tKixx57TJJUUFCgESNGqLy8XJK0cuVKzZo1Sxs2bND8+fO1c+dOHTt2TNu3b+/bRwIAABKS7TLy8MMP649//KOeeeYZBQIB3X333dq3b1/Hm1QbGhqUlPTXCy4zZ85UZWWlnn76aT355JO68847VVVVpYkTJ/bdo4iRx+PRmjVrOr0khL5Fzs4ha2eQszPI2Rl/Dzm7rGv9vg0AAEAc8dk0AADAKMoIAAAwijICAACMoowAAACj+n0Z2bJli2677TalpqZqxowZeu+993pcv2vXLo0dO1apqamaNGmS9u7d69Ckic1Ozq+++qq+/e1va/DgwRo8eLDmzJlzzX8X/JXd53S7nTt3yuVyKT8/P74D9hN2c7506ZIKCws1bNgweTwe3XXXXfz/oxfs5vzSSy/pm9/8pgYOHKisrCw98cQT+uKLLxyaNjEdOnRICxYs0PDhw+VyuXr8bLh2NTU1ysnJkcfj0ZgxY1RRURHfIa1+bOfOnVZKSor1+uuvW6dOnbKWL19u3XzzzVZTU1OX63/3u99ZycnJ1vPPP299+OGH1tNPP2253W7rxIkTDk+eWOzmvHjxYmvLli1WXV2d9dFHH1nf//73rUGDBln/93//5/Dkicdu1u3OnTtnjRgxwvr2t79tPfTQQ84Mm8Ds5hwKhaxp06ZZDzzwgHX48GHr3LlzVk1NjVVfX+/w5InFbs6//OUvLY/HY/3yl7+0zp07Z+3fv98aNmyY9cQTTzg8eWLZu3ev9dRTT1m//vWvLUnWnj17elx/9uxZKy0tzSoqKrI+/PBD62c/+5mVnJxs7du3L24z9usyMn36dKuwsLDjdltbmzV8+HCrvLy8y/Xf+973rPnz50cdmzFjhvXP//zPcZ0z0dnN+W99+eWXVnp6uvXGG2/Ea8R+I5asv/zyS2vmzJnWjh07rCVLllBGesFuzlu3brVuv/126+rVq06N2C/YzbmwsND67ne/G3WsqKjIuvfee+M6Z3/SmzKyevVqa8KECVHHHn74YSsvLy9uc/Xbl2muXr2q48ePa86cOR3HkpKSNGfOHNXW1nZ5Tm1tbdR6ScrLy+t2PWLL+W+1trYqHA7rlltuideY/UKsWT/77LMaOnSoli5d6sSYCS+WnP/zP/9TPp9PhYWFysjI0MSJE7V27Vq1tbU5NXbCiSXnmTNn6vjx4x0v5Zw9e1Z79+7VAw884MjMNwoTPwv/Lj+1ty9cvHhRbW1tHX8Ztl1GRoY+/vjjLs8JBAJdrg8EAnGbM9HFkvPf+rd/+zcNHz6805Mf0WLJ+vDhw3rttddUX1/vwIT9Qyw5nz17Vu+++64eeeQR7d27V2fOnNGPfvQjhcNhrVmzxomxE04sOS9evFgXL17UP/7jP8qyLH355Zf64Q9/qCeffNKJkW8Y3f0sDAaD+stf/qKBAwf2+X322ysjSAzr1q3Tzp07tWfPHqWmppoep1+5fPmyHn30Ub366qsaMmSI6XH6tUgkoqFDh2r79u2aOnWqHn74YT311FPatm2b6dH6lZqaGq1du1Yvv/yyPvjgA/3617/W22+/rbKyMtOj4Tr12ysjQ4YMUXJyspqamqKONzU1KTMzs8tzMjMzba1HbDm3W79+vdatW6ff/va3ys7OjueY/YLdrP/nf/5H//u//6sFCxZ0HItEIpKkAQMG6PTp07rjjjviO3QCiuU5PWzYMLndbiUnJ3ccGzdunAKBgK5evaqUlJS4zpyIYsn5Jz/5iR599FEtW7ZMkjRp0iS1tLToBz/4gZ566qmoz0VD7Lr7Wej1euNyVUTqx1dGUlJSNHXqVFVXV3cci0Qiqq6uls/n6/Icn88XtV6S/H5/t+sRW86S9Pzzz6usrEz79u3TtGnTnBg14dnNeuzYsTpx4oTq6+s7vh588EHdd999qq+vV1ZWlpPjJ4xYntP33nuvzpw501H2JOm///u/NWzYMIpIN2LJubW1tVPhaC+AFh+z1meM/CyM21tj/w7s3LnT8ng8VkVFhfXhhx9aP/jBD6ybb77ZCgQClmVZ1qOPPmr9+Mc/7lj/u9/9zhowYIC1fv1666OPPrLWrFnDr/b2gt2c161bZ6WkpFi7d++2GhsbO74uX75s6iEkDLtZ/y1+m6Z37Obc0NBgpaenWytWrLBOnz5tvfXWW9bQoUOtn/70p6YeQkKwm/OaNWus9PR06z/+4z+ss2fPWgcOHLDuuOMO63vf+56ph5AQLl++bNXV1Vl1dXWWJOvFF1+06urqrPPnz1uWZVk//vGPrUcffbRjffuv9q5atcr66KOPrC1btvCrvdfrZz/7mTVq1CgrJSXFmj59unX06NGO782aNctasmRJ1Ppf/epX1l133WWlpKRYEyZMsN5++22HJ05MdnK+9dZbLUmdvtasWeP84AnI7nP66ygjvWc35yNHjlgzZsywPB6Pdfvtt1vPPfec9eWXXzo8deKxk3M4HLZKSkqsO+64w0pNTbWysrKsH/3oR9bnn3/u/OAJ5L/+67+6/H9ue7ZLliyxZs2a1emcu+++20pJSbFuv/126+c//3lcZ3RZFte2AACAOf32PSMAACAxUEYAAIBRlBEAAGAUZQQAABhFGQEAAEZRRgAAgFGUEQAAYBRlBAAAGEUZAQAARlFGAACAUZQRAABgFGUEAAAY9f97/TNw/lre5AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results.recall_retriever.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(results.recall_llm == 1.0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LLM recall when retriever is right: 0.5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvP0lEQVR4nO3de3BUZZ7G8acTkg5xCBeRXDBAvIFyi8KSDTorOIHAUCxs1SqXXcGU4o6SKrF3ZI0CCYQ1LKsIzmTMiiC4JYZxV3F3ZAIxTqQYAhSX1IgCK4iikI7ACE2SsdPQZ/+w0tImQE5Md94O309Vl5633/Pmd35JOE+dPp12WJZlCQAAwGBRHV0AAADA1RBYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADG69LRBbQHv9+vkydPqlu3bnI4HB1dDgAAaAXLsnT+/HmlpKQoKurK11A6RWA5efKkUlNTO7oMAADQBl9++aVuvPHGK87pFIGlW7dukr474ISEhHZd2+fzaevWrRo/frxiYmLadW18jz6HB30OH3odHvQ5PELVZ4/Ho9TU1MB5/Eo6RWBpehkoISEhJIElPj5eCQkJ/DKEEH0OD/ocPvQ6POhzeIS6z625nYObbgEAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeLYCS1FRkf7qr/5K3bp1U58+fTR16lQdPnz4qvu99dZbGjRokOLi4jR06FBt3rw56HnLsrRo0SIlJyera9euysrK0qeffmrvSAAAQKdlK7B8+OGHmjt3rnbu3Kny8nL5fD6NHz9e9fX1l91nx44dmjFjhh5++GHt379fU6dO1dSpU3XgwIHAnOXLl+ull15SSUmJdu3apeuuu07Z2dn69ttv235kAACg07D14YdlZWVB2+vWrVOfPn20d+9e/c3f/E2L+6xatUoTJkzQU089JUkqLCxUeXm5fv3rX6ukpESWZWnlypVasGCBpkyZIkl6/fXXlZiYqE2bNmn69OltOS4AANCJ/KhPaz537pwkqVevXpedU1VVJZfLFTSWnZ2tTZs2SZKOHTsmt9utrKyswPPdu3dXRkaGqqqqWgwsXq9XXq83sO3xeCR992mSPp+vzcfTkqb12ntdBKPP4UGfw4dehwd9Do9Q9dnOem0OLH6/X/PmzdPdd9+tIUOGXHae2+1WYmJi0FhiYqLcbnfg+aaxy835oaKiIi1evLjZ+NatWxUfH2/rOFqrvLw8JOsiGH0OD/ocPvQ6POhzeLR3nxsaGlo9t82BZe7cuTpw4IC2b9/e1iXaLC8vL+iqjcfjUWpqqsaPH6+EhIR2/Vo+n0/l5eVauCdKXr+jXdcOpQMF2R1dgi1NfR43bpxiYmI6upxOiz6HD70Oj0jt85CCLR1dgi3OKEuFI/3t3uemV0hao02BJTc3V7/73e+0bds23XjjjVecm5SUpNra2qCx2tpaJSUlBZ5vGktOTg6ak56e3uKaTqdTTqez2XhMTEzIfmC9foe8FyMnsETSL+6lQvk9xPfoc/jQ6/CItD5H0vnkUu3dZztr2XqXkGVZys3N1TvvvKMPPvhAaWlpV90nMzNTFRUVQWPl5eXKzMyUJKWlpSkpKSlojsfj0a5duwJzAADAtc3WFZa5c+dqw4YNevfdd9WtW7fAPSbdu3dX165dJUmzZs1S3759VVRUJEl64okndO+99+qFF17QpEmTVFpaqj179uiVV16RJDkcDs2bN09Lly7VrbfeqrS0NC1cuFApKSmaOnVqOx4qAACIVLYCy8svvyxJGjNmTND4a6+9poceekiSdPz4cUVFfX/hZvTo0dqwYYMWLFigZ555Rrfeeqs2bdoUdKPu/PnzVV9fr0cffVRnz57VPffco7KyMsXFxbXxsAAAQGdiK7BYlnXVOZWVlc3G7r//ft1///2X3cfhcGjJkiVasmSJnXIAAMA1gs8SAgAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGsx1Ytm3bpsmTJyslJUUOh0ObNm264vyHHnpIDoej2WPw4MGBOQUFBc2eHzRokO2DAQAAnZPtwFJfX6/hw4eruLi4VfNXrVqlmpqawOPLL79Ur169dP/99wfNGzx4cNC87du32y0NAAB0Ul3s7jBx4kRNnDix1fO7d++u7t27B7Y3bdqkb775Rjk5OcGFdOmipKQku+UAAIBrgO3A8mOtWbNGWVlZ6t+/f9D4p59+qpSUFMXFxSkzM1NFRUXq169fi2t4vV55vd7AtsfjkST5fD75fL52rbdpPWeU1a7rhlp79yHUmuqNtLojDX0OH3odHpHaZ2d0ZJ1Tms6BoTrHtobDsqw2d83hcOidd97R1KlTWzX/5MmT6tevnzZs2KAHHnggMP773/9edXV1GjhwoGpqarR48WKdOHFCBw4cULdu3ZqtU1BQoMWLFzcb37Bhg+Lj49t6OAAAIIwaGho0c+ZMnTt3TgkJCVecG9bAUlRUpBdeeEEnT55UbGzsZeedPXtW/fv314oVK/Twww83e76lKyypqak6ffr0VQ/YLp/Pp/Lyci3cEyWv39Gua4fSgYLsji7BlqY+jxs3TjExMR1dTqdFn8OHXodHpPZ5SMGWji7BFmeUpcKR/nbvs8fjUe/evVsVWML2kpBlWVq7dq0efPDBK4YVSerRo4duu+02HTlypMXnnU6nnE5ns/GYmJiQ/cB6/Q55L0ZOYImkX9xLhfJ7iO/R5/Ch1+ERaX2OpPPJpdq7z3bWCtvfYfnwww915MiRFq+Y/FBdXZ2OHj2q5OTkMFQGAABMZzuw1NXVqbq6WtXV1ZKkY8eOqbq6WsePH5ck5eXladasWc32W7NmjTIyMjRkyJBmz/3yl7/Uhx9+qM8//1w7duzQ3/3d3yk6OlozZsywWx4AAOiEbL8ktGfPHo0dOzaw7XK5JEmzZ8/WunXrVFNTEwgvTc6dO6f//u//1qpVq1pc86uvvtKMGTN05swZ3XDDDbrnnnu0c+dO3XDDDXbLAwAAnZDtwDJmzBhd6T7ddevWNRvr3r27GhoaLrtPaWmp3TIAAMA1hM8SAgAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGsx1Ytm3bpsmTJyslJUUOh0ObNm264vzKyko5HI5mD7fbHTSvuLhYAwYMUFxcnDIyMrR79267pQEAgE7KdmCpr6/X8OHDVVxcbGu/w4cPq6amJvDo06dP4LmNGzfK5XIpPz9f+/bt0/Dhw5Wdna2vv/7abnkAAKAT6mJ3h4kTJ2rixIm2v1CfPn3Uo0ePFp9bsWKF5syZo5ycHElSSUmJ3nvvPa1du1ZPP/207a8FAAA6F9uBpa3S09Pl9Xo1ZMgQFRQU6O6775YkNTY2au/evcrLywvMjYqKUlZWlqqqqlpcy+v1yuv1BrY9Ho8kyefzyefztWvdTes5o6x2XTfU2rsPodZUb6TVHWnoc/jQ6/CI1D47oyPrnNJ0DgzVObY1Qh5YkpOTVVJSopEjR8rr9erVV1/VmDFjtGvXLt111106ffq0Ll68qMTExKD9EhMTdejQoRbXLCoq0uLFi5uNb926VfHx8SE5jsKR/pCsGyqbN2/u6BLapLy8vKNLuCbQ5/Ch1+ERaX1ePqqjK2ib9u5zQ0NDq+eGPLAMHDhQAwcODGyPHj1aR48e1Ysvvqj//M//bNOaeXl5crlcgW2Px6PU1FSNHz9eCQkJP7rmS/l8PpWXl2vhnih5/Y52XTuUDhRkd3QJtjT1edy4cYqJienocjot+hw+9Do8IrXPQwq2dHQJtjijLBWO9Ld7n5teIWmNsL0kdKlRo0Zp+/btkqTevXsrOjpatbW1QXNqa2uVlJTU4v5Op1NOp7PZeExMTMh+YL1+h7wXIyewRNIv7qVC+T3E9+hz+NDr8Ii0PkfS+eRS7d1nO2t1yN9hqa6uVnJysiQpNjZWI0aMUEVFReB5v9+viooKZWZmdkR5AADAMLavsNTV1enIkSOB7WPHjqm6ulq9evVSv379lJeXpxMnTuj111+XJK1cuVJpaWkaPHiwvv32W7366qv64IMPtHXr1sAaLpdLs2fP1siRIzVq1CitXLlS9fX1gXcNAQCAa5vtwLJnzx6NHTs2sN10L8ns2bO1bt061dTU6Pjx44HnGxsb9c///M86ceKE4uPjNWzYML3//vtBa0ybNk2nTp3SokWL5Ha7lZ6errKysmY34gIAgGuT7cAyZswYWdbl3461bt26oO358+dr/vz5V103NzdXubm5dssBAADXAD5LCAAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYz3Zg2bZtmyZPnqyUlBQ5HA5t2rTpivPffvttjRs3TjfccIMSEhKUmZmpLVu2BM0pKCiQw+EIegwaNMhuaQAAoJOyHVjq6+s1fPhwFRcXt2r+tm3bNG7cOG3evFl79+7V2LFjNXnyZO3fvz9o3uDBg1VTUxN4bN++3W5pAACgk+pid4eJEydq4sSJrZ6/cuXKoO3nnntO7777rv73f/9Xd9555/eFdOmipKQku+UAAIBrgO3A8mP5/X6dP39evXr1Chr/9NNPlZKSori4OGVmZqqoqEj9+vVrcQ2v1yuv1xvY9ng8kiSfzyefz9eu9Tat54yy2nXdUGvvPoRaU72RVnekoc/hQ6/DI1L77IyOrHNK0zkwVOfY1nBYltXmrjkcDr3zzjuaOnVqq/dZvny5li1bpkOHDqlPnz6SpN///veqq6vTwIEDVVNTo8WLF+vEiRM6cOCAunXr1myNgoICLV68uNn4hg0bFB8f39bDAQAAYdTQ0KCZM2fq3LlzSkhIuOLcsAaWDRs2aM6cOXr33XeVlZV12Xlnz55V//79tWLFCj388MPNnm/pCktqaqpOnz591QO2y+fzqby8XAv3RMnrd7Tr2qF0oCC7o0uwpanP48aNU0xMTEeX02nR5/Ch1+ERqX0eUrDl6pMM4oyyVDjS3+599ng86t27d6sCS9heEiotLdUjjzyit95664phRZJ69Oih2267TUeOHGnxeafTKafT2Ww8JiYmZD+wXr9D3ouRE1gi6Rf3UqH8HuJ79Dl86HV4RFqfI+l8cqn27rOdtcLyd1jefPNN5eTk6M0339SkSZOuOr+urk5Hjx5VcnJyGKoDAACms32Fpa6uLujKx7Fjx1RdXa1evXqpX79+ysvL04kTJ/T6669L+u5loNmzZ2vVqlXKyMiQ2+2WJHXt2lXdu3eXJP3yl7/U5MmT1b9/f508eVL5+fmKjo7WjBkz2uMYAQBAhLN9hWXPnj268847A29JdrlcuvPOO7Vo0SJJUk1NjY4fPx6Y/8orr+jChQuaO3eukpOTA48nnngiMOerr77SjBkzNHDgQD3wwAO6/vrrtXPnTt1www0/9vgAAEAnYPsKy5gxY3Sl+3TXrVsXtF1ZWXnVNUtLS+2WAQAAriF8lhAAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMJ7twLJt2zZNnjxZKSkpcjgc2rRp01X3qays1F133SWn06lbbrlF69atazanuLhYAwYMUFxcnDIyMrR79267pQEAgE7KdmCpr6/X8OHDVVxc3Kr5x44d06RJkzR27FhVV1dr3rx5euSRR7Rly5bAnI0bN8rlcik/P1/79u3T8OHDlZ2dra+//tpueQAAoBPqYneHiRMnauLEia2eX1JSorS0NL3wwguSpNtvv13bt2/Xiy++qOzsbEnSihUrNGfOHOXk5AT2ee+997R27Vo9/fTTdksEAACdjO3AYldVVZWysrKCxrKzszVv3jxJUmNjo/bu3au8vLzA81FRUcrKylJVVVWLa3q9Xnm93sC2x+ORJPl8Pvl8vnatv2k9Z5TVruuGWnv3IdSa6o20uiMNfQ4feh0ekdpnZ3RknVOazoGhOse2RsgDi9vtVmJiYtBYYmKiPB6P/vKXv+ibb77RxYsXW5xz6NChFtcsKirS4sWLm41v3bpV8fHx7Vf8JQpH+kOybqhs3ry5o0tok/Ly8o4u4ZpAn8OHXodHpPV5+aiOrqBt2rvPDQ0NrZ4b8sASCnl5eXK5XIFtj8ej1NRUjR8/XgkJCe36tXw+n8rLy7VwT5S8fke7rh1KBwqyO7oEW5r6PG7cOMXExHR0OZ0WfQ4feh0ekdrnIQVbrj7JIM4oS4Uj/e3e56ZXSFoj5IElKSlJtbW1QWO1tbVKSEhQ165dFR0drejo6BbnJCUltbim0+mU0+lsNh4TExOyH1iv3yHvxcgJLJH0i3upUH4P8T36HD70Ojwirc+RdD65VHv32c5aIf87LJmZmaqoqAgaKy8vV2ZmpiQpNjZWI0aMCJrj9/tVUVERmAMAAK5ttgNLXV2dqqurVV1dLem7ty1XV1fr+PHjkr57uWbWrFmB+b/4xS/02Wefaf78+Tp06JB+85vf6Le//a2efPLJwByXy6XVq1dr/fr1OnjwoB577DHV19cH3jUEAACubbZfEtqzZ4/Gjh0b2G66l2T27Nlat26dampqAuFFktLS0vTee+/pySef1KpVq3TjjTfq1VdfDbylWZKmTZumU6dOadGiRXK73UpPT1dZWVmzG3EBAMC1yXZgGTNmjCzr8m/Haumv2I4ZM0b79++/4rq5ubnKzc21Ww4AALgG8FlCAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4bQosxcXFGjBggOLi4pSRkaHdu3dfdu6YMWPkcDiaPSZNmhSY89BDDzV7fsKECW0pDQAAdEJd7O6wceNGuVwulZSUKCMjQytXrlR2drYOHz6sPn36NJv/9ttvq7GxMbB95swZDR8+XPfff3/QvAkTJui1114LbDudTrulAQCATsr2FZYVK1Zozpw5ysnJ0R133KGSkhLFx8dr7dq1Lc7v1auXkpKSAo/y8nLFx8c3CyxOpzNoXs+ePdt2RAAAoNOxdYWlsbFRe/fuVV5eXmAsKipKWVlZqqqqatUaa9as0fTp03XdddcFjVdWVqpPnz7q2bOn7rvvPi1dulTXX399i2t4vV55vd7AtsfjkST5fD75fD47h3RVTes5o6x2XTfU2rsPodZUb6TVHWnoc/jQ6/CI1D47oyPrnNJ0DgzVObY1HJZltbprJ0+eVN++fbVjxw5lZmYGxufPn68PP/xQu3btuuL+u3fvVkZGhnbt2qVRo0YFxktLSxUfH6+0tDQdPXpUzzzzjH7yk5+oqqpK0dHRzdYpKCjQ4sWLm41v2LBB8fHxrT0cAADQgRoaGjRz5kydO3dOCQkJV5xr+x6WH2PNmjUaOnRoUFiRpOnTpwf+f+jQoRo2bJhuvvlmVVZW6mc/+1mzdfLy8uRyuQLbHo9HqampGj9+/FUP2C6fz6fy8nIt3BMlr9/RrmuH0oGC7I4uwZamPo8bN04xMTEdXU6nRZ/Dh16HR6T2eUjBlo4uwRZnlKXCkf5273PTKyStYSuw9O7dW9HR0aqtrQ0ar62tVVJS0hX3ra+vV2lpqZYsWXLVr3PTTTepd+/eOnLkSIuBxel0tnhTbkxMTMh+YL1+h7wXIyewRNIv7qVC+T3E9+hz+NDr8Ii0PkfS+eRS7d1nO2vZuuk2NjZWI0aMUEVFRWDM7/eroqIi6CWilrz11lvyer36x3/8x6t+na+++kpnzpxRcnKynfIAAEAnZftdQi6XS6tXr9b69et18OBBPfbYY6qvr1dOTo4kadasWUE35TZZs2aNpk6d2uxG2rq6Oj311FPauXOnPv/8c1VUVGjKlCm65ZZblJ0dWS9rAACA0LB9D8u0adN06tQpLVq0SG63W+np6SorK1NiYqIk6fjx44qKCs5Bhw8f1vbt27V169Zm60VHR+tPf/qT1q9fr7NnzyolJUXjx49XYWEhf4sFAABIauNNt7m5ucrNzW3xucrKymZjAwcO1OXejNS1a1dt2RJZNx8BAIDw4rOEAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDx2hRYiouLNWDAAMXFxSkjI0O7d+++7Nx169bJ4XAEPeLi4oLmWJalRYsWKTk5WV27dlVWVpY+/fTTtpQGAAA6IduBZePGjXK5XMrPz9e+ffs0fPhwZWdn6+uvv77sPgkJCaqpqQk8vvjii6Dnly9frpdeekklJSXatWuXrrvuOmVnZ+vbb7+1f0QAAKDTsR1YVqxYoTlz5ignJ0d33HGHSkpKFB8fr7Vr1152H4fDoaSkpMAjMTEx8JxlWVq5cqUWLFigKVOmaNiwYXr99dd18uRJbdq0qU0HBQAAOpcudiY3NjZq7969ysvLC4xFRUUpKytLVVVVl92vrq5O/fv3l9/v11133aXnnntOgwcPliQdO3ZMbrdbWVlZgfndu3dXRkaGqqqqNH369Gbreb1eeb3ewLbH45Ek+Xw++Xw+O4d0VU3rOaOsdl031Nq7D6HWVG+k1R1p6HP40OvwiNQ+O6Mj65zSdA4M1Tm2NWwFltOnT+vixYtBV0gkKTExUYcOHWpxn4EDB2rt2rUaNmyYzp07p+eff16jR4/Wxx9/rBtvvFFutzuwxg/XbHruh4qKirR48eJm41u3blV8fLydQ2q1wpH+kKwbKps3b+7oEtqkvLy8o0u4JtDn8KHX4RFpfV4+qqMraJv27nNDQ0Or59oKLG2RmZmpzMzMwPbo0aN1++236z/+4z9UWFjYpjXz8vLkcrkC2x6PR6mpqRo/frwSEhJ+dM2X8vl8Ki8v18I9UfL6He26digdKMju6BJsaerzuHHjFBMT09HldFr0OXzodXhEap+HFGzp6BJscUZZKhzpb/c+N71C0hq2Akvv3r0VHR2t2traoPHa2lolJSW1ao2YmBjdeeedOnLkiCQF9qutrVVycnLQmunp6S2u4XQ65XQ6W1w7VD+wXr9D3ouRE1gi6Rf3UqH8HuJ79Dl86HV4RFqfI+l8cqn27rOdtWzddBsbG6sRI0aooqIiMOb3+1VRURF0FeVKLl68qI8++igQTtLS0pSUlBS0psfj0a5du1q9JgAA6NxsvyTkcrk0e/ZsjRw5UqNGjdLKlStVX1+vnJwcSdKsWbPUt29fFRUVSZKWLFmiv/7rv9Ytt9yis2fP6t///d/1xRdf6JFHHpH03TuI5s2bp6VLl+rWW29VWlqaFi5cqJSUFE2dOrX9jhQAAEQs24Fl2rRpOnXqlBYtWiS326309HSVlZUFbpo9fvy4oqK+v3DzzTffaM6cOXK73erZs6dGjBihHTt26I477gjMmT9/vurr6/Xoo4/q7Nmzuueee1RWVtbsD8wBAIBrU5tuus3NzVVubm6Lz1VWVgZtv/jii3rxxRevuJ7D4dCSJUu0ZMmStpQDAAA6OT5LCAAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYr02Bpbi4WAMGDFBcXJwyMjK0e/fuy85dvXq1fvrTn6pnz57q2bOnsrKyms1/6KGH5HA4gh4TJkxoS2kAAKATsh1YNm7cKJfLpfz8fO3bt0/Dhw9Xdna2vv766xbnV1ZWasaMGfrDH/6gqqoqpaamavz48Tpx4kTQvAkTJqimpibwePPNN9t2RAAAoNOxHVhWrFihOXPmKCcnR3fccYdKSkoUHx+vtWvXtjj/jTfe0OOPP6709HQNGjRIr776qvx+vyoqKoLmOZ1OJSUlBR49e/Zs2xEBAIBOp4udyY2Njdq7d6/y8vICY1FRUcrKylJVVVWr1mhoaJDP51OvXr2CxisrK9WnTx/17NlT9913n5YuXarrr7++xTW8Xq+8Xm9g2+PxSJJ8Pp98Pp+dQ7qqpvWcUVa7rhtq7d2HUGuqN9LqjjT0OXzodXhEap+d0ZF1Tmk6B4bqHNsaDsuyWt21kydPqm/fvtqxY4cyMzMD4/Pnz9eHH36oXbt2XXWNxx9/XFu2bNHHH3+suLg4SVJpaani4+OVlpamo0eP6plnntFPfvITVVVVKTo6utkaBQUFWrx4cbPxDRs2KD4+vrWHAwAAOlBDQ4Nmzpypc+fOKSEh4YpzbV1h+bGWLVum0tJSVVZWBsKKJE2fPj3w/0OHDtWwYcN08803q7KyUj/72c+arZOXlyeXyxXY9ng8gXtjrnbAdvl8PpWXl2vhnih5/Y52XTuUDhRkd3QJtjT1edy4cYqJienocjot+hw+9Do8IrXPQwq2dHQJtjijLBWO9Ld7n5teIWkNW4Gld+/eio6OVm1tbdB4bW2tkpKSrrjv888/r2XLlun999/XsGHDrjj3pptuUu/evXXkyJEWA4vT6ZTT6Ww2HhMTE7IfWK/fIe/FyAkskfSLe6lQfg/xPfocPvQ6PCKtz5F0PrlUe/fZzlq2brqNjY3ViBEjgm6YbbqB9tKXiH5o+fLlKiwsVFlZmUaOHHnVr/PVV1/pzJkzSk5OtlMeAADopGy/S8jlcmn16tVav369Dh48qMcee0z19fXKycmRJM2aNSvoptx/+7d/08KFC7V27VoNGDBAbrdbbrdbdXV1kqS6ujo99dRT2rlzpz7//HNVVFRoypQpuuWWW5SdHVkvawAAgNCwfQ/LtGnTdOrUKS1atEhut1vp6ekqKytTYmKiJOn48eOKivo+B7388stqbGzU3//93wetk5+fr4KCAkVHR+tPf/qT1q9fr7NnzyolJUXjx49XYWFhiy/7AACAa0+bbrrNzc1Vbm5ui89VVlYGbX/++edXXKtr167asiWybj4CAADhxWcJAQAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjtSmwFBcXa8CAAYqLi1NGRoZ27959xflvvfWWBg0apLi4OA0dOlSbN28Oet6yLC1atEjJycnq2rWrsrKy9Omnn7alNAAA0AnZDiwbN26Uy+VSfn6+9u3bp+HDhys7O1tff/11i/N37NihGTNm6OGHH9b+/fs1depUTZ06VQcOHAjMWb58uV566SWVlJRo165duu6665Sdna1vv/227UcGAAA6DduBZcWKFZozZ45ycnJ0xx13qKSkRPHx8Vq7dm2L81etWqUJEyboqaee0u23367CwkLddddd+vWvfy3pu6srK1eu1IIFCzRlyhQNGzZMr7/+uk6ePKlNmzb9qIMDAACdQxc7kxsbG7V3717l5eUFxqKiopSVlaWqqqoW96mqqpLL5Qoay87ODoSRY8eOye12KysrK/B89+7dlZGRoaqqKk2fPr3Zml6vV16vN7B97tw5SdKf//xn+Xw+O4d0VT6fTw0NDerii9JFv6Nd1w6lM2fOdHQJtjT1+cyZM4qJienocjot+hw+9Do8IrXPXS7Ud3QJtnTxW2po8Ld7n8+fPy/pu4sXV63BzsKnT5/WxYsXlZiYGDSemJioQ4cOtbiP2+1ucb7b7Q483zR2uTk/VFRUpMWLFzcbT0tLa92BXAN6v9DRFQAAOpOZIVz7/Pnz6t69+xXn2AospsjLywu6auP3+/XnP/9Z119/vRyO9r0K4vF4lJqaqi+//FIJCQntuja+R5/Dgz6HD70OD/ocHqHqs2VZOn/+vFJSUq4611Zg6d27t6Kjo1VbWxs0Xltbq6SkpBb3SUpKuuL8pv/W1tYqOTk5aE56enqLazqdTjmdzqCxHj162DkU2xISEvhlCAP6HB70OXzodXjQ5/AIRZ+vdmWlia2bbmNjYzVixAhVVFQExvx+vyoqKpSZmdniPpmZmUHzJam8vDwwPy0tTUlJSUFzPB6Pdu3addk1AQDAtcX2S0Iul0uzZ8/WyJEjNWrUKK1cuVL19fXKycmRJM2aNUt9+/ZVUVGRJOmJJ57QvffeqxdeeEGTJk1SaWmp9uzZo1deeUWS5HA4NG/ePC1dulS33nqr0tLStHDhQqWkpGjq1Kntd6QAACBi2Q4s06ZN06lTp7Ro0SK53W6lp6errKwscNPs8ePHFRX1/YWb0aNHa8OGDVqwYIGeeeYZ3Xrrrdq0aZOGDBkSmDN//nzV19fr0Ucf1dmzZ3XPPfeorKxMcXFx7XCIP47T6VR+fn6zl6DQvuhzeNDn8KHX4UGfw8OEPjus1ryXCAAAoAPxWUIAAMB4BBYAAGA8AgsAADAegQUAABiPwCKpuLhYAwYMUFxcnDIyMrR79+4rzn/rrbc0aNAgxcXFaejQodq8eXOYKo1sdvq8evVq/fSnP1XPnj3Vs2dPZWVlXfX7gu/Y/XluUlpaKofDwZ8TaCW7fT579qzmzp2r5ORkOZ1O3Xbbbfzb0Up2e71y5UoNHDhQXbt2VWpqqp588kl9++23Yao28mzbtk2TJ09WSkqKHA5Hqz54uLKyUnfddZecTqduueUWrVu3LuR1yrrGlZaWWrGxsdbatWutjz/+2JozZ47Vo0cPq7a2tsX5f/zjH63o6Ghr+fLl1ieffGItWLDAiomJsT766KMwVx5Z7PZ55syZVnFxsbV//37r4MGD1kMPPWR1797d+uqrr8JceWSx2+cmx44ds/r27Wv99Kc/taZMmRKeYiOY3T57vV5r5MiR1s9//nNr+/bt1rFjx6zKykqruro6zJVHHru9fuONNyyn02m98cYb1rFjx6wtW7ZYycnJ1pNPPhnmyiPH5s2brWeffdZ6++23LUnWO++8c8X5n332mRUfH2+5XC7rk08+sX71q19Z0dHRVllZWUjrvOYDy6hRo6y5c+cGti9evGilpKRYRUVFLc5/4IEHrEmTJgWNZWRkWP/0T/8U0jojnd0+/9CFCxesbt26WevXrw9ViZ1CW/p84cIFa/To0darr75qzZ49m8DSCnb7/PLLL1s33XST1djYGK4SOw27vZ47d6513333BY25XC7r7rvvDmmdnUVrAsv8+fOtwYMHB41NmzbNys7ODmFllnVNvyTU2NiovXv3KisrKzAWFRWlrKwsVVVVtbhPVVVV0HxJys7Ovux8tK3PP9TQ0CCfz6devXqFqsyI19Y+L1myRH369NHDDz8cjjIjXlv6/D//8z/KzMzU3LlzlZiYqCFDhui5557TxYsXw1V2RGpLr0ePHq29e/cGXjb67LPPtHnzZv385z8PS83Xgo46D0bkpzW3l9OnT+vixYuBv9LbJDExUYcOHWpxH7fb3eJ8t9sdsjojXVv6/EP/8i//opSUlGa/JPheW/q8fft2rVmzRtXV1WGosHNoS58/++wzffDBB/qHf/gHbd68WUeOHNHjjz8un8+n/Pz8cJQdkdrS65kzZ+r06dO65557ZFmWLly4oF/84hd65plnwlHyNeFy50GPx6O//OUv6tq1a0i+7jV9hQWRYdmyZSotLdU777xjxMc1dBbnz5/Xgw8+qNWrV6t3794dXU6n5vf71adPH73yyisaMWKEpk2bpmeffVYlJSUdXVqnU1lZqeeee06/+c1vtG/fPr399tt67733VFhY2NGl4Ue6pq+w9O7dW9HR0aqtrQ0ar62tVVJSUov7JCUl2ZqPtvW5yfPPP69ly5bp/fff17Bhw0JZZsSz2+ejR4/q888/1+TJkwNjfr9fktSlSxcdPnxYN998c2iLjkBt+XlOTk5WTEyMoqOjA2O333673G63GhsbFRsbG9KaI1Vber1w4UI9+OCDeuSRRyRJQ4cODXxW3bPPPhv0WXdom8udBxMSEkJ2dUW6xq+wxMbGasSIEaqoqAiM+f1+VVRUKDMzs8V9MjMzg+ZLUnl5+WXno219lqTly5ersLBQZWVlGjlyZDhKjWh2+zxo0CB99NFHqq6uDjz+9m//VmPHjlV1dbVSU1PDWX7EaMvP8913360jR44EAqEk/d///Z+Sk5MJK1fQll43NDQ0CyVNQdHio/PaRYedB0N6S28EKC0ttZxOp7Vu3Trrk08+sR599FGrR48eltvttizLsh588EHr6aefDsz/4x//aHXp0sV6/vnnrYMHD1r5+fm8rbkV7PZ52bJlVmxsrPVf//VfVk1NTeBx/vz5jjqEiGC3zz/Eu4Rax26fjx8/bnXr1s3Kzc21Dh8+bP3ud7+z+vTpYy1durSjDiFi2O11fn6+1a1bN+vNN9+0PvvsM2vr1q3WzTffbD3wwAMddQjGO3/+vLV//35r//79liRrxYoV1v79+60vvvjCsizLevrpp60HH3wwML/pbc1PPfWUdfDgQau4uJi3NYfLr371K6tfv35WbGysNWrUKGvnzp2B5+69915r9uzZQfN/+9vfWrfddpsVGxtrDR482HrvvffCXHFkstPn/v37W5KaPfLz88NfeISx+/N8KQJL69nt844dO6yMjAzL6XRaN910k/Wv//qv1oULF8JcdWSy02ufz2cVFBRYN998sxUXF2elpqZajz/+uPXNN9+Ev/AI8Yc//KHFf2+b+jp79mzr3nvvbbZPenq6FRsba910003Wa6+9FvI6HZbFNTIAAGC2a/oeFgAAEBkILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAw3v8DDd1H/0hYCgEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "right_retriever = results.loc[results.recall_retriever == 1.0]\n",
    "right_retriever.recall_llm.hist()\n",
    "print(f\"LLM recall when retriever is right: {(right_retriever.recall_llm == 1.0).mean()}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lavague",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}