refactor tutorial 06 to make it testable (deepset-ai#11)

blancadesal · Sep 16, 2022 · a708730 · a708730
1 parent 868c7de
commit a708730
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 61 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -26,6 +26,7 @@ jobs:
  - 03_Basic_QA_Pipeline_without_Elasticsearch
  - 04_FAQ_style_QA
  - 05_Evaluation
+ - 06_Better_Retrieval_via_Embedding_Retrieval
 
  env:
  ELASTICSEARCH_HOST: "elasticsearch"

diff --git a/markdowns/6.md b/markdowns/6.md
@@ -59,20 +59,23 @@ Make sure you enable the GPU runtime to experience decent speed in this tutorial
 
 <img src="https://raw.githubusercontent.com/deepset-ai/haystack/main/docs/img/colab_gpu_runtime.jpg">
 
+You can double check whether the GPU runtime is enabled with the following command:
 
-```python
-# Make sure you have a GPU running
-!nvidia-smi
+
+```bash
+%%bash
+
+nvidia-smi
 ```
 
+To start, install the latest release of Haystack with `pip`:
 
-```python
-# Install the latest release of Haystack in your own environment
-#! pip install farm-haystack
 
-# Install the latest main of Haystack
-!pip install --upgrade pip
-!pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,faiss]
+```bash
+%%bash
+
+pip install --upgrade pip
+pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,faiss]
 ```
 
 ## Logging
@@ -90,12 +93,6 @@ logging.basicConfig(format="%(levelname)s - %(name)s - %(message)s", level=logg
 logging.getLogger("haystack").setLevel(logging.INFO)
 ```
 
-
-```python
-from haystack.utils import clean_wiki_text, convert_files_to_docs, fetch_archive_from_http, print_answers
-from haystack.nodes import FARMReader, TransformersReader
-```
-
 ### Document Store
 
 #### Option 1: FAISS
@@ -143,6 +140,9 @@ Similarly to the previous tutorials, we download, convert and index some Game of
 
 
 ```python
+from haystack.utils import clean_wiki_text, convert_files_to_docs, fetch_archive_from_http
+
+
 # Let's first get some files that we want to use
 doc_dir = "data/tutorial6"
 s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt6.zip"
@@ -196,9 +196,11 @@ Here we use a FARMReader with the *deepset/roberta-base-squad2* model (see: http
 
 
 ```python
+from haystack.nodes import FARMReader
+
+
 # Load a local model or any of the QA models on
 # Hugging Face's model hub (https://huggingface.co/models)
-
 reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True)
 ```
 
@@ -229,6 +231,9 @@ prediction = pipe.run(
 
 
 ```python
+from haystack.utils import print_answers
+
+
 print_answers(prediction, details="minimum")
 ```
 

diff --git a/tutorials/06_Better_Retrieval_via_Embedding_Retrieval.ipynb b/tutorials/06_Better_Retrieval_via_Embedding_Retrieval.ipynb
@@ -62,82 +62,84 @@
  "Make sure you enable the GPU runtime to experience decent speed in this tutorial.\n",
  "**Runtime -> Change Runtime type -> Hardware accelerator -> GPU**\n",
  "\n",
- "<img src=\"https://raw.githubusercontent.com/deepset-ai/haystack/main/docs/img/colab_gpu_runtime.jpg\">"
+ "<img src=\"https://raw.githubusercontent.com/deepset-ai/haystack/main/docs/img/colab_gpu_runtime.jpg\">\n",
+ "\n",
+ "You can double check whether the GPU runtime is enabled with the following command:"
  ]
  },
  {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
- "id": "JlZgP8q1A6NW"
+ "id": "JlZgP8q1A6NW",
+ "vscode": {
+ "languageId": "shellscript"
+ }
  },
  "outputs": [],
  "source": [
- "# Make sure you have a GPU running\n",
- "!nvidia-smi"
+ "%%bash\n",
+ "\n",
+ "nvidia-smi"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "To start, install the latest release of Haystack with `pip`:"
  ]
  },
  {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
- "id": "NM36kbRFA6Nc"
+ "id": "NM36kbRFA6Nc",
+ "vscode": {
+ "languageId": "shellscript"
+ }
  },
  "outputs": [],
  "source": [
- "# Install the latest release of Haystack in your own environment\n",
- "#! pip install farm-haystack\n",
+ "%%bash\n",
  "\n",
- "# Install the latest main of Haystack\n",
- "!pip install --upgrade pip\n",
- "!pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,faiss]"
+ "pip install --upgrade pip\n",
+ "pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,faiss]"
  ]
  },
  {
  "cell_type": "markdown",
+ "metadata": {
+ "collapsed": false,
+ "id": "GbM2ml-ozqLX",
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
  "source": [
  "## Logging\n",
  "\n",
  "We configure how logging messages should be displayed and which log level should be used before importing Haystack.\n",
  "Example log message:\n",
  "INFO - haystack.utils.preprocessing - Converting data/tutorial1/218_Olenna_Tyrell.txt\n",
  "Default log level in basicConfig is WARNING so the explicit parameter is not necessary but can be changed easily:"
- ],
- "metadata": {
- "collapsed": false,
- "pycharm": {
- "name": "#%% md\n"
- },
- "id": "GbM2ml-ozqLX"
- }
+ ]
  },
  {
  "cell_type": "code",
  "execution_count": null,
- "outputs": [],
- "source": [
- "import logging\n",
- "\n",
- "logging.basicConfig(format=\"%(levelname)s - %(name)s - %(message)s\", level=logging.WARNING)\n",
- "logging.getLogger(\"haystack\").setLevel(logging.INFO)"
- ],
  "metadata": {
+ "id": "kQWEUUMnzqLX",
  "pycharm": {
  "name": "#%%\n"
- },
- "id": "kQWEUUMnzqLX"
- }
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "xmRuhTQ7A6Nh"
+ }
  },
  "outputs": [],
  "source": [
- "from haystack.utils import clean_wiki_text, convert_files_to_docs, fetch_archive_from_http, print_answers\n",
- "from haystack.nodes import FARMReader, TransformersReader"
+ "import logging\n",
+ "\n",
+ "logging.basicConfig(format=\"%(levelname)s - %(name)s - %(message)s\", level=logging.WARNING)\n",
+ "logging.getLogger(\"haystack\").setLevel(logging.INFO)"
  ]
  },
  {
@@ -179,10 +181,10 @@
  "cell_type": "markdown",
  "metadata": {
  "collapsed": false,
+ "id": "s4HK5l0qzqLZ",
  "pycharm": {
  "name": "#%% md\n"
- },
- "id": "s4HK5l0qzqLZ"
+ }
  },
  "source": [
  "#### Option 2: Milvus\n",
@@ -197,10 +199,10 @@
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
+ "id": "2Ur4h-E3zqLZ",
  "pycharm": {
  "name": "#%%\n"
- },
- "id": "2Ur4h-E3zqLZ"
+ }
  },
  "outputs": [],
  "source": [
@@ -242,6 +244,9 @@
  },
  "outputs": [],
  "source": [
+ "from haystack.utils import clean_wiki_text, convert_files_to_docs, fetch_archive_from_http\n",
+ "\n",
+ "\n",
  "# Let's first get some files that we want to use\n",
  "doc_dir = \"data/tutorial6\"\n",
  "s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt6.zip\"\n",
@@ -324,9 +329,11 @@
  },
  "outputs": [],
  "source": [
+ "from haystack.nodes import FARMReader\n",
+ "\n",
+ "\n",
  "# Load a local model or any of the QA models on\n",
  "# Hugging Face's model hub (https://huggingface.co/models)\n",
- "\n",
  "reader = FARMReader(model_name_or_path=\"deepset/roberta-base-squad2\", use_gpu=True)"
  ]
  },
@@ -389,6 +396,9 @@
  },
  "outputs": [],
  "source": [
+ "from haystack.utils import print_answers\n",
+ "\n",
+ "\n",
  "print_answers(prediction, details=\"minimum\")"
  ]
  },
@@ -426,8 +436,9 @@
  "name": "Tutorial6_Better_Retrieval_via_Embedding_Retrieval.ipynb",
  "provenance": []
  },
+ "gpuClass": "standard",
  "kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3.10.6 64-bit",
  "language": "python",
  "name": "python3"
  },
@@ -441,10 +452,14 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.6.9"
+ "version": "3.10.6"
  },
- "gpuClass": "standard"
+ "vscode": {
+ "interpreter": {
+ "hash": "bda33b16be7e844498c7c2d368d72665b4f1d165582b9547ed22a0249a29ca2e"
+ }
+ }
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}