Run Tutorials Nightly #692
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Tutorials Nightly | |
| on: | |
| workflow_dispatch: # Activate this workflow manually | |
| schedule: | |
| - cron: "0 0 * * *" | |
| jobs: | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix_v1: ${{ steps.generator.outputs.matrix_v1 }} | |
| matrix_v2: ${{ steps.generator.outputs.matrix_v2 }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - id: generator | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Get tutorial notebooks for v1 | |
| VERSION=$(gh api /repos/deepset-ai/haystack/releases | \ | |
| jq -r '[.[].tag_name | select(test("^v1.[0-9]+.[0-9]+$"))] | first') | |
| NOTEBOOKS=$(python ./scripts/generate_matrix.py --haystack-version "$VERSION" --include-main) | |
| echo "matrix_v1={\"include\":$NOTEBOOKS}" >> "$GITHUB_OUTPUT" | |
| # Get tutorial notebooks for v2 | |
| VERSION=$(gh api /repos/deepset-ai/haystack/releases | \ | |
| jq -r '[.[].tag_name | select(test("^v2.[0-9]+.[0-9]+$"))] | first') | |
| NOTEBOOKS=$(python ./scripts/generate_matrix.py --haystack-version "$VERSION" --include-main) | |
| echo "matrix_v2={\"include\":$NOTEBOOKS}" >> "$GITHUB_OUTPUT" | |
| run-tutorials-v1: | |
| needs: generate-matrix | |
| runs-on: ubuntu-latest | |
| container: deepset/haystack:base-cpu-${{ matrix.haystack_version }} | |
| services: | |
| elasticsearch: | |
| image: elasticsearch:7.9.2 | |
| env: | |
| discovery.type: "single-node" | |
| ES_JAVA_OPTS: "-Xms128m -Xmx256m" | |
| strategy: | |
| max-parallel: 2 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix_v1) }} | |
| env: | |
| HAYSTACK_TELEMETRY_ENABLED: "False" | |
| ELASTICSEARCH_HOST: "elasticsearch" | |
| HF_API_KEY: ${{ secrets.HF_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| # remove pip install pyzmq when this is resolved https://github.com/zeromq/pyzmq/issues/1764 | |
| run: | | |
| apt-get update && apt-get install -y build-essential gcc libsndfile1 ffmpeg && rm -rf /var/lib/apt/lists/* | |
| pip install pyzmq==23.2.1 | |
| pip install nbconvert ipython | |
| pip install "pyworld<=0.2.12" espnet espnet-model-zoo pydub | |
| pip install farm-haystack[pdf] | |
| pip install scipy --upgrade | |
| pip install farm-haystack-text2speech | |
| pip install "datasets>=2.6.1" | |
| pip install ipywidgets | |
| - name: Convert notebook to Python | |
| run: | | |
| jupyter nbconvert --to python --RegexRemovePreprocessor.patterns '%%bash' ./tutorials/${{ matrix.notebook }}.ipynb | |
| - name: Run the converted notebook | |
| run: | | |
| python ./tutorials/${{ matrix.notebook }}.py | |
| - name: Send Failure to Datadog | |
| if: failure() | |
| uses: masci/datadog@v1 | |
| with: | |
| api-key: ${{ secrets.CORE_DATADOG_API_KEY }} | |
| api-url: https://api.datadoghq.eu | |
| events: | | |
| - title: "Tutorial ${{ matrix.notebook }} failed" | |
| text: "Branch ${{ github.ref_name }} tests failed" | |
| alert_type: "error" | |
| source_type_name: "Github" | |
| host: ${{ github.repository_owner }} | |
| tags: | |
| - "project:${{ github.repository }}" | |
| - "name:${{ matrix.notebook }}" | |
| - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| run-tutorials-v2: | |
| needs: generate-matrix | |
| runs-on: ubuntu-latest | |
| container: deepset/haystack:base-${{ matrix.haystack_version }} | |
| strategy: | |
| max-parallel: 2 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix_v2) }} | |
| env: | |
| HAYSTACK_TELEMETRY_ENABLED: "False" | |
| HF_API_TOKEN: ${{ secrets.HF_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install common dependencies | |
| run: | | |
| apt-get update && apt-get install -y \ | |
| build-essential \ | |
| gcc \ | |
| libsndfile1 \ | |
| ffmpeg | |
| pip install nbconvert ipython | |
| - name: Install tutorial dependencies | |
| if: toJSON(matrix.dependencies) != '[]' | |
| run: | | |
| pip install "${{ join(matrix.dependencies, '" "')}}" | |
| - name: Convert notebook to Python | |
| run: | | |
| jupyter nbconvert --to python --RegexRemovePreprocessor.patterns '%%bash' ./tutorials/${{ matrix.notebook }}.ipynb | |
| - name: Run the converted notebook | |
| run: | | |
| python ./tutorials/${{ matrix.notebook }}.py | |
| - name: Send Failure to Datadog | |
| if: failure() | |
| uses: masci/datadog@v1 | |
| with: | |
| api-key: ${{ secrets.CORE_DATADOG_API_KEY }} | |
| api-url: https://api.datadoghq.eu | |
| events: | | |
| - title: "Tutorial ${{ matrix.notebook }} failed" | |
| text: "Branch ${{ github.ref_name }} tests failed" | |
| alert_type: "error" | |
| source_type_name: "Github" | |
| host: ${{ github.repository_owner }} | |
| tags: | |
| - "project:${{ github.repository }}" | |
| - "name:${{ matrix.notebook }}" | |
| - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" |