Merge branch 'main' into main

.github/actions/send_failure/action.yml

@@ -0,0 +1,28 @@
+name: "Send failure event to Datadog"
+inputs:
+ api-key:
+ description: "Datadog API key"
+ required: true
+ title:
+ description: "Custom title for the event"
+ required: true 
+runs:
+ using: "composite"
+ steps:
+ - uses: masci/datadog@v1
+ with:
+ api-key: ${{ inputs.api-key }}
+ api-url: https://api.datadoghq.eu
+ events: |
+ - title: "${{ inputs.title }}"
+ text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
+ alert_type: "error"
+ source_type_name: "Github"
+ host: ${{ github.repository_owner }}
+ tags:
+ - "project:${{ github.repository }}"
+ - "job:${{ github.job }}"
+ - "run_id:${{ github.run_id }}"
+ - "workflow:${{ github.workflow }}"
+ - "branch:${{ github.ref_name }}"
+ - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/CI_project.yml

@@ -10,7 +10,7 @@ jobs:
  name: Add new issues to project for triage
  runs-on: ubuntu-latest
  steps:
- - uses: actions/add-to-project@v0.5.0
+ - uses: actions/add-to-project@v1.0.0
  with:
  project-url: https://github.com/orgs/deepset-ai/projects/5
  github-token: ${{ secrets.GH_PROJECT_PAT }}

.github/workflows/CI_readme_sync.yml

@@ -2,18 +2,25 @@ name: Core / Sync docs with Readme
 
 on:
  push:
- branches:
- - main
+ tags:
+ - "**-v[0-9].[0-9]+.[0-9]+"
+ workflow_dispatch: # Activate this workflow manually
+ inputs:
+ tag:
+ description: "Tag with this format: integrations/<INTEGRATION_FOLDER_NAME>-v1.0.0"
+ required: true
+ type: string
+ default: integrations/<INTEGRATION_FOLDER_NAME>-v1.0.0
+
+env:
+ TAG: ${{ inputs.tag || github.ref_name }}
 
 jobs:
  sync:
  runs-on: ubuntu-latest
  steps:
  - name: Checkout this repo
  uses: actions/checkout@v4
- with:
- # This will tell tj-actions/changed-files to compare the current pushed commit with the latest in main
- fetch-depth: 0
 
  - name: Set up Python
  uses: actions/setup-python@v5
@@ -25,39 +32,29 @@ jobs:
  python -m pip install --upgrade pip
  pip install -U haystack-pydoc-tools hatch
 
- # We look into the changeset in order to understand
- # which integrations were modified by the last commit.
- - name: Get changed files
- id: changed-files
- uses: tj-actions/changed-files@v42
- with:
- files: integrations/**
- # Output unique changed directories instead of filenames
- dir_names: true
- # We only care about the name of the integration, i.e. integrations/FOO
- dir_names_max_depth: 2
+ - name: Get project folder
+ id: pathfinder
+ shell: python
+ run: |
+ import os
+ project_path = os.environ["TAG"].rsplit("-", maxsplit=1)[0]
+ with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+ print(f'project_path={project_path}', file=f) 
 
  - name: Generate docs
- if: steps.changed-files.outputs.all_changed_files != ''
+ working-directory: ${{ steps.pathfinder.outputs.project_path }}
  env:
  # This is necessary to fetch the documentation categories
  # from Readme.io as we need them to associate the slug
  # in config files with their id.
  README_API_KEY: ${{ secrets.README_API_KEY }}
- ALL_CHANGED_DIRS: ${{ steps.changed-files.outputs.all_changed_files }}
  run: |
- for d in $ALL_CHANGED_DIRS; do
- cd $d
- hatch run docs
- hatch env prune # clean up the environment after docs generation
- cd -
- done
+ hatch run docs
  mkdir tmp
  find . -name "_readme_*.md" -exec cp "{}" tmp \;
  ls tmp
 
  - name: Sync API docs
- if: steps.changed-files.outputs.all_changed_files != ''
  uses: readmeio/rdme@v8
  with:
- rdme: docs ./tmp --key=${{ secrets.README_API_KEY }} --version=2.0
+ rdme: docs ${{ steps.pathfinder.outputs.project_path }}/tmp --key=${{ secrets.README_API_KEY }} --version=2.0

.github/workflows/amazon_bedrock.yml

@@ -68,4 +68,22 @@ jobs:
  role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
 
  - name: Run tests
+ id: tests
  run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/amazon_sagemaker.yml

@@ -57,4 +57,22 @@ jobs:
  run: hatch run docs 
 
  - name: Run tests
+ id: tests
  run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/anthropic.yml

@@ -0,0 +1,74 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / anthropic
+
+on:
+ schedule:
+ - cron: "0 0 * * *"
+ pull_request:
+ paths:
+ - "integrations/anthropic/**"
+ - ".github/workflows/anthropic.yml"
+
+defaults:
+ run:
+ working-directory: integrations/anthropic
+
+concurrency:
+ group: cohere-${{ github.head_ref }}
+ cancel-in-progress: true
+
+env:
+ PYTHONUNBUFFERED: "1"
+ FORCE_COLOR: "1"
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+
+jobs:
+ run:
+ name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest, macos-latest]
+ python-version: ["3.9", "3.10"]
+
+ steps:
+ - name: Support longpaths
+ if: matrix.os == 'windows-latest'
+ working-directory: .
+ run: git config --system core.longpaths true
+
+ - uses: actions/checkout@v4
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install Hatch
+ run: pip install --upgrade hatch
+
+ - name: Lint
+ if: matrix.python-version == '3.9' && runner.os == 'Linux'
+ run: hatch run lint:all
+
+ - name: Run tests
+ run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/astra.yml

@@ -59,6 +59,24 @@ jobs:
 
  - name: Run tests
  env:
- ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_API_ENDPOINT }}
- ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_TOKEN }}
- run: hatch run cov
+ ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }}
+ ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}
+ id: tests
+ run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure:
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/chroma.yml

@@ -57,4 +57,22 @@ jobs:
  run: hatch run docs
 
  - name: Run tests
+ id: tests
  run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/cohere.yml

@@ -53,5 +53,27 @@ jobs:
  if: matrix.python-version == '3.9' && runner.os == 'Linux'
  run: hatch run lint:all
 
+ - name: Generate docs
+ if: matrix.python-version == '3.9' && runner.os == 'Linux'
+ run: hatch run docs 
+
  - name: Run tests
+ id: tests
  run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/deepeval.yml

@@ -21,6 +21,7 @@ concurrency:
 env:
  PYTHONUNBUFFERED: "1"
  FORCE_COLOR: "1"
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
 jobs:
  run:
@@ -57,4 +58,22 @@ jobs:
  run: hatch run docs
 
  - name: Run tests
+ id: tests
  run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/elasticsearch.yml

@@ -55,4 +55,22 @@ jobs:
  run: hatch run docs
 
  - name: Run tests
+ id: tests
  run: hatch run cov
+
+ - name: Nightly - run unit tests with Haystack main branch
+ if: github.event_name == 'schedule'
+ id: nightly-haystack-main
+ run: |
+ hatch run pip install git+https://github.com/deepset-ai/haystack.git
+ hatch run test -m "not integration"
+ 
+ - name: Send event to Datadog for nightly failures
+ if: failure() && github.event_name == 'schedule'
+ uses: ./.github/actions/send_failure
+ with:
+ title: |
+ core-integrations failure: 
+ ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+ - ${{ github.workflow }}
+ api-key: ${{ secrets.CORE_DATADOG_API_KEY }}