From 07cf3c614a1bcb97531d233dd933d750c689a5d1 Mon Sep 17 00:00:00 2001 From: Sara Zan Date: Fri, 28 Jan 2022 17:21:23 +0100 Subject: [PATCH] Disable cache on the CI (#2083) * Disable cache on the CI * Reintroduce paths * Add most files to the cache key * remove date and path from cache key * Try double install with cache * Try to cache more stuff, on a per-commit basis * Fix windows CI too * Add comment on how to speed up the CI with better caching --- .github/workflows/linux_ci.yml | 22 +++++++++++++++------- .github/workflows/windows_ci.yml | 16 +++++++++++----- haystack/__init__.py | 1 + setup.cfg | 3 +++ 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.github/workflows/linux_ci.yml b/.github/workflows/linux_ci.yml index 98a63a6874..b0eee08cd0 100644 --- a/.github/workflows/linux_ci.yml +++ b/.github/workflows/linux_ci.yml @@ -28,18 +28,23 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7 - - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - name: Cache id: cache-python-env uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: linux-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + # The cache currently misses at every change in the Python files to ensure + # that the code changes to Haystack are not being cached along with the dependencies. + # TODO for the tests refactoring: We could speed up the process by caching only + # the dependencies and installing Haystack again separately for each test runner. + key: linux-${{ hashFiles('**/*.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + - name: Install dependencies if: steps.cache-python-env.outputs.cache-hit != 'true' run: | - python -m pip install --upgrade pip - pip install --upgrade --upgrade-strategy eager -e .[test] + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager .[test] pip install torch-scatter -f https://data.pyg.org/whl/torch-1.10.0+cpu.html prepare-build: @@ -52,6 +57,7 @@ jobs: echo "::set-output name=matrix::$(find $(find . -type d -name test -not -path "./*env*/*") -type f -name test_*.py | jq -SR . | jq -cs .)" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} + build: needs: prepare-build runs-on: ubuntu-20.04 @@ -61,16 +67,18 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v2 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: python-version: 3.7 - - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - - name: Cache + + - name: Cache Python uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: linux-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + key: linux-${{ hashFiles('**/*.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + - name: Run Elasticsearch run: docker run -d -p 9200:9200 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms128m -Xmx128m" elasticsearch:7.9.2 diff --git a/.github/workflows/windows_ci.yml b/.github/workflows/windows_ci.yml index e4d9a1f189..5b7e1581f1 100644 --- a/.github/workflows/windows_ci.yml +++ b/.github/workflows/windows_ci.yml @@ -29,21 +29,27 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7 - - run: echo "date=$(date +'%Y-%m-%d')" >> $env:GITHUB_ENV + - name: Cache id: cache-python-env uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: windows-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + # The cache currently misses at every change in the Python files to ensure + # that the code changes to Haystack are not being cached along with the dependencies. + # TODO for the tests refactoring: We could speed up the process by caching only + # the dependencies and installing Haystack again separately for each test runner. + key: windows-${{ hashFiles('**/*.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + - name: Install Pytorch on windows run: | pip install torch==1.8.1+cpu -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + - name: Install dependencies if: steps.cache-python-env.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip - pip install --upgrade --upgrade-strategy eager -e .[test] + pip install --upgrade --upgrade-strategy eager .[test] pip install torch-scatter -f https://data.pyg.org/whl/torch-1.10.0+cpu.html prepare-build: @@ -71,12 +77,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.7 - - run: echo "date=$(date +'%Y-%m-%d')" >> $env:GITHUB_ENV + - name: Cache uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: windows-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} + key: windows-${{ hashFiles('**/*.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }} # Windows runner can't run Linux containers. Refer https://github.com/actions/virtual-environments/issues/1143 - name: Set up Windows test env diff --git a/haystack/__init__.py b/haystack/__init__.py index 0af1eded18..7b593a9eaf 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -9,6 +9,7 @@ __version__ = metadata.version('farm-haystack') + # This configuration must be done before any import to apply to all submodules import logging logging.basicConfig(format="%(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.WARNING) diff --git a/setup.cfg b/setup.cfg index aa63d9950c..f9018d898e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,9 @@ classifiers = use_scm_version = True python_requires = >=3.7 packages = find: +setup_requires = + setuptools + wheel install_requires = importlib-metadata; python_version < '3.8' torch>1.9,<1.11