CI Build doc on CircleCI (scikit-learn#25466)

.circleci/config.yml

@@ -1,38 +1,78 @@
 version: 2.1
 
-# Parameters required to trigger the execution
-# of the "doc-min-dependencies" and "doc" jobs
-parameters:
- GITHUB_RUN_URL:
- type: string
- default: "none"
-
 jobs:
  doc-min-dependencies:
  docker:
  - image: cimg/python:3.8.12
  environment:
- - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip
+ - MKL_NUM_THREADS: 2
+ - OPENBLAS_NUM_THREADS: 2
+ - CONDA_ENV_NAME: testenv
+ - LOCK_FILE: build_tools/circle/doc_min_dependencies_linux-64_conda.lock
+ # Sphinx race condition in doc-min-dependencies is causing job to stall
+ # Here we run the job serially
+ - SPHINX_NUMJOBS: 1
  steps:
  - checkout
- - run: bash build_tools/circle/download_documentation.sh
+ - run: ./build_tools/circle/checkout_merge_commit.sh
+ - restore_cache:
+ key: v1-doc-min-deps-datasets-{{ .Branch }}
+ - restore_cache:
+ keys:
+ - doc-min-deps-ccache-{{ .Branch }}
+ - doc-min-deps-ccache
+ - run: ./build_tools/circle/build_doc.sh
+ - save_cache:
+ key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }}
+ paths:
+ - ~/.ccache
+ - ~/.cache/pip
+ - save_cache:
+ key: v1-doc-min-deps-datasets-{{ .Branch }}
+ paths:
+ - ~/scikit_learn_data
  - store_artifacts:
  path: doc/_build/html/stable
  destination: doc
+ - store_artifacts:
+ path: ~/log.txt
+ destination: log.txt
 
  doc:
  docker:
  - image: cimg/python:3.8.12
  environment:
- - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip
+ - MKL_NUM_THREADS: 2
+ - OPENBLAS_NUM_THREADS: 2
+ - CONDA_ENV_NAME: testenv
+ - LOCK_FILE: build_tools/circle/doc_linux-64_conda.lock
  steps:
  - checkout
- - run: bash build_tools/circle/download_documentation.sh
+ - run: ./build_tools/circle/checkout_merge_commit.sh
+ - restore_cache:
+ key: v1-doc-datasets-{{ .Branch }}
+ - restore_cache:
+ keys:
+ - doc-ccache-{{ .Branch }}
+ - doc-ccache
+ - run: ./build_tools/circle/build_doc.sh
+ - save_cache:
+ key: doc-ccache-{{ .Branch }}-{{ .BuildNum }}
+ paths:
+ - ~/.ccache
+ - ~/.cache/pip
+ - save_cache:
+ key: v1-doc-datasets-{{ .Branch }}
+ paths:
+ - ~/scikit_learn_data
  - store_artifacts:
  path: doc/_build/html/stable
  destination: doc
- # Persists the generated documentation, so that it
- # can be attached and deployed in the "deploy" job
+ - store_artifacts:
+ path: ~/log.txt
+ destination: log.txt
+ # Persists generated documentation so that it can be attached and deployed
+ # in the 'deploy' step.
  - persist_to_workspace:
  root: doc/_build/html
  paths: .
@@ -54,36 +94,12 @@ jobs:
  bash build_tools/circle/push_doc.sh doc/_build/html/stable
  fi
 
- # This noop job is required for the pipeline to exist, so that the
- # documentation related jobs can be triggered.
- noop:
- docker:
- - image: cimg/python:3.8.12
- steps:
- - run: |
- echo "This is no-op job for the pipeline to exist, so that it triggers "
- echo "Circle CI jobs pushing the artifacts of the documentation built "
- echo "via GitHub actions."
-
 workflows:
  version: 2
-
  build-doc-and-deploy:
- when:
- not:
- equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ]
- # The jobs should run only when triggered by the workflow
  jobs:
- - doc-min-dependencies
  - doc
+ - doc-min-dependencies
  - deploy:
  requires:
  - doc
-
- noop:
- when:
- equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ]
- # Prevent double execution of this job: on push
- # by default and when triggered by the workflow
- jobs:
- - noop

build_tools/github/build_doc.sh → build_tools/circle/build_doc.sh

@@ -16,9 +16,12 @@ set -e
 # If the inspection of the current commit fails for any reason, the default
 # behavior is to quick build the documentation.
 
+# defines the get_dep and show_installed_libraries functions
+source build_tools/shared.sh
+
 if [ -n "$GITHUB_ACTION" ]
 then
- # Map the variables for the new documentation builder to the old one
+ # Map the variables from Github Action to CircleCI
  CIRCLE_SHA1=$(git log -1 --pretty=format:%H)
 
  CIRCLE_JOB=$GITHUB_JOB
@@ -169,11 +172,12 @@ ccache -M 512M
 export CCACHE_COMPRESS=1
 
 # pin conda-lock to latest released version (needs manual update from time to time)
-mamba install conda-lock==1.0.5 -y
+mamba install "$(get_dep conda-lock min)" -y
+
 conda-lock install --log-level WARNING --name $CONDA_ENV_NAME $LOCK_FILE
 source activate $CONDA_ENV_NAME
 
-mamba list
+show_installed_libraries
 
 # Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI
 # workers with 2 cores when building the compiled extensions of scikit-learn.
@@ -185,12 +189,18 @@ ccache -s
 
 export OMP_NUM_THREADS=1
 
+# Avoid CI job getting killed because it uses too much memory
+if [[ -z $SPHINX_NUMJOBS ]]; then
+ export SPHINX_NUMJOBS=2
+fi
+
 if [[ "$CIRCLE_BRANCH" =~ ^main$ && -z "$CI_PULL_REQUEST" ]]
 then
  # List available documentation versions if on main
  python build_tools/circle/list_versions.py > doc/versions.rst
 fi
 
+
 # The pipefail is requested to propagate exit code
 set -o pipefail && cd doc && make $make_args 2>&1 | tee ~/log.txt
 

build_tools/update_environments_and_lock_files.py

@@ -251,7 +251,7 @@ def remove_from(alist, to_remove):
  },
  {
  "build_name": "doc_min_dependencies",
- "folder": "build_tools/github",
+ "folder": "build_tools/circle",
  "platform": "linux-64",
  "channel": "conda-forge",
  "conda_dependencies": common_dependencies_without_coverage
@@ -286,7 +286,7 @@ def remove_from(alist, to_remove):
  },
  {
  "build_name": "doc",
- "folder": "build_tools/github",
+ "folder": "build_tools/circle",
  "platform": "linux-64",
  "channel": "conda-forge",
  "conda_dependencies": common_dependencies_without_coverage