build helper, take 2 (#3021)

* loud pcm

* fix plugin doccpi_

* build stuff

* more

* full linux

* patchup

* more cmake vars, ipi, gdma manual

* better output notation

* start ecosys

* d4

* pyyaml

* offline conda

* other arch

* another

* iter

* iter 2

* iter 3

* iter 4

* iter 5

* iter 6

* iter 7

* iter 8

* iter 9

* iter 10

* iter 11

* iter 12

* iter 13

* iter 14

* iter 15

* iter 16

* iter 17

* iter 18

* iter 19

* iter 20

* iter 21

* iter 22

* iter 23

* iter 24

* iter 25

* iter 26

* iter 27

* iter 28

* iter 29

* iter 30

* init 31

* init 32

* init 33

* init 34

* init 35

* init 36

* init 37

* init 38

* init 39

* init 40

* init 41

* init 42

* init 43

* iter 44

* iter 45

* iter 46

* iter 47

* iter 48

* iter 49

* iter 50

* iter 51

* iter 52

* iter 53

* iter 54

* iter 55

* iter 56

* iter 57

* iter 58

* iter 59 mac

* iter 60

* iter 61

* iter 62

* iter 63

* iter 64

* iter 65

* iter 66

* iter 67

* iter 68

* iter 69

* iter 70

* iter 71

* info from group debugging

* Update run_psi4.py

* Update azure-pipelines-linux.yml

.azure-pipelines/azure-pipelines-linux.yml

@@ -17,6 +17,7 @@ jobs:
  BUILD_TYPE: 'release'
  APT_INSTALL: 'gfortran gcc-7 g++-7'
  vmImage: 'ubuntu-20.04'
+ PYTEST_MARKER_EXPR: "quick"
 
  gcc_latest:
  CXX_COMPILER: 'g++'
@@ -25,7 +26,8 @@ jobs:
  F_COMPILER: 'gfortran'
  BUILD_TYPE: 'release'
  APT_INSTALL: 'gfortran'
- vmImage: 'ubuntu-20.04'
+ vmImage: 'ubuntu-latest'
+ PYTEST_MARKER_EXPR: 'api and not stdsuite and not (smoke or quick or medlong or long)'
 
  clang_latest:
  CXX_COMPILER: 'clang++'
@@ -35,6 +37,7 @@ jobs:
  BUILD_TYPE: 'release'
  APT_INSTALL: 'gfortran clang-6.0'
  vmImage: 'ubuntu-20.04'
+ PYTEST_MARKER_EXPR: 'cli and not (smoke or quick or medlong or long)'
 
  pool:
  vmImage: $(vmImage)
@@ -79,61 +82,54 @@ jobs:
  - bash: |
  echo "##vso[task.prependpath]$CONDA/bin"
  conda config --set always_yes yes
+ conda config --set solver libmamba
  displayName: "Add Conda to PATH"
 
  - bash: |
- conda create -q \
- -n p4env \
- python=$PYTHON_VER \
- psi4/label/dev::gau2grid \
- psi4/label/testing::libint2=2.7.2=h2fe1556_0 \
- psi4/label/dev::libxc=5 \
- psi4/label/dev::dftd3 \
- psi4/label/dev::gcp \
- psi4/label/dev::pybind11-headers \
- blas=*=mkl \
- mkl-include \
- networkx \
- conda-forge::pytest>=7.0.1 \
- eigen \
- libboost-headers \
- conda-forge::pytest-xdist \
- msgpack-python \
- conda-forge::qcelemental \
- conda-forge::qcengine \
- conda-forge::optking \
- scipy
+ conda install pyyaml -c conda-forge
+ conda/psi4-path-advisor.py \
+ env \
+ --name p4env \
+ --python $PYTHON_VER \
+ --disable compilers addons docs
+ echo "\n<<< env_p4env.yaml >>>\n"
+ cat env_p4env.yaml
+ # test some alternative packages
+ if [[ "${PYTHON_VER}" != "3.12" ]]; then
+ echo " - psi4::gcp" >> env_p4env.yaml
+ echo " - psi4::dftd3" >> env_p4env.yaml
+ fi
+ sed -i "s;- libxc-c;- psi4::libxc=5;g" env_p4env.yaml
+ sed -i "s;- conda-forge/label/libint_dev;#- conda-forge/label/libint_dev;g" env_p4env.yaml
+ sed -i "s;#- psi4/label/testing::libint2;- psi4/label/testing::libint2;g" env_p4env.yaml
+ echo "\n<<< env_p4env.yaml >>>\n"
+ cat env_p4env.yaml
+ conda env create -n p4env -f env_p4env.yaml
  source activate p4env
- which python
+ conda info
  conda list
  displayName: 'Build Environment'
 
  - bash: |
  source activate p4env
  python -V
  python -c 'import numpy; print(numpy.version.version)'
- export CXX=${CXX_COMPILER}
- export CC=${C_COMPILER}
- export FC=${F_COMPILER}
- # * can't use conda dist of the more complicated lib pkgs (e.g., CheMPS2, PCMSolver, v2rdm)
- # b/c their c++ symbols don't mix with the different Travis compilers. for other
- # reasons, pkgs are being compiled less statically, sad for CI.
- # * can't enable trivial plugins b/c no psi4 for them to detect at start (e.g., snsmp2)
- cmake -Bbuild -S. \
- -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
- -DCMAKE_C_COMPILER=${C_COMPILER} \
- -DCMAKE_Fortran_COMPILER=${F_COMPILER} \
- -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
- -DMAX_AM_ERI=5 \
- -DCMAKE_PREFIX_PATH=${CONDA}/envs/p4env \
- -DPython_EXECUTABLE="${CONDA}/envs/p4env/bin/python" \
- -DMPFR_ROOT=${CONDA}/envs/p4env \
- -DCMAKE_INSIST_FIND_PACKAGE_gau2grid=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_Libint2=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_Libxc=ON \
- -DENABLE_PLUGIN_TESTING=ON \
- -DBUILD_Libint2_GENERATOR=OFF \
- -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/Install
+ conda/psi4-path-advisor.py \
+ cmake \
+ --objdir build \
+ --insist
+ echo "\n<<< [email protected] >>>\n"
+ cat [email protected]
+ cmake \
+ -S. \
+ -B build \
+ -C $(Build.SourcesDirectory)/[email protected] \
+ -D CMAKE_CXX_COMPILER=${CXX_COMPILER} \
+ -D CMAKE_C_COMPILER=${C_COMPILER} \
+ -D CMAKE_Fortran_COMPILER=${F_COMPILER} \
+ -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+ -D ENABLE_PLUGIN_TESTING=ON \
+ -D CMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/Install
  displayName: 'Configure Build'
 
  - bash: |
@@ -160,5 +156,5 @@ jobs:
  - bash: |
  cd build
  source activate p4env
- PYTHONPATH=stage/lib/ pytest -v -rws --durations=15 -n 2 -m 'quick' stage/lib/psi4/tests/
+ PYTHONPATH=stage/lib/ pytest -v -rws --durations=15 --durations-min=40 --strict-markers --color yes -n 2 -m "$PYTEST_MARKER_EXPR" stage/lib/psi4/tests/
  displayName: 'PyTest Tests'

.azure-pipelines/azure-pipelines-windows.yml

@@ -107,11 +107,13 @@ jobs:
 
  # Configure Miniconda
  # conda config --append channels conda-forge
+ # conda=23.10 forced a bugfix at conda-build stage
  - script: |
  conda config --set always_yes yes
  conda config --add channels conda-forge/label/libint_dev
  conda config --set channel_priority strict
  conda config --set solver libmamba
+ conda install conda=23.10 -n base -c conda-forge
  conda info
  conda list
  displayName: "Configure Miniconda"
@@ -123,6 +125,7 @@ jobs:
  conda install cmake ^
  anaconda-project ^
  conda-build ^
+ m2-sed ^
  eigen ^
  libboost-headers ^
  conda-verify ^

.github/workflows/docs-pr.yml

@@ -7,8 +7,13 @@ on:
 jobs:
 
  docs:
- runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ cfg:
+ - runs-on: ubuntu-latest
+ python-version: "3.10"
  name: Propose Docs
+ runs-on: ${{ matrix.cfg.runs-on }}
  defaults:
  run:
  shell: bash -l {0}
@@ -31,17 +36,37 @@ jobs:
  path: docs
  ref: master
 
+ - name: Write Conda environment files
+ run: |
+ echo "::group::Prepare OS Python"
+ which python
+ python --version
+ pip install PyYAML
+ echo "::endgroup::"
+ #
+ code/conda/psi4-path-advisor.py \
+ env \
+ --name p4docs \
+ --python ${{ matrix.cfg.python-version }} \
+ --disable addons \
+ --offline-conda
+ #
+ echo "::group::View Env Spec File for Conda"
+ printf "\n<<< env_p4docs.yaml >>>\n"
+ pygmentize -g -O style=friendly,linenos=1 env_p4docs.yaml
+ echo "::endgroup::"
+
  - name: Create Conda Environment
  if: ${{ github.repository == 'psi4/psi4' }}
  uses: conda-incubator/setup-miniconda@v2
  with:
  miniforge-variant: Mambaforge
  use-mamba: true
- activate-environment: test
+ activate-environment: p4docs
  add-pip-as-python-dependency: true
  auto-activate-base: false
  channels: conda-forge
- environment-file: code/devtools/conda-envs/docs-cf.yaml
+ environment-file: env_p4docs.yaml
  show-channel-urls: true
 
  - name: Environment Information
@@ -51,47 +76,39 @@ jobs:
  conda list
  which conda python cmake $CXX
 
- # This patch and the jinja2<3 restriction were needed as recently as March 2023, but when I try a py311 env with newest everything, both can be released.
- #- name: Patch Sphinx for Pybind11
- # if: ${{ github.repository == 'psi4/psi4' }}
- # run: sed -i "s/app\.add_autodoc_attrgetter/# app.add_autodoc_attrgetter/g" ${CONDA_PREFIX}/lib/python3.10/site-packages/sphinx_automodapi/autodoc_enhancements.py
-
- # docs are not Ninja ready
+ # docs are finally Ninja ready
  - name: Configure with CMake (Conda Gnu + MKL)
  if: ${{ github.repository == 'psi4/psi4' }}
  working-directory: ./code
  run: |
+ conda/psi4-path-advisor.py \
+ cache \
+ --objdir objdir \
+ --insist
+ echo "::group::View Cache File for CMake"
+ printf "\n<<< [email protected] >>>\n"
+ pygmentize -g -O style=colorful,linenos=1 [email protected]
+ echo "::endgroup::"
+ #
  cmake \
  -S. \
- -Bobjdir \
- -G"Unix Makefiles" \
- -DCMAKE_BUILD_TYPE=Debug \
- -DCMAKE_C_COMPILER=${CC} \
- -DCMAKE_CXX_COMPILER=${CXX} \
- -DCMAKE_CXX_FLAGS="-O0" \
- -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \
- -DCMAKE_INSIST_FIND_PACKAGE_gau2grid=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_Libint2=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_pybind11=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_Libxc=ON \
- -DENABLE_ecpint=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_ecpint=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_optking=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_qcelemental=ON \
- -DCMAKE_INSIST_FIND_PACKAGE_qcengine=ON \
- -DSPHINXMAN_STRICT=ON \
- -DSPHINX_ROOT=${CONDA_PREFIX}
+ -B objdir \
+ -G Ninja \
+ -C ${{ github.workspace }}/code/[email protected] \
+ -D CMAKE_BUILD_TYPE=Debug \
+ -D CMAKE_CXX_FLAGS="-O0" \
+ -D SPHINXMAN_STRICT=ON
 
  - name: Compile Psi4
  if: ${{ github.repository == 'psi4/psi4' }}
  working-directory: ./code
- run: cmake --build objdir -j`getconf _NPROCESSORS_ONLN`
+ run: cmake --build objdir
 
  - name: Compile & Pack Docs
  if: ${{ github.repository == 'psi4/psi4' }}
  working-directory: ./code
  run: |
- cmake --build objdir --target sphinxman -j`getconf _NPROCESSORS_ONLN`
+ cmake --build objdir --target sphinxman
  cd objdir/doc/sphinxman
  tar -zcf sphinxman.tar.gz html/