ENH: Add Environment Locking (#735)

* ✨ Add environment locking script

* 📝 ✨ Finish script, add documentation

* 🐛 Change Windows env file in workflow

* 📝 🐛 Add review changes + fixes

* 🚧 Temporarily alter tests and conda channels

* 🚧 Add logging

* ✅ Fix TestSubmodule env file

* 🔥 Delete env test

* 🧑‍💻 Add warning to environment.yml

* 📝 ⚰️ Update based on review comments

* 📝 Add final changes

.github/workflows/linting_and_hello_world.yml

@@ -67,7 +67,7 @@ jobs:
  with:
  activate-environment: InnerEye
  auto-activate-base: false
- environment-file: environment.yml
+ environment-file: environment_win.yml
  python-version: 3.7
  if: always()
 

InnerEye/Azure/azure_util.py

@@ -234,6 +234,7 @@ def get_all_environment_files(project_root: Path) -> List[Path]:
  files = [innereye_yaml]
  if innereye_yaml != project_yaml:
  files.append(project_yaml)
+
  return files
 
 

InnerEye/ML/runner.py

@@ -306,6 +306,7 @@ def after_submission_hook(azure_run: Run) -> None:
  # be necessary if the innereye package is installed. It is necessary when working with an outer project
  # and InnerEye as a git submodule and submitting jobs from the local machine.
  # In case of version conflicts, the package version in the outer project is given priority.
+ logging.info(f"Attempting to merge the following conda files: {source_config.conda_dependencies_files}")
  merge_conda_files(source_config.conda_dependencies_files, temp_conda)
 
  # Calls like `self.azure_config.get_workspace()` will fail if we have no AzureML credentials set up, and so

TestSubmodule/environment.yml

@@ -1,7 +1,311 @@
-name: TestSubmodule
+# WARNING - DO NOT EDIT THIS FILE MANUALLY
+# Please refer to the environment documentation for instructions on how to create a new version of this file: https://github.com/microsoft/InnerEye-DeepLearning/blob/main/docs/environment.md
+name: InnerEye
 channels:
- - defaults
  - pytorch
+ - defaults
 dependencies:
- - pip=20.1.1
+ - _libgcc_mutex=0.1=main
+ - _openmp_mutex=5.1=1_gnu
+ - blas=1.0=mkl
+ - blosc=1.21.0=h4ff587b_1
+ - bzip2=1.0.8=h7b6447c_0
+ - ca-certificates=2022.4.26=h06a4308_0
+ - certifi=2022.5.18.1=py37h06a4308_0
+ - cudatoolkit=11.3.1=h2bc3f7f_2
+ - ffmpeg=4.2.2=h20bf706_0
+ - freetype=2.11.0=h70c0345_0
+ - giflib=5.2.1=h7b6447c_0
+ - gmp=6.2.1=h2531618_2
+ - gnutls=3.6.15=he1e5248_0
+ - intel-openmp=2021.4.0=h06a4308_3561
+ - jpeg=9e=h7f8727e_0
+ - lame=3.100=h7b6447c_0
+ - lcms2=2.12=h3be6417_0
+ - libedit=3.1.20210910=h7f8727e_0
+ - libffi=3.2.1=hf484d3e_1007
+ - libgcc-ng=11.2.0=h1234567_0
+ - libgomp=11.2.0=h1234567_0
+ - libidn2=2.3.2=h7f8727e_0
+ - libopus=1.3.1=h7b6447c_0
+ - libpng=1.6.37=hbc83047_0
+ - libstdcxx-ng=11.2.0=h1234567_0
+ - libtasn1=4.16.0=h27cfd23_0
+ - libtiff=4.2.0=h2818925_1
+ - libunistring=0.9.10=h27cfd23_0
+ - libuv=1.40.0=h7b6447c_0
+ - libvpx=1.7.0=h439df22_0
+ - libwebp=1.2.2=h55f646e_0
+ - libwebp-base=1.2.2=h7f8727e_0
+ - lz4-c=1.9.3=h295c915_1
+ - mkl=2021.4.0=h06a4308_640
+ - mkl-service=2.4.0=py37h7f8727e_0
+ - mkl_fft=1.3.1=py37hd3c417c_0
+ - mkl_random=1.2.2=py37h51133e4_0
+ - ncurses=6.3=h7f8727e_2
+ - nettle=3.7.3=hbbd107a_1
+ - openh264=2.1.1=h4ff587b_0
+ - openssl=1.1.1o=h7f8727e_0
+ - pip=20.1.1=py37_1
  - python=3.7.3
+ - python-blosc=1.7.0=py37h7b6447c_0
+ - pytorch=1.10.0=py3.7_cuda11.3_cudnn8.2.0_0
+ - pytorch-mutex=1.0=cuda
+ - readline=7.0=h7b6447c_5
+ - setuptools=61.2.0=py37h06a4308_0
+ - sqlite=3.33.0=h62c20be_0
+ - tk=8.6.11=h1ccaba5_1
+ - torchvision=0.11.1=py37_cu113
+ - typing_extensions=4.1.1=pyh06a4308_0
+ - wheel=0.37.1=pyhd3eb1b0_0
+ - x264=1!157.20191217=h7b6447c_0
+ - xz=5.2.5=h7f8727e_1
+ - zlib=1.2.12=h7f8727e_2
+ - zstd=1.5.2=ha4553b6_0
+ - pip:
+ - absl-py==1.0.0
+ - adal==1.2.7
+ - aiohttp==3.8.1
+ - aiosignal==1.2.0
+ - alembic==1.7.7
+ - ansiwrap==0.8.4
+ - applicationinsights==0.11.10
+ - argon2-cffi==21.3.0
+ - argon2-cffi-bindings==21.2.0
+ - async-timeout==4.0.2
+ - asynctest==0.13.0
+ - attrs==21.4.0
+ - azure-common==1.1.28
+ - azure-core==1.24.0
+ - azure-graphrbac==0.61.1
+ - azure-identity==1.7.0
+ - azure-mgmt-authorization==0.61.0
+ - azure-mgmt-containerregistry==10.0.0
+ - azure-mgmt-core==1.3.0
+ - azure-mgmt-datafactory==1.1.0
+ - azure-mgmt-keyvault==9.3.0
+ - azure-mgmt-resource==12.1.0
+ - azure-mgmt-storage==11.2.0
+ - azure-storage-blob==12.6.0
+ - azureml-automl-core==1.36.1
+ - azureml-core==1.36.0.post2
+ - azureml-dataprep==2.24.4
+ - azureml-dataprep-native==38.0.0
+ - azureml-dataprep-rslex==2.0.3
+ - azureml-dataset-runtime==1.36.0
+ - azureml-mlflow==1.36.0
+ - azureml-pipeline==1.36.0
+ - azureml-pipeline-core==1.36.0
+ - azureml-pipeline-steps==1.36.0
+ - azureml-sdk==1.36.0
+ - azureml-telemetry==1.36.0
+ - azureml-tensorboard==1.36.0
+ - azureml-train-automl-client==1.36.0
+ - azureml-train-core==1.36.0
+ - azureml-train-restclients-hyperdrive==1.36.0
+ - backcall==0.2.0
+ - backports-tempfile==1.0
+ - backports-weakref==1.0.post1
+ - beautifulsoup4==4.11.1
+ - black==22.3.0
+ - bleach==5.0.0
+ - cachetools==4.2.4
+ - cffi==1.15.0
+ - charset-normalizer==2.0.12
+ - click==8.1.3
+ - cloudpickle==1.6.0
+ - colorama==0.4.4
+ - commonmark==0.9.1
+ - conda-merge==0.1.5
+ - contextlib2==21.6.0
+ - coverage==6.4
+ - cryptography==3.3.2
+ - cucim==21.10.1
+ - cycler==0.11.0
+ - databricks-cli==0.16.6
+ - dataclasses-json==0.5.2
+ - debugpy==1.6.0
+ - decorator==5.1.1
+ - defusedxml==0.7.1
+ - deprecated==1.2.13
+ - distro==1.7.0
+ - docker==4.3.1
+ - dotnetcore2==2.1.23
+ - entrypoints==0.4
+ - execnet==1.9.0
+ - fastjsonschema==2.15.3
+ - flake8==3.8.3
+ - flask==2.1.2
+ - frozenlist==1.3.0
+ - fsspec==2022.5.0
+ - fusepy==3.0.1
+ - future==0.18.2
+ - gitdb==4.0.9
+ - gitpython==3.1.7
+ - google-auth==1.35.0
+ - google-auth-oauthlib==0.4.6
+ - gputil==1.4.0
+ - greenlet==1.1.2
+ - grpcio==1.46.3
+ - gunicorn==20.1.0
+ - h5py==2.10.0
+ - humanize==4.1.0
+ - idna==3.3
+ - imageio==2.15.0
+ - importlib-metadata==4.11.4
+ - importlib-resources==5.7.1
+ - iniconfig==1.1.1
+ - innereye-dicom-rt==1.0.3
+ - ipykernel==6.13.0
+ - ipython==7.31.1
+ - ipython-genutils==0.2.0
+ - ipywidgets==7.7.0
+ - isodate==0.6.1
+ - itsdangerous==2.1.2
+ - jedi==0.18.1
+ - jeepney==0.8.0
+ - jinja2==3.1.2
+ - jmespath==0.10.0
+ - joblib==0.16.0
+ - jsonpickle==2.2.0
+ - jsonschema==4.5.1
+ - jupyter==1.0.0
+ - jupyter-client==6.1.5
+ - jupyter-console==6.4.3
+ - jupyter-core==4.10.0
+ - jupyterlab-pygments==0.2.2
+ - jupyterlab-widgets==1.1.0
+ - kiwisolver==1.4.2
+ - lightning-bolts==0.4.0
+ - llvmlite==0.34.0
+ - mako==1.2.0
+ - markdown==3.3.7
+ - markupsafe==2.1.1
+ - marshmallow==3.16.0
+ - marshmallow-enum==1.5.1
+ - matplotlib==3.3.0
+ - matplotlib-inline==0.1.3
+ - mccabe==0.6.1
+ - mistune==0.8.4
+ - mlflow==1.23.1
+ - mlflow-skinny==1.26.1
+ - monai==0.6.0
+ - more-itertools==8.13.0
+ - msal==1.17.0
+ - msal-extensions==0.3.1
+ - msrest==0.6.21
+ - msrestazure==0.6.4
+ - multidict==6.0.2
+ - mypy==0.910
+ - mypy-extensions==0.4.3
+ - nbclient==0.6.3
+ - nbconvert==6.5.0
+ - nbformat==5.4.0
+ - ndg-httpsclient==0.5.1
+ - nest-asyncio==1.5.5
+ - networkx==2.6.3
+ - nibabel==3.2.2
+ - notebook==6.4.11
+ - numba==0.51.2
+ - numpy==1.19.1
+ - oauthlib==3.2.0
+ - opencv-python-headless==4.5.1.48
+ - packaging==21.3
+ - pandas==1.1.0
+ - pandocfilters==1.5.0
+ - papermill==2.2.2
+ - param==1.9.3
+ - parso==0.8.3
+ - pathspec==0.9.0
+ - pexpect==4.8.0
+ - pickleshare==0.7.5
+ - pillow==9.0.0
+ - platformdirs==2.5.2
+ - pluggy==0.13.1
+ - portalocker==2.4.0
+ - prometheus-client==0.14.1
+ - prometheus-flask-exporter==0.20.1
+ - prompt-toolkit==3.0.29
+ - protobuf==3.20.1
+ - psutil==5.7.2
+ - ptyprocess==0.7.0
+ - py==1.11.0
+ - pyarrow==3.0.0
+ - pyasn1==0.4.8
+ - pyasn1-modules==0.2.8
+ - pycodestyle==2.6.0
+ - pycparser==2.21
+ - pydeprecate==0.3.1
+ - pydicom==2.0.0
+ - pyflakes==2.2.0
+ - pygments==2.12.0
+ - pyjwt==1.7.1
+ - pynndescent==0.5.7
+ - pyopenssl==20.0.1
+ - pyparsing==3.0.9
+ - pyrsistent==0.18.1
+ - pytest==6.0.1
+ - pytest-cov==2.10.1
+ - pytest-forked==1.3.0
+ - pytest-xdist==1.34.0
+ - python-dateutil==2.8.2
+ - pytorch-lightning==1.5.5
+ - pytz==2022.1
+ - pywavelets==1.3.0
+ - pyyaml==6.0
+ - pyzmq==23.0.0
+ - qtconsole==5.3.0
+ - qtpy==2.1.0
+ - querystring-parser==1.2.4
+ - requests==2.27.1
+ - requests-oauthlib==1.3.1
+ - rich==10.13.0
+ - rpdb==0.1.6
+ - rsa==4.8
+ - ruamel-yaml==0.16.12
+ - ruamel-yaml-clib==0.2.6
+ - runstats==1.8.0
+ - scikit-image==0.17.2
+ - scikit-learn==0.23.2
+ - scipy==1.5.2
+ - seaborn==0.10.1
+ - secretstorage==3.3.2
+ - send2trash==1.8.0
+ - simpleitk==1.2.4
+ - six==1.15.0
+ - smmap==5.0.0
+ - soupsieve==2.3.2.post1
+ - sqlalchemy==1.4.36
+ - sqlparse==0.4.2
+ - stopit==1.1.2
+ - stringcase==1.2.0
+ - tabulate==0.8.7
+ - tenacity==8.0.1
+ - tensorboard==2.3.0
+ - tensorboard-plugin-wit==1.8.1
+ - tensorboardx==2.1
+ - terminado==0.15.0
+ - textwrap3==0.9.2
+ - threadpoolctl==3.1.0
+ - tifffile==2021.11.2
+ - tinycss2==1.1.1
+ - toml==0.10.2
+ - tomli==2.0.1
+ - torchio==0.18.74
+ - torchmetrics==0.6.0
+ - tornado==6.1
+ - tqdm==4.64.0
+ - traitlets==5.2.1.post0
+ - typed-ast==1.4.3
+ - typing-inspect==0.7.1
+ - umap-learn==0.5.2
+ - urllib3==1.26.7
+ - wcwidth==0.2.5
+ - webencodings==0.5.1
+ - websocket-client==1.3.2
+ - werkzeug==2.1.2
+ - widgetsnbextension==3.6.0
+ - wrapt==1.14.1
+ - yacs==0.1.8
+ - yarl==1.7.2
+ - zipp==3.8.0