testing tf 1.15 compat.

.gitlab-ci.yml

@@ -59,97 +59,6 @@ testjob:
  - wget -N https://www.dropbox.com/s/gt0hm6o61rlsfcc/csv_data.tar.gz
  - tar -C data -xzvf csv_data.tar.gz
 
- #### python 2 tests ###################################
- # save NiftyNet folder path just in case
- - export niftynet_dir=$(pwd)
-
- # create a virtual env to dev-test
- - venv="niftynet-dev-test-py2"
- - mypython=$(which python2)
- - virtualenv -p $mypython $venv
- - cd $venv
- - venv_dir=$(pwd)
- - source bin/activate
- # print Python version to CI output
- - which python
- - python --version
- - cd $niftynet_dir
-
- - pip install -r requirements-gpu.txt
- - python -c "import tensorflow as tf; print(tf.__version__)"
- - python -c "import tensorflow as tf; from tensorflow.python.client import device_lib; print(device_lib.list_local_devices())"
-
- - echo $(python tests/get_gpu_index.py)
- - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)
-
- # check whether version command works
- # Git-clone:
- - python net_segment.py --version 2>&1 | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+'
- - python -c 'import niftynet; print(niftynet.__version__)' | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+'
- # downloaded ZIP ball:
- - export check_version_zip=/tmp/check_version_zip-$(git rev-parse --short HEAD)
- - mkdir $check_version_zip
- - git archive --format=tar HEAD | (cd $check_version_zip && tar xf -)
- - cd $check_version_zip
- - python net_segment.py --version 2>&1 | grep -v 'unknown'
- - python -c 'import niftynet; print(niftynet.__version__)' | grep -v 'unknown'
- - cd $niftynet_dir
- - rm -rf $check_version_zip
- - unset check_version_zip
-
- # tests
- - python net_download.py testing -r
-
- - python net_segment.py train -c config/highres3dnet_config.ini --batch_size=1 --num_threads=2 --queue_length=40 --max_iter=10
- - python net_segment.py inference -c config/highres3dnet_config.ini --batch_size 8 --spatial_window_size 64,64,64 --queue_length 64
-
- - python net_segment.py train -c config/scalenet_config.ini --batch_size 1 --queue_length 5 --num_threads 2
- - python net_segment.py inference -c config/scalenet_config.ini --batch_size 16 --spatial_window_size 64,64,64 --queue_length 32
-
- - python net_segment.py train -c config/vnet_config.ini --batch_size 1 --queue_length 5 --num_threads 2 --activation_function relu
- - python net_segment.py inference -c config/vnet_config.ini --batch_size 16 --spatial_window_size 64,64,64 --queue_length 32 --activation_function relu
-
- # need a large GPU to run
- #- python net_segment.py train -c config/unet_config.ini --batch_size 1 --queue_length 5 --num_threads 2
- #- python net_segment.py inference -c config/unet_config.ini --batch_size 1 --spatial_window_size 96,96,96 --queue_length 5
-
- #- python net_segment.py train -c config/deepmedic_config.ini --batch_size 128 --queue_length 48 --num_threads 4
- #- python net_segment.py inference -c config/deepmedic_config.ini --batch_size 12 --spatial_window_size 135,135,135 --queue_length 128
-
- - python net_segment.py train -c config/default_segmentation.ini --batch_size 3 --queue_length 6
- - python net_segment.py train -c config/default_segmentation.ini --batch_size 3 --queue_length 6 --exclude_fraction_for_validation 0.1 --validation_every_n 3
- - python net_segment.py train -c config/default_segmentation.ini --batch_size 3 --queue_length 6 --starting_iter 0 --max_iter 15
- - python net_segment.py inference -c config/default_segmentation.ini --spatial_window_size 84,84,84 --batch_size 7 --queue_length 14 --inference_iter 15
- - python net_segment.py evaluation -c config/default_segmentation.ini
- - python net_segment.py inference -c config/default_segmentation.ini --spatial_window_size 84,84,84 --batch_size 7 --queue_length 14 --inference_iter 15 --dataset_to_infer validation
-
- - python net_segment.py train -c config/default_multimodal_segmentation.ini --batch_size 3
- - python net_segment.py inference -c config/default_multimodal_segmentation.ini --spatial_window_size 64,64,64 --batch_size 7
-
- - python net_classify.py train -c testing_data/test_classification.ini
- - python net_classify.py inference -c testing_data/test_classification.ini
- #- python net_classify.py evaluation -c testing_data/test_classification.ini
-
- - python net_regress.py train -c config/default_monomodal_regression.ini --batch_size=1 --name toynet --max_iter 10
- - python net_regress.py inference -c config/default_monomodal_regression.ini --batch_size=7 --name toynet --spatial_window_size 84,84,84
- - python net_regress.py evaluation -c config/default_monomodal_regression.ini
-
- - python net_gan.py train -c config/GAN_demo_train_config.ini --max_iter 5
- - python net_gan.py inference -c config/GAN_demo_train_config.ini
-
- - python net_autoencoder.py train -c config/vae_config.ini --max_iter 5
- - python net_autoencoder.py inference -c config/vae_config.ini --inference_type sample
- - python net_autoencoder.py inference -c config/vae_config.ini --inference_type encode --save_seg_dir output/vae_demo_features
- - python net_autoencoder.py inference -c config/vae_config.ini --inference_type encode-decode
-
- - python -m tests.test_model_zoo
- # - python -m unittest discover -s "tests" -p "*_test.py"
-
- # deactivate virtual environment
- - deactivate
- - cd $niftynet_dir
- ###############end of python2
-
  ######### Python 3 ###################### run python3 code with coverage wrapper
  # save NiftyNet folder path just in case
  - export niftynet_dir=$(pwd)
@@ -167,7 +76,8 @@ testjob:
  - cd $niftynet_dir
 
  - pip install coverage
- - pip install -r requirements-gpu.txt
+ - pip install --upgrade pip
+ - pip install -r requirements.txt
 
  - echo $(python tests/get_gpu_index.py)
  - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)
@@ -269,16 +179,16 @@ quicktest:
  - wget -q https://www.dropbox.com/s/5p5fdgy053tgmdj/testing_data_v0_3.tar.gz
  - mkdir -p testing_data
  - tar -xzvf testing_data_v0_3.tar.gz -C testing_data
- - wget -N https://www.dropbox.com/s/gt0hm6o61rlsfcc/csv_data.tar.gz
+ - wget -q https://www.dropbox.com/s/gt0hm6o61rlsfcc/csv_data.tar.gz
  - tar -C data -xzvf csv_data.tar.gz
 
- ######### Python 2 ###################### run python2 code with coverage wrapper
+ ######### Python 3 ###################### run python3 code with coverage wrapper
  # save NiftyNet folder path just in case
  - export niftynet_dir=$(pwd)
 
  # create a virtual env to dev-test
- - venv="niftynet-dev-test-py2"
- - mypython=$(which python2)
+ - venv="niftynet-dev-test-py3"
+ - mypython=$(which python3)
  - virtualenv -p $mypython $venv
  - cd $venv
  - venv_dir=$(pwd)
@@ -289,14 +199,13 @@ quicktest:
  - cd $niftynet_dir
 
  - pip install coverage
- - pip install -r requirements-gpu.txt
+ - pip install --upgrade pip
+ - pip install -r requirements.txt
 
- - python -c "import tensorflow as tf; print tf.__version__"
+ - python -c "import tensorflow as tf; print(tf.__version__)"
  #- python -c "import tensorflow as tf; from tensorflow.python.client import device_lib; print device_lib.list_local_devices()"
  - coverage erase
  # run only fast tests
- - pip install -r requirements-gpu.txt
-
  - echo $(python tests/get_gpu_index.py)
  - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)
 
@@ -306,7 +215,7 @@ quicktest:
  # deactivate virtual environment
  - deactivate
  - cd $niftynet_dir
- ###############end of python2
+ ###############end of python3
 
  - echo 'finished quick tests'
  tags:
@@ -329,7 +238,8 @@ pip-installer:
  # following three lines copied over from dev script:
  - ls -la /dev | grep nvidia
  # install TF
- - pip install tensorflow-gpu==1.13.2
+ - pip install --upgrade pip
+ - pip install "tensorflow==1.15.*"
  - echo $(python tests/get_gpu_index.py)
  - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)
  # create a Python file that will import all available packages from the pip installer
@@ -343,110 +253,6 @@ pip-installer:
  - sh ci/bundlewheel.sh
  - source ci/findwheel.sh
  - echo $niftynet_wheel
- # ============= Python 2 ============================
- # create a virtual env to test pip installer
- - venv="niftynet-pip-installer-venv-py2"
- - mypython=$(which python2)
- - virtualenv -p $mypython $venv
- - cd $venv
- - venv_dir=$(pwd)
- - source bin/activate
- # print Python version to CI output
- - which python
- - python --version
- # NiftyNet console entries should fail gracefully if TF not installed
- # i.e. check that the warning displays the TF website
- - cd $niftynet_dir
-
- - echo $(python tests/get_gpu_index.py)
- - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)
-
- - set +e
- - python -c "import niftynet" 2>&1 | grep "https://www.tensorflow.org/"
- - set -e
- - cd $venv_dir
- # install TF
- - pip install tensorflow-gpu==1.13.2
- # install using built NiftyNet wheel
- - pip install $niftynet_wheel
- # install SimpleITK for package importer test to work properly
- - pip install simpleitk
- # create symlink to required datasets
- - ln -s /home/gitlab-runner/environments/niftynet/data/example_volumes ./example_volumes
- # check user-defined networks discovered
- # 1) create configuration + NiftyNet home
- - CONFIG_HOME=~/.niftynet
- - mkdir -p $CONFIG_HOME
- - NIFTYNET_HOME=~/niftynet-$LAST_COMMIT
- - mkdir $NIFTYNET_HOME
- - GLOBAL_CONFIG=$CONFIG_HOME/config.ini
- - echo "[global]" > $GLOBAL_CONFIG
- - echo "home = $NIFTYNET_HOME" >> $GLOBAL_CONFIG
- - cat $GLOBAL_CONFIG
- # 2) create extension hierarchy
- - mkdir -p $NIFTYNET_HOME/niftynetext/network
- - touch $NIFTYNET_HOME/niftynetext/__init__.py
- - touch $NIFTYNET_HOME/niftynetext/network/__init__.py
- # 3) replicate working ToyNet example
- - TOYNET_REPLICA=$NIFTYNET_HOME/niftynetext/network/notoynet.py
- - cp $niftynet_dir/niftynet/network/toynet.py $TOYNET_REPLICA
- - sed -i 's/ToyNet/NoToyNet/g' $TOYNET_REPLICA
- # 4) run e.g. segmentation using ToyNet replica
- - net_segment train -c $niftynet_dir/config/default_segmentation.ini --name niftynetext.network.notoynet.NoToyNet --batch_size 3 --max_iter 5
- - net_segment inference -c $niftynet_dir/config/default_segmentation.ini --name niftynetext.network.notoynet.NoToyNet --spatial_window_size 80,80,80 --batch_size 8
- # 5) clean up created config + NiftyNet home hierarchy
- - rm -rf $CONFIG_HOME
- - unset CONFIG_HOME
- - rm -rf $NIFTYNET_HOME
- - unset NIFTYNET_HOME
- # check whether all packages are importable
- - cat $package_importer
- - python $package_importer
- # check whether version command works
- - net_segment --version 2>&1 | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+'
- - python -c 'import niftynet; print(niftynet.__version__)' | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+'
- # test niftynet command
- - net_download testing -r
-
- - net_segment train -c $niftynet_dir/config/default_segmentation.ini --name toynet --batch_size 3 --max_iter 5
- - net_segment inference -c $niftynet_dir/config/default_segmentation.ini --name toynet --spatial_window_size 80,80,80 --batch_size 8
- - net_segment evaluation -c $niftynet_dir/config/default_segmentation.ini
- - net_run train --app net_segment -c $niftynet_dir/config/default_segmentation.ini --name toynet --batch_size 3 --max_iter 5
- - net_run inference --app net_segment -c $niftynet_dir/config/default_segmentation.ini --name toynet --spatial_window_size 80,80,80 --batch_size 8
- - net_run evaluation --app net_segment -c $niftynet_dir/config/default_segmentation.ini
-
- - net_classify train -c extensions/testing/test_classification.ini
- - net_classify inference -c extensions/testing/test_classification.ini
- #- net_classify evaluation -c extensions/testing/test_classification.ini
- - net_run --app net_classify train -c extensions/testing/test_classification.ini
- - net_run --app net_classify inference -c extensions/testing/test_classification.ini
- #- net_run --app net_classify evaluation -c extensions/testing/test_classification.ini
-
- - net_regress train -c $niftynet_dir/config/default_monomodal_regression.ini --max_iter 10 --name toynet --batch_size=2
- - net_regress inference -c $niftynet_dir/config/default_monomodal_regression.ini --name toynet --spatial_window_size 84,84,84 --batch_size 7
- - net_regress evaluation -c $niftynet_dir/config/default_monomodal_regression.ini
- - net_run train -a net_regress -c $niftynet_dir/config/default_monomodal_regression.ini --max_iter 10 --name toynet --batch_size=2
- - net_run inference -a net_regress -c $niftynet_dir/config/default_monomodal_regression.ini --name toynet --spatial_window_size 84,84,84 --batch_size 7
- - net_run evaluation -a net_regress -c $niftynet_dir/config/default_monomodal_regression.ini
-
- - net_gan train -c $niftynet_dir/config/GAN_demo_train_config.ini --max_iter 5
- - net_gan inference -c $niftynet_dir/config/GAN_demo_train_config.ini
- - net_run train --app net_gan -c $niftynet_dir/config/GAN_demo_train_config.ini --max_iter 5
- - net_run inference --app net_gan -c $niftynet_dir/config/GAN_demo_train_config.ini
-
- - net_autoencoder train -c $niftynet_dir/config/vae_config.ini --max_iter 5
- - net_autoencoder inference -c $niftynet_dir/config/vae_config.ini --inference_type sample
- - net_autoencoder inference -c $niftynet_dir/config/vae_config.ini --inference_type encode --save_seg_dir output/vae_demo_features
- - net_autoencoder inference -c $niftynet_dir/config/vae_config.ini --inference_type encode-decode
- - net_run train --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --max_iter 5
- - net_run inference --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --inference_type sample
- - net_run inference --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --inference_type encode --save_seg_dir output/vae_demo_features
- - net_run inference --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --inference_type encode-decode
-
- - net_download dense_vnet_abdominal_ct_model_zoo -r
- # deactivate virtual environment
- - deactivate
- - cd $niftynet_dir
  # ============= Python 3 ============================
  # create a virtual env to test pip installer
  - venv="niftynet-pip-installer-venv-py3"
@@ -470,7 +276,8 @@ pip-installer:
  - set -e
  - cd $venv_dir
  # install TF
- - pip install tensorflow-gpu==1.13.2
+ - pip install --upgrade pip
+ - pip install "tensorflow==1.15.*"
  # install using built NiftyNet wheel
  - pip install $niftynet_wheel
  # install SimpleITK for package importer test to work properly
@@ -547,8 +354,9 @@ pip-camera-ready:
  - virtualenv $check_version_venv
  - source $check_version_venv/bin/activate
  # install TF
- - pip install tensorflow-gpu==1.13.2
- - pip install -r requirements-gpu.txt
+ - pip install --upgrade pip
+ - pip install "tensorflow==1.15.*"
+ - pip install -r requirements.txt
  - pip install $niftynet_wheel
  - net_segment --version 2>&1 | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+' | grep -v +
  - python -c 'import niftynet; print(niftynet.__version__)' | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+' | grep -v +

README.md

@@ -44,8 +44,7 @@ HIG -- High-dimensional Imaging Group, UCL), where BMEIS acts as the consortium
 ### Installation
 
 1. Please install the appropriate [TensorFlow][tf] package*:
- * [`pip install "tensorflow-gpu>=1.13.2, <=1.14"`][tf-pypi-gpu] for TensorFlow with GPU support
- * [`pip install "tensorflow>=1.13.2, <=1.14"`][tf-pypi] for CPU-only TensorFlow
+ * [`pip install "tensorflow==1.15.*"`][tf-pypi]
 1. [`pip install niftynet`](https://pypi.org/project/NiftyNet/)
 
  <sup>All other NiftyNet dependencies are installed automatically as part of the pip installation process.

doc/requirements.txt

@@ -9,7 +9,7 @@ nibabel>=2.1.0
 numpy>=1.13.3
 scipy>=0.18
 configparser
-tensorflow>=1.13.2, <=1.14
+tensorflow==1.15.*
 pandas
 pillow
 blinker

doc/source/installation.rst

@@ -3,8 +3,7 @@ Installation
 
 1. Installing the appropriate `TensorFlow`_ package:
 
- - ``pip install "tensorflow-gpu>=1.13.2, <=1.14"`` for TensorFlow with GPU support
- - ``pip install "tensorflow>=1.13.2, <=1.14"`` for CPU-only TensorFlow
+ - ``pip install "tensorflow==1.15.*"``
 
 2. Installing NiftyNet package
 

requirements-cpu.txt → requirements.txt

@@ -3,7 +3,7 @@ nibabel>=2.1.0
 numpy>=1.13.3
 scipy>=0.18
 configparser
-tensorflow>=1.13.2, <=1.14
+tensorflow==1.15.*
 pandas
 pillow
 blinker