Merge branch 'main' of github.com:Kaggle/docker-python

Dockerfile.tmpl

@@ -14,6 +14,12 @@ FROM gcr.io/kaggle-images/python-torch-whl:${GPU_BASE_IMAGE_NAME}-${BASE_IMAGE_T
 FROM ${BASE_IMAGE_REPO}/${GPU_BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
 ENV CUDA_MAJOR_VERSION=11
 ENV CUDA_MINOR_VERSION=0
+# NVIDIA binaries from the host are mounted to /opt/bin.
+ENV PATH=/opt/bin:${PATH}
+# Add CUDA stubs to LD_LIBRARY_PATH to support building the GPU image on a CPU machine.
+ENV LD_LIBRARY_PATH_NO_STUBS="$LD_LIBRARY_PATH"
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs"
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 {{ else }}
 FROM ${BASE_IMAGE_REPO}/${CPU_BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
 {{ end }}
@@ -36,9 +42,6 @@ ADD clean-layer.sh /tmp/clean-layer.sh
 ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl
 ADD patches/template_conf.json /opt/kaggle/conf.json
 
-# Adds the libcuda.so to LD_LIBRARY_PATH which is necessary for the GPU mxnet package.
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/compat
-
 {{ if eq .Accelerator "gpu" }}
 # b/200968891 Keeps horovod once torch is upgraded.
 RUN pip uninstall -y horovod && \
@@ -70,7 +73,7 @@ RUN conda config --add channels nvidia && \
  /tmp/clean-layer.sh
 
 {{ if eq .Accelerator "gpu" }}
-RUN conda install cudf=21.08 cuml=21.08 cudatoolkit=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION && \
+RUN conda install cudf=21.10 cuml=21.10 cudatoolkit=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION && \
  /tmp/clean-layer.sh
 {{ end }}
 
@@ -134,6 +137,7 @@ RUN pip install pysal && \
  pip install seaborn python-dateutil dask python-igraph && \
  pip install pyyaml joblib husl geopy ml_metrics mne pyshp && \
  pip install pandas && \
+ pip install flax && \
  # Install h2o from source.
  # Use `conda install -c h2oai h2o` once Python 3.7 version is released to conda.
  apt-get install -y default-jre-headless && \
@@ -236,6 +240,8 @@ RUN pip install mpld3 && \
  pip install haversine && \
  pip install toolz cytoolz && \
  pip install plotly && \
+ # b/206631257: hyperopt requires py4j >= 0.2.6 requires py4j but doesn't list it as a dependency. Remove once hyperopt properly list it.
+ pip install py4j && \
  pip install hyperopt && \
  pip install fitter && \
  pip install langid && \
@@ -407,8 +413,7 @@ RUN pip install bleach && \
  pip install feather-format && \
  pip install fastai && \
  pip install allennlp && \
- # https://b.corp.google.com/issues/184685619#comment9: 3.9.0 is causing a major performance degradation with spacy 2.3.5
- pip install importlib-metadata==3.4.0 && \
+ pip install importlib-metadata && \
  python -m spacy download en_core_web_sm && python -m spacy download en_core_web_lg && \
  apt-get install -y ffmpeg && \
  /tmp/clean-layer.sh
@@ -446,13 +451,18 @@ RUN pip install flashtext && \
  pip install https://github.com/hbasria/ggpy/archive/0.11.5.zip && \
  pip install cesium && \
  pip install rgf_python && \
+ # b/205704651 remove install cmd for matrixprofile after version > 1.1.10 is released.
+ pip install git+git:https://github.com/matrix-profile-foundation/matrixprofile.git@6bea7d4445284dbd9700a097974ef6d4613fbca7 && \
  pip install tsfresh && \
+ # b/204442455 unpin once tsfresh is compatible with latest version of statsmodels.
+ pip install statsmodels==0.12.2 && \
  pip install pykalman && \
  pip install optuna && \
  pip install plotly_express && \
  pip install albumentations && \
  pip install catalyst && \
- pip install osmnx && \
+ # b/206990323 osmx 1.1.2 requires numpy >= 1.21 which we don't want. 
+ pip install osmnx==1.1.1 && \
  apt-get -y install libspatialindex-dev && \
  pip install pytorch-ignite && \
  pip install qgrid && \
@@ -564,3 +574,8 @@ LABEL kaggle-lang=python
 
 # Correlate current release with the git hash inside the kernel editor by running `!cat /etc/git_commit`.
 RUN echo "$GIT_COMMIT" > /etc/git_commit && echo "$BUILD_DATE" > /etc/build_date
+
+{{ if eq .Accelerator "gpu" }}
+# Remove the CUDA stubs.
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH_NO_STUBS"
+{{ end }}

Jenkinsfile

@@ -162,6 +162,16 @@ pipeline {
  '''
  }
  }
+
+ stage('Delete Old Unversioned Images') {
+ steps {
+ sh '''#!/bin/bash
+ set -exo pipefail
+ gcloud container images list-tags gcr.io/kaggle-images/python --filter="NOT tags:v* AND timestamp.datetime < -P6M" --format='get(digest)' --limit 100 | xargs -I {} gcloud container images delete gcr.io/kaggle-images/python@{} --quiet --force-delete-tags
+ gcloud container images list-tags gcr.io/kaggle-private-byod/python --filter="NOT tags:v* AND timestamp.datetime < -P6M" --format='get(digest)' --limit 100 | xargs -I {} gcloud container images delete gcr.io/kaggle-private-byod/python@{} --quiet --force-delete-tags
+ '''
+ }
+ }
  }
 
  post {

README.md

@@ -2,19 +2,13 @@
 
 [Kaggle Notebooks](https://www.kaggle.com/notebooks) allow users to run a Python Notebook in the cloud against our competitions and datasets without having to download data or set up their environment.
 
-This repository includes our Dockerfiles for building the [CPU-only](Dockerfile) and [GPU](gpu.Dockerfile) image that runs Python Notebooks on Kaggle.
+This repository includes the [Dockerfile](Dockerfile.tmpl) for building the CPU-only and GPU image that runs Python Notebooks on Kaggle.
 
-Our Python Docker images are stored on Google Container Registry at:
+Our Python Docker images are stored on the Google Container Registry at:
 
 * CPU-only: [gcr.io/kaggle-images/python](https://gcr.io/kaggle-images/python)
 * GPU: [gcr.io/kaggle-gpu-images/python](https://gcr.io/kaggle-gpu-images/python)
 
-Note: The base image for the GPU image is our CPU-only image. The [gpu.Dockerfile](gpu.Dockerfile) adds a few extra layers to install GPU related libraries and packages (cuda, libcudnn, pycuda etc.) and reinstall packages with specific GPU builds (torch, tensorflow and a few mores).
-
-## Getting started
-
-To get started with this image, read our [guide](https://medium.com/@kaggleteam/how-to-get-started-with-data-science-in-containers-6ed48cb08266) to using it yourself, or browse [Kaggle Notebooks](https://www.kaggle.com/notebooks) for ideas.
-
 ## Requesting new packages
 
 First, evaluate whether installing the package yourself in your own notebooks suits your needs. See [guide](https://github.com/Kaggle/docker-python/wiki/Missing-Packages).
@@ -23,9 +17,7 @@ If you the first step above doesn't work for your use case, [open an issue](http
 
 ## Opening a pull request
 
-1. Update the *Dockerfile*
- 1. For changes specific to the GPU image, update the [gpu.Dockerfile](gpu.Dockerfile).
- 1. Otherwise, update the [Dockerfile](Dockerfile).
+1. Edit the [Dockerfile](Dockerfile.tmpl).
 1. Follow the instructions below to build a new image.
 1. Add tests for your new package. See this [example](https://github.com/Kaggle/docker-python/blob/main/tests/test_fastai.py).
 1. Follow the instructions below to test the new image.
@@ -74,4 +66,4 @@ docker run --runtime nvidia --rm -it kaggle/python-gpu-build /bin/bash
 docker run --runtime nvidia --rm -it gcr.io/kaggle-gpu-images/python /bin/bash
 ```
 
-To ensure your container can access the GPU, follow the instructions posted [here](https://github.com/Kaggle/docker-python/issues/361#issuecomment-448093930).
+To ensure your container can access the GPU, follow the instructions posted [here](https://github.com/Kaggle/docker-python/issues/361#issuecomment-448093930).

config.txt

@@ -1,9 +1,9 @@
 BASE_IMAGE_REPO=gcr.io/deeplearning-platform-release
-BASE_IMAGE_TAG=m80
+BASE_IMAGE_TAG=m86
 CPU_BASE_IMAGE_NAME=tf2-cpu.2-6
 GPU_BASE_IMAGE_NAME=tf2-gpu.2-6
-LIGHTGBM_VERSION=3.2.1
+LIGHTGBM_VERSION=3.3.1
 TORCH_VERSION=1.9.1
 TORCHAUDIO_VERSION=0.9.1
 TORCHTEXT_VERSION=0.10.1
-TORCHVISION_VERSION=0.10.1
+TORCHVISION_VERSION=0.10.1

tests/test_flax.py

@@ -0,0 +1,15 @@
+import unittest
+
+import jax.numpy as jnp
+import numpy as np
+
+from flax import linen as nn
+
+
+class TestFlax(unittest.TestCase):
+
+ def test_bla(self):
+ x = jnp.full((1, 3, 3, 1), 2.)
+ mul_reduce = lambda x, y: x * y
+ y = nn.pooling.pool(x, 1., mul_reduce, (2, 2), (1, 1), 'VALID')
+ np.testing.assert_allclose(y, np.full((1, 2, 2, 1), 2. ** 4))