Update Dockerfiles to ubuntu:24.04 (project-chip#33637)

* Update to ubuntu 24.04 and bump version

* Fix comment

* Dependencies update

* Use the latest gcc by default

* Fix up clang save & cleanup

* More build fixes

* Add more --break-system-packages

* Do not remove libllvm17t64 ... I am unsure about that one

* Remove it after all ... seems some size savings  here

* More cleanup

* Drop alternative install

* Install gcc without a specific version. Undoes project-chip#28478

* Need llvm18-dev as well

* Update to lib64 paths

* Also install xz

* xz is in utils

* Fix asr depends when no recommends is installed

* Clean up infineon, make the curl download to NOT be silent

* using -j8 is arbitrary ... use nproc instead

* update what qemu means

* Install patch for the qemu image build

* Adapt Tizen Dockerfiles

* Update ubuntu version

* Update Dockerfile for chip-build-linux-qemu image

* Restyle

* Add ability to skip already built things, otherwise docker keeps re-building stuff

* Docker image inspect is MUCH faster than docker images

* Fix nrf build

* Fix ameba dockerfile build

* Fix cirque image build

* Make imx image work

* Make build ordering predictable

* Fix k32w image build

* Fix telink image build

* Speed up android image build by a lot

* Bump up espressif qemu build, do not use the 3-year old version

* Update versions to match system package versions, so that bootstrap is faster

* Make sure that patch is available in the base image, we seem to use it ocassionally

* One documentation update

* Also allow compilation of mbedos bits in the vscode image

* Include what you use from system

---------

Co-authored-by: Andrei Litvin <[email protected]>
Co-authored-by: Maksymilian Knust <[email protected]>
Co-authored-by: Jakub Latusek <[email protected]>

integrations/docker/build-all.sh

@@ -25,19 +25,33 @@ set -e
 
 SUB_PATH="*"
 ALL_ARGS=""
+SKIP_BUILT=0
 
 for i in "$@"; do
  case $i in
+ -h | --help)
+ echo "Supported arguments:"
+ echo " -h/--help This help text"
+ echo " -p/--path sub-path in integrations/docker/images to build (defaults to * for everything)"
+ echo " -skip-built Do not rebuild images already found as built"
+ exit 0
+ ;;
  -p=* | --path=*)
  SUB_PATH="${i#*=}"
  ;;
+ --skip-built)
+ SKIP_BUILT=1
+ ;;
  *)
  ALL_ARGS="$ALL_ARGS ${i#*=}"
  ;;
  esac
 done
 
+VERSION=${DOCKER_BUILD_VERSION:-$(sed 's/ .*//' "$(git rev-parse --show-toplevel)"/integrations/docker/images/base/chip-build/version)}
+
 echo "SUB_PATH: $SUB_PATH"
+echo "VERSION: $VERSION"
 
 function build_image() {
  PARSE_PATH=$1
@@ -46,8 +60,17 @@ function build_image() {
  echo "PARSE_PATH: $PARSE_PATH"
  echo "ARGS_TO_PASS: $ARGS_TO_PASS"
 
- find "$(git rev-parse --show-toplevel)"/integrations/docker/images/$PARSE_PATH -name Dockerfile ! -path "*chip-cert-bins/*" | while read -r dockerfile; do
- echo "$(dirname "$dockerfile")"
+ find "$(git rev-parse --show-toplevel)"/integrations/docker/images/$PARSE_PATH -name Dockerfile ! -path "*chip-cert-bins/*" | sort | while read -r dockerfile; do
+ # Images are of the form `ghcr.io/project-chip/{name}` and tagged as "${VERSION}"
+ DOCKER_PATH=$(dirname $dockerfile) # Drop the file name
+ IMAGE_NAME="ghcr.io/project-chip/${DOCKER_PATH##*/}:${VERSION}" # Drop directory prefix
+
+ if [ $SKIP_BUILT -ne 0 ] && docker image inspect "${IMAGE_NAME}" >/dev/null 2>&1; then
+ echo "Image ${IMAGE_NAME} already exists. Skipping build"
+ continue
+ fi
+
+ echo "BUILDING $(dirname "$dockerfile") (i.e. ${IMAGE_NAME})"
  pushd "$(dirname "$dockerfile")" >/dev/null
  ./build.sh "$ARGS_TO_PASS"
  popd >/dev/null

integrations/docker/images/base/chip-build-minimal/Dockerfile

@@ -1,27 +1,19 @@
 # This minimal build image is intentionally not based on chip-build
-FROM ubuntu:focal
+FROM ubuntu:24.04
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 
-# ARG NINJA_VERSION=v1.11.1
-ARG GN_HASH=5a004f9427a050c6c393c07ddb85cba8ff3849fa
-
 RUN set -x \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
- build-essential ca-certificates git pkg-config python3-venv ninja-build \
+ build-essential \
+ ca-certificates \
+ generate-ninja \
+ git pkg-config \
+ ninja-build \
+ python3-venv \
  && git config --global advice.detachedHead false
 
-# RUN set -x && cd /var/tmp \
-# && git clone --branch "$NINJA_VERSION" https://github.com/ninja-build/ninja.git \
-# && ( cd ninja && ./configure.py --bootstrap && install -m 0755 ninja /usr/local/bin/ ) \
-# && rm -rf ninja
-
-RUN set -x && cd /var/tmp \
- && git clone https://gn.googlesource.com/gn \
- && ( cd gn && git checkout "$GN_HASH" && CXX=g++ build/gen.py && ninja -C out && install -m 0755 out/gn /usr/local/bin/ ) \
- && rm -rf gn
-
 # CHIP build dependencies
 RUN set -x \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \

integrations/docker/images/base/chip-build/Dockerfile

@@ -1,5 +1,4 @@
-# start with Ubuntu 20.04LTS
-FROM ubuntu:focal
+FROM ubuntu:24.04
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 
 VOLUME "/var/source"
@@ -30,7 +29,7 @@ RUN case ${TARGETPLATFORM} in \
 # base build and check tools and libraries layer
 RUN set -x \
  && apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing --no-install-recommends \
  autoconf \
  automake \
  bison \
@@ -39,23 +38,27 @@ RUN set -x \
  clang \
  clang-format \
  clang-tidy \
+ cmake \
  curl \
  flex \
- g++-10 \
+ g++ \
+ generate-ninja \
  git \
  git-lfs \
  gperf \
  iproute2 \
+ iwyu \
  jq \
  lcov \
  libavahi-client-dev \
  libavahi-common-dev \
- libcairo-dev \
  libcairo2-dev \
+ libcairo-dev \
  libdbus-1-dev \
  libdbus-glib-1-dev \
  libdmalloc-dev \
  libgif-dev \
+ libgirepository-1.0-1 \
  libglib2.0-dev \
  libical-dev \
  libjpeg-dev \
@@ -68,8 +71,8 @@ RUN set -x \
  libpango1.0-dev \
  libpixman-1-dev \
  libreadline-dev \
- libsdl-pango-dev \
  libsdl2-dev \
+ libsdl-pango-dev \
  libssl-dev \
  libtool \
  libudev-dev \
@@ -80,11 +83,19 @@ RUN set -x \
  meson \
  net-tools \
  ninja-build \
+ nodejs \
+ patch \
  pkg-config \
+ python3 \
+ python3-click \
+ python3-coloredlogs \
+ python3-dev \
+ python3-future \
+ python3-pandas \
+ python3-pip \
+ python3-tabulate \
+ python3-venv \
  python-is-python3 \
- python3.9 \
- python3.9-dev \
- python3.9-venv \
  rsync \
  shellcheck \
  strace \
@@ -97,73 +108,19 @@ RUN set -x \
  && git lfs install \
  && : # last line
 
-# Set gcc 10 as a default compiler to work with TSAN
-RUN set -x \
- && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 \
- && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 \
- && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 \
- && update-alternatives --set cc /usr/bin/gcc \
- && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 \
- && update-alternatives --set c++ /usr/bin/g++ \
- && : # last line
-
-# Cmake v3.23.1
-ENV CMAKE_PLATFORM_VERSION=
-RUN case ${TARGETPLATFORM} in \
- "linux/amd64") CMAKE_PLATFORM_VERSION="x86_64";; \
- "linux/arm64") CMAKE_PLATFORM_VERSION="aarch64";; \
- *) \
- test -n "$TARGETPLATFORM" \
- echo "Unsupported platform ${TARGETPLATFORM}" \
- && return 1 ;\
- ;; \
- esac \
- && set -x \
- && (cd /tmp \
- && wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-Linux-$CMAKE_PLATFORM_VERSION.sh \
- && sh cmake-3.23.1-Linux-$CMAKE_PLATFORM_VERSION.sh --exclude-subdir --prefix=/usr/local \
- && rm -rf cmake-3.23.1-Linux-$CMAKE_PLATFORM_VERSION.sh) \
- && exec bash \
- && : # last line
-
-# Python 3.9 and PIP
-RUN set -x \
- && DEBIAN_FRONTEND=noninteractive apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y libgirepository1.0-dev \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common \
- && add-apt-repository universe \
- && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
- && python3.9 get-pip.py \
- && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 \
- && rm -rf /var/lib/apt/lists/ \
- && : # last line
-
+# NOTE: this install is NOT ideal as we are installing over system
+# we should migrate to venvs for packages we do not directly have
+# provided by the system
 RUN set -x \
- && pip3 install --no-cache-dir \
+ && pip3 install --break-system-packages --no-cache-dir --break-system-packages \
  attrs \
- click \
- coloredlogs \
  cxxfilt \
- future \
  ghapi \
  mobly \
- pandas \
  portpicker \
  pygit \
  PyGithub \
  ruff \
- tabulate \
- && : # last line
-
-# build and install gn
-RUN set -x \
- && git clone https://gn.googlesource.com/gn \
- && cd gn \
- && python3 build/gen.py \
- && ninja -C out \
- && cp out/gn /usr/local/bin \
- && cd .. \
- && rm -rf gn \
  && : # last line
 
 # Install bloat comparison tools
@@ -172,38 +129,13 @@ RUN set -x \
  && mkdir -p bloaty/build \
  && cd bloaty/build \
  && cmake -DCMAKE_BUILD_TYPE=MinSizeRel ../ \
- && make -j8 \
+ && make -j$(nproc) \
  && strip bloaty \
  && make install \
  && cd ../.. \
  && rm -rf bloaty \
  && : # last line
 
-# Need newer version of include-what-you-use
-RUN set -x \
- && apt-get update \
- # Install build and runtime requirements for IWYU
- && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing clang-12 libclang-12-dev \
- # Build and install IWYU
- && git clone --depth=1 --branch=clang_12 https://github.com/include-what-you-use/include-what-you-use.git \
- && mkdir -p include-what-you-use/build \
- && cd include-what-you-use/build \
- && cmake -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_PREFIX_PATH=/usr/lib/llvm-12 -DIWYU_LINK_CLANG_DYLIB=OFF .. \
- && make -j8 \
- && strip bin/include-what-you-use \
- && make install \
- # Save clang-12 files, so we can restore them after build dependencies cleanup
- && tar -cf clang-12-files.tar $(dpkg -L libclang-common-12-dev |grep /include) /usr/lib/llvm-12/lib/libLLVM-12.so.1 \
- # Cleanup build dependencies
- && apt autopurge -fy clang-12 libclang-12-dev \
- && rm -rf /var/lib/apt/lists/ \
- # Restore clang-12 files
- && tar -xf clang-12-files.tar -C / \
- # Cleanup
- && cd ../.. \
- && rm -rf include-what-you-use \
- && : # last line
-
 # Build glib-2.0 from source with enabled thread sanitizer. This is needed for
 # running CHIP tests with TSAN enabled. When running applications with TSAN
 # all shared libraries should be built with TSAN enabled, otherwise TSAN might
@@ -219,7 +151,7 @@ RUN case ${TARGETPLATFORM} in \
  && git clone --depth=1 --branch=$GLIB_VERSION https://github.com/GNOME/glib.git \
  && CFLAGS="-O2 -g -fsanitize=thread" meson glib/build glib -Dtests=false \
  && DESTDIR=../build-image ninja -C glib/build install \
- && mv glib/build-image/usr/local/lib/x86_64-linux-gnu/lib* $LD_LIBRARY_PATH_TSAN \
+ && mv glib/build-image/usr/local/lib64/lib* $LD_LIBRARY_PATH_TSAN \
  && rm -rf glib \
  ;; \
  "linux/arm64") \
@@ -231,41 +163,6 @@ RUN case ${TARGETPLATFORM} in \
  ;; \
  esac
 
-# NodeJS: install a newer version than what apt-get would read
-# This installs the latest LTS version of nodejs
-#
-# NodeJS is required by github actions, we use Wandalen/[email protected]
-# and that seems to use the built-in node installation in the image
-#
-# This is not a CHIP dependency directly, but used by CI
-ENV CHIP_NODE_VERSION=v16.13.2
-ENV NODE_PLATFORM_VERSION=
-RUN case ${TARGETPLATFORM} in \
- "linux/amd64") NODE_PLATFORM_VERSION=x64;; \
- "linux/arm64") NODE_PLATFORM_VERSION=arm64;; \
- *) \
- test -n "$TARGETPLATFORM" \
- echo "Unsupported platform ${TARGETPLATFORM}" \
- && return 1 ;\
- ;; \
- esac \
- && set -x \
- && mkdir node_js \
- && cd node_js \
- && wget https://nodejs.org/dist/$CHIP_NODE_VERSION/node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION.tar.xz \
- && tar xfvJ node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION.tar.xz \
- && mv node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION /opt/ \
- && ln -s /opt/node-$CHIP_NODE_VERSION-linux-$NODE_PLATFORM_VERSION /opt/node \
- && ln -s /opt/node/bin/* /usr/bin \
- && cd .. \
- && rm -rf node_js \
- && : # last line
-
 # Some things that save space
 # Protoc goes from 108M to 4.6M
 RUN strip /usr/local/bin/protoc*
-
-# CMake documentation not needed, saves 34MB
-# /usr/local/man contains cmake documentation
-RUN rm -rf /usr/local/doc/cmake
-RUN rm -rf /usr/local/man

integrations/docker/images/base/chip-build/version

@@ -1 +1 @@
-54 : [Telink] Update Docker image (Zephyr update)
+55 : Update to Ubuntu 24.04 as the base build image

integrations/docker/images/chip-cert-bins/Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Setup dependencies (based on chip-build).
-FROM ubuntu:22.04 as chip-build-cert
+FROM ubuntu:24.04 as chip-build-cert
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 ARG TARGETPLATFORM
 # COMMITHASH defines the target commit to build from. May be passed in using --build-arg.
@@ -292,12 +292,12 @@ COPY --from=chip-build-cert-bins /root/connectedhomeip/out/python_env python_env
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing python_testing
 
 COPY --from=chip-build-cert-bins /root/connectedhomeip/scripts/tests/requirements.txt /tmp/requirements.txt
-RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+RUN pip install --break-system-packages -r /tmp/requirements.txt && rm /tmp/requirements.txt
 
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing/requirements.txt /tmp/requirements.txt
-RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+RUN pip install --break-system-packages -r /tmp/requirements.txt && rm /tmp/requirements.txt
 
 # PIP requires MASON package compilation, which seems to require a JDK
 RUN set -x && DEBIAN_FRONTEND=noninteractive apt-get install -fy openjdk-8-jdk
 
-RUN pip install --no-cache-dir python_lib/controller/python/chip*.whl
+RUN pip install --break-system-packages --no-cache-dir python_lib/controller/python/chip*.whl

integrations/docker/images/stage-1/chip-build-crosscompile/Dockerfile

@@ -6,6 +6,7 @@ RUN set -x \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
  git \
+ xz-utils \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/ \
  && : # last line

integrations/docker/images/stage-2/chip-build-ameba/Dockerfile

@@ -2,6 +2,13 @@ ARG VERSION=1
 FROM ghcr.io/project-chip/chip-build:${VERSION}
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 
+RUN set -x \
+ && apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
+ bzip2 \
+ && rm -rf /var/lib/apt/lists/ \
+ && : # last line
+
 # Setup Ameba
 ARG AMEBA_DIR=/opt/ameba
 ARG TAG_NAME=ameba_update_2024_03_22

integrations/docker/images/stage-2/chip-build-asr/Dockerfile

@@ -5,6 +5,8 @@ LABEL org.opencontainers.image.source https://github.com/project-chip/connectedh
 RUN set -x \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
+ bzip2 \
+ tar \
  wget \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/ \