feat(installer): support containerd installation

Makefile

@@ -139,7 +139,7 @@ release.build:
 ifeq ($(NEED_BUILD_PROVIDER),true)
  cd build/docker/tools/provider-res && make all
 endif
- make push.multiarch
+ make manifest.multiarch
 
 ## release: Release tke
 .PHONY: release

build/docker/tools/provider-res/Dockerfile

@@ -29,6 +29,9 @@ COPY linux-arm64/conntrack-tools-*.tar.gz res/linux-arm64/
 COPY linux-amd64/docker-*.tar.gz res/linux-amd64/
 COPY linux-arm64/docker-*.tar.gz res/linux-arm64/
 
+COPY linux-amd64/nerdctl-*.tar.gz res/linux-amd64/
+COPY linux-arm64/nerdctl-*.tar.gz res/linux-arm64/
+
 COPY linux-amd64/containerd-*.tar.gz res/linux-amd64/
 COPY linux-arm64/containerd-*.tar.gz res/linux-arm64/
 

build/docker/tools/provider-res/download.sh

@@ -59,16 +59,35 @@ function download::docker() {
 function download::containerd() {
  if [ "${arch}" == "amd64" ]; then
  containerd_arch=amd64
+ for version in ${CONTAINERD_VERSIONS}; do
+ wget -c "https://github.com/containerd/containerd/releases/download/v${version}/cri-containerd-cni-${version}-linux-${containerd_arch}.tar.gz" \
+ -O "containerd-${platform}-${version}.tar.gz"
+ done
  elif [ "${arch}" == "arm64" ]; then
- containerd_arch=amd64
+ containerd_arch=arm64
+ for version in ${CONTAINERD_VERSIONS}; do
+ wget -c https://tke-release-1251707795.cos.ap-guangzhou.myqcloud.com/cri-containerd-cni-${version}-linux-{containerd_arch}.tar.gz \
+ -O "containerd-${platform}-${version}.tar.gz"
+ done
  else
  echo "[ERROR] Fail to get containerd ${arch} on ${platform} platform."
  exit 255
  fi
+}
+
+function download::nerdctl() {
+ if [ "${arch}" == "amd64" ]; then
+ nerdctl_arch=x86_64
+ elif [ "${arch}" == "arm64" ]; then
+ nerdctl_arch=arm64
+ else
+ echo "[ERROR] Fail to get nerdctl ${arch} on ${platform} platform."
+ exit 255
+ fi
 
- for version in ${CONTAINERD_VERSIONS}; do
- wget -c "https://github.com/containerd/containerd/releases/download/v${version}/cri-containerd-cni-${version}-linux-${containerd_arch}.tar.gz" \
- -O "containerd-${platform}-${version}.tar.gz"
+ for version in ${NERDCTL_VERSIONS}; do
+ wget -c "https://github.com/containerd/nerdctl/releases/download/v${version}/nerdctl-${version}-linux-${arch}.tar.gz" \
+ -O "nerdctl-${platform}-${version}.tar.gz"
  done
 }
 
@@ -148,6 +167,7 @@ for os in ${OSS}; do
  download::cni_plugins
  download::docker
  download::containerd
+ download::nerdctl
  download::critools
  download::kubernetes
  download::nvidia_driver

build/docker/tools/tke-installer/Dockerfile

@@ -14,24 +14,32 @@
 # WARRANTIES OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
-FROM alpine:3.10
+FROM alpine:3.14.0
 
 RUN echo "hosts: files dns" >> /etc/nsswitch.conf
 
 WORKDIR /app
 
 ENV PATH="/app/bin:$PATH"
 ENV DOCKER_CLI_EXPERIMENTAL=enabled
+ARG ENV_ARCH
 
 RUN apk add --no-cache \
  bash \
  busybox-extras \
  curl \
  tcpdump \
  docker \
- ansible
+ ansible \
+ containerd
 
 RUN apk --update-cache --repository http:https://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted add lrzsz
+RUN wget -O nerdctl-0.10.0-linux.tar.gz https://github.com/containerd/nerdctl/releases/download/v0.10.0/nerdctl-0.10.0-linux-"$ENV_ARCH".tar.gz \
+ && tar -zvxf nerdctl-0.10.0-linux.tar.gz -C /usr/local/bin/ \
+ && rm -rf nerdctl-0.10.0-linux.tar.gz \
+ && wget -O crictl-v1.20.0-linux.tar.gz https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.20.0/crictl-v1.20.0-linux-"$ENV_ARCH".tar.gz \
+ && tar -zvxf crictl-v1.20.0-linux.tar.gz -C /usr/local/bin/ \
+ && rm -rf crictl-v1.20.0-linux.tar.gz
 
 ADD . /app
 

build/docker/tools/tke-installer/install.sh

@@ -29,18 +29,30 @@ unset LD_LIBRARY_PATH
 export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
 
 VERSION=latest
+REGISTRY_VERSION=2.7.1
 
 INSTALL_DIR=/opt/tke-installer
 DATA_DIR=$INSTALL_DIR/data
+REGISTRY_DIR=$INSTALL_DIR/registry
 OPTIONS="--name tke-installer -d --privileged --net=host --restart=always
 -v /etc/hosts:/app/hosts
--v /etc/docker:/etc/docker
--v /var/run/docker.sock:/var/run/docker.sock
+-v /var/run/containerd/:/var/run/containerd/
+-v /run/containerd/:/run/containerd/
 -v $DATA_DIR:/app/data
 -v $INSTALL_DIR/conf:/app/conf
 -v registry-certs:/app/certs
 -v tke-installer-bin:/app/bin
 "
+RegistryHTTPOptions="--name registry-http -d --net=host --restart=always -p 80:5000
+-v $REGISTRY_DIR:/var/lib/registry
+"
+RegistryHTTPSOptions="--name registry-https -d --net=host --restart=always -p 443:443
+-v $REGISTRY_DIR:/var/lib/registry
+-v registry-certs:/certs
+-e REGISTRY_HTTP_ADDR=0.0.0.0:443
+-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/server.crt
+-e REGISTRY_HTTP_TLS_KEY=/certs/server.key
+"
 
 declare -A archMap=(
  [x86_64]=amd64
@@ -81,59 +93,61 @@ function check::disk() {
  echo "available disk space($path): $disk_avail GiB"
 }
 
-function ensure_docker() {
- echo "Step.2 check docker status"
+function ensure_containerd() {
+ echo "Step.2 check containerd status"
 
- if ! [ -x "$(command -v docker)" ]; then
- echo "command docker not find"
- install_docker
+ if ! [ -x "$(command -v nerdctl)" ]; then
+ echo "command nerdctl not find"
+ install_containerd
  fi
- if ! systemctl is-active --quiet docker; then
- echo "docker status is not running"
- install_docker
+ if ! systemctl is-active --quiet containerd; then
+ echo "containerd status is not running"
+ install_containerd
  fi
 }
 
-function install_docker() {
- echo "install docker [in process]"
+function install_containerd() {
+ echo "install containerd [in process]"
 
- tar xvaf "res/docker.tgz" -C /usr/bin --strip-components=1
- cp -v res/docker.service /etc/systemd/system
- mkdir -p /etc/docker
- cp -v res/daemon.json /etc/docker/
+ tar xvaf "res/containerd.tar.gz" -C / >/dev/null 2>&1
+ tar xvaf "res/nerdctl.tar.gz" -C /usr/local/bin/ >/dev/null 2>&1
 
  systemctl daemon-reload
 
  # becuase first start docker may be restart some times
- systemctl start docker || :
+ systemctl start containerd || :
  maxSecond=60
  for i in $(seq 1 $maxSecond); do
- if systemctl is-active --quiet docker; then
+ if systemctl is-active --quiet containerd; then
  break
  fi
  sleep 1
  done
  if ((i == maxSecond)); then
- echo "start docker failed, please check docker service."
+ echo "start containerd failed, please check containerd service."
  exit 1
  fi
 
- echo "install docker [done]"
+ echo "install containerd [done]"
+ rm -rf /etc/cni
 }
 
 function load_image() {
  echo "Step.3 load tke-installer image [in process]"
 
- docker load -i res/tke-installer.tgz
-
+ nerdctl load -i res/tke-installer.tar
+ nerdctl load -i res/registry.tar
  echo "Step.3 load tke-installer image [done]"
 }
 
 function clean_old_data() {
  echo "Step.4 clean old data [in process]"
 
- docker rm -f tke-installer >/dev/null 2>&1 || :
- docker volume prune -f >/dev/null 2>&1 || :
+ nerdctl stop tke-installer && nerdctl rm tke-installer >/dev/null 2>&1 || :
+ nerdctl stop registry-http && nerdctl rm registry-http >/dev/null 2>&1 || :
+ nerdctl stop registry-https && nerdctl rm registry-https >/dev/null 2>&1 || :
+ nerdctl volume rm tke-installer-bin >/dev/null 2>&1 || :
+ nerdctl volume rm registry-certs >/dev/null 2>&1 || :
 
  if [ -d "$DATA_DIR" ]; then
  rm -f $DATA_DIR/tke.json >/dev/null 2>&1 || :
@@ -145,37 +159,48 @@ function clean_old_data() {
 
 function start_installer() {
  echo "Step.5 start tke-installer [in process]"
-
- docker run $OPTIONS "tkestack/tke-installer-${ARCH}:$VERSION" $@
+ mkdir -p $DATA_DIR
+ mkdir -p $INSTALL_DIR/conf
+ nerdctl run $OPTIONS "tkestack/tke-installer-${ARCH}:$VERSION" $@
 
  echo "Step.5 start tke-installer [done]"
 }
 
+function start_registry() {
+ echo "Step.6 start regisry [in process]"
+
+ mkdir -p $REGISTRY_DIR
+ nerdctl run $RegistryHTTPOptions "tkestack/registry-${ARCH}:$REGISTRY_VERSION" $@
+ nerdctl run $RegistryHTTPSOptions "tkestack/registry-${ARCH}:$REGISTRY_VERSION" $@
+ echo "Step.6 start registry [done]"
+}
+
 
 function check_installer() {
  s=10
  for i in $(seq 1 $s)
  do
- echo "Step.6 check tke-installer status [in process]"
+ echo "Step.7 check tke-installer status [in process]"
  url="http:https://127.0.0.1:8080/index.html"
  if ! curl -sSf "$url" >/dev/null 2>&1; then
  sleep 3
- echo "Step.6 retries left $(($s-$i))"
+ echo "Step.7 retries left $(($s-$i))"
  continue
  else
- echo "Step.6 check tke-installer status [done]"
+ echo "Step.7 check tke-installer status [done]"
  echo "Please use your browser which can connect this machine to open $url for install TKE!"
  exit 0
  fi
  done
  echo "check installer status error"
- docker logs tke-installer
- exit 1 
+ nerdctl logs tke-installer
+ exit 1
 }
 
 preflight
-ensure_docker
+ensure_containerd
 load_image
 clean_old_data
 start_installer $@
+start_registry $@
 check_installer

build/docker/tools/tke-installer/release.sh

@@ -23,9 +23,12 @@ set -o pipefail
 REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack}
 BUILDER=${BUILDER:-default}
 VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')}
+REGISTRY_VERSION=2.7.1
 PROVIDER_RES_VERSION=v1.20.4-2
 K8S_VERSION=${PROVIDER_RES_VERSION%-*}
-DOCKER_VERSION=19.03.14
+DOCKER_VERSION=20.10.7
+NERDCTL_VERSION=0.10.0
+CONTAINERD_VERSION=1.5.2
 OSS=(linux)
 ARCHS=(amd64 arm64)
 OUTPUT_DIR=_output
@@ -83,7 +86,7 @@ function prepare::tke_installer() {
 function build::installer_image() {
  local -r arch="$1"
 
- docker build --platform="${arch}" --pull -t "${REGISTRY_PREFIX}/tke-installer-${arch}:$VERSION" -f "${SCRIPT_DIR}/Dockerfile" "${DST_DIR}"
+ docker build --platform="${arch}" --build-arg ENV_ARCH="${arch}" --pull -t "${REGISTRY_PREFIX}/tke-installer-${arch}:$VERSION" -f "${SCRIPT_DIR}/Dockerfile" "${DST_DIR}"
 }
 
 function build::installer() {
@@ -102,10 +105,20 @@ function build::installer() {
  "${INSTALLER_DIR}/res/docker.tgz"
  cp -v pkg/platform/provider/baremetal/conf/docker/docker.service "${INSTALLER_DIR}/res/"
  cp -v build/docker/tools/tke-installer/daemon.json "${INSTALLER_DIR}/res/"
-
- docker save "${REGISTRY_PREFIX}/tke-installer-${arch}:$VERSION" | gzip -c > "${INSTALLER_DIR}/res/tke-installer.tgz"
+ cp -v "${DST_DIR}/provider/baremetal/res/${target_platform}/containerd-${target_platform}-${CONTAINERD_VERSION}.tar.gz" \
+ "${INSTALLER_DIR}/res/containerd.tar.gz"
+ cp -v "${DST_DIR}/provider/baremetal/res/${target_platform}/nerdctl-${target_platform}-${NERDCTL_VERSION}.tar.gz" \
+ "${INSTALLER_DIR}/res/nerdctl.tar.gz"
+ cp -v pkg/platform/provider/baremetal/conf/containerd/containerd.service "${INSTALLER_DIR}/res/"
+ cp -v pkg/platform/provider/baremetal/conf/containerd/config.toml "${INSTALLER_DIR}/res/"
+
+ docker save "${REGISTRY_PREFIX}/tke-installer-${arch}:$VERSION" -o "${INSTALLER_DIR}/res/tke-installer.tar"
+ ctr images pull "docker.io/${REGISTRY_PREFIX}/registry-${arch}:$REGISTRY_VERSION"
+ ctr images tag "docker.io/${REGISTRY_PREFIX}/registry-${arch}:$REGISTRY_VERSION" "${REGISTRY_PREFIX}/registry-${arch}:$REGISTRY_VERSION"
+ ctr images export "${INSTALLER_DIR}/res/registry.tar" "${REGISTRY_PREFIX}/registry-${arch}:$REGISTRY_VERSION"
 
  sed -i "s;VERSION=.*;VERSION=$VERSION;g" "${INSTALLER_DIR}/install.sh"
+ sed -i "s;REGISTRY_VERSION=.*;REGISTRY_VERSION=$REGISTRY_VERSION;g" "${INSTALLER_DIR}/install.sh"
 
  "${INSTALLER_DIR}/build.sh" "${installer}"
  cp -v "${INSTALLER_DIR}/${installer}" $OUTPUT_DIR
@@ -126,8 +139,26 @@ function prepare::images() {
  make build BINS=generate-images VERSION="$VERSION"
 
  $GENERATE_IMAGES_BIN
- $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | xargs -n1 -I{} sh -c "docker pull {} || exit 255"
- $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | xargs docker save | gzip -c >"${DST_DIR}"/images.tar.gz
+# $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | xargs -n1 -I{} sh -c "docker pull {} || exit 255"
+# $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | xargs docker save | gzip -c >"${DST_DIR}"/images.tar.gz
+ for((retrynum = 1; retrynum <= 50; retrynum++))
+ do
+ set +e
+ echo "ctr start pulling all images..."
+ $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u | xargs -n1 -I{} sh -c "ctr images pull docker.io/{} --all-platforms"
+ if [ $? -eq 0 ]; then
+ echo "ctr pull image succeed!"
+ break
+ else
+ echo "ctr pull image failed retry pull again..."
+ fi
+ done
+ if [ $retrynum -eq 51 ]; then
+ echo "pull image failed after retry"
+ exit 1
+ fi
+ $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u | xargs -n1 -I{} sh -c "ctr images tag docker.io/{} {}" || true
+ ctr images export "${DST_DIR}"/images.tar `$GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u` --all-platforms || exit 255
 }
 
 pwd

cmd/setup-env/main.go

@@ -36,6 +36,7 @@ func main() {
  env = append(env, fmt.Sprintf("CONTAINERD_VERSIONS=%s", strings.Join(spec.ContainerdVersions, " ")))
  env = append(env, fmt.Sprintf("CRITOOLS_VERSIONS=%s", strings.Join(spec.CriToolsVersions, " ")))
  env = append(env, fmt.Sprintf("CNI_PLUGINS_VERSIONS=%s", strings.Join(spec.CNIPluginsVersions, " ")))
+ env = append(env, fmt.Sprintf("NERDCTL_VERSIONS=%s", strings.Join(spec.NerdctlVersions, " ")))
  env = append(env, fmt.Sprintf("NVIDIA_DRIVER_VERSIONS=%s", strings.Join(spec.NvidiaDriverVersions, " ")))
  env = append(env, fmt.Sprintf("NVIDIA_CONTAINER_RUNTIME_VERSIONS=%s", strings.Join(spec.NvidiaContainerRuntimeVersions, " ")))
 

cmd/tke-installer/app/installer/constants/constants.go

@@ -47,7 +47,7 @@ const (
 
  DevRegistryDomain = "docker.io"
  DevRegistryNamespace = "tkestack"
- ImagesFile = "images.tar.gz"
+ ImagesFile = "images.tar"
  ImagesPattern = DevRegistryNamespace + "/*"
 
  OIDCClientSecretFile = DataDir + "oidc_client_secret"