Merge branch 'master' into lis-support-containerd-install

build/docker/tools/provider-res/download.sh

@@ -59,17 +59,20 @@ function download::docker() {
 function download::containerd() {
  if [ "${arch}" == "amd64" ]; then
  containerd_arch=amd64
+ for version in ${CONTAINERD_VERSIONS}; do
+ wget -c "https://github.com/containerd/containerd/releases/download/v${version}/cri-containerd-cni-${version}-linux-${containerd_arch}.tar.gz" \
+ -O "containerd-${platform}-${version}.tar.gz"
+ done
  elif [ "${arch}" == "arm64" ]; then
- containerd_arch=amd64
+ containerd_arch=arm64
+ for version in ${CONTAINERD_VERSIONS}; do
+ wget -c https://tke-release-1251707795.cos.ap-guangzhou.myqcloud.com/cri-containerd-cni-${version}-linux-{containerd_arch}.tar.gz \
+ -O "containerd-${platform}-${version}.tar.gz"
+ done
  else
  echo "[ERROR] Fail to get containerd ${arch} on ${platform} platform."
  exit 255
  fi
-
- for version in ${CONTAINERD_VERSIONS}; do
- wget -c "https://github.com/containerd/containerd/releases/download/v${version}/cri-containerd-cni-${version}-linux-${containerd_arch}.tar.gz" \
- -O "containerd-${platform}-${version}.tar.gz"
- done
 }
 
 function download::nerdctl() {

build/docker/tools/tke-installer/Dockerfile

@@ -22,6 +22,7 @@ WORKDIR /app
 
 ENV PATH="/app/bin:$PATH"
 ENV DOCKER_CLI_EXPERIMENTAL=enabled
+ARG ENV_ARCH
 
 RUN apk add --no-cache \
  bash \
@@ -33,12 +34,12 @@ RUN apk add --no-cache \
  containerd
 
 RUN apk --update-cache --repository http:https://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted add lrzsz
-RUN wget https://github.com/containerd/nerdctl/releases/download/v0.10.0/nerdctl-0.10.0-linux-amd64.tar.gz \
- && tar -zvxf nerdctl-0.10.0-linux-amd64.tar.gz -C /usr/local/bin/ \
- && rm -rf nerdctl-0.10.0-linux-amd64.tar.gz \
- && wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.20.0/crictl-v1.20.0-linux-amd64.tar.gz \
- && tar -zvxf crictl-v1.20.0-linux-amd64.tar.gz -C /usr/local/bin/ \
- && rm -rf crictl-v1.20.0-linux-amd64.tar.gz
+RUN wget -O nerdctl-0.10.0-linux.tar.gz https://github.com/containerd/nerdctl/releases/download/v0.10.0/nerdctl-0.10.0-linux-"$ENV_ARCH".tar.gz \
+ && tar -zvxf nerdctl-0.10.0-linux.tar.gz -C /usr/local/bin/ \
+ && rm -rf nerdctl-0.10.0-linux.tar.gz \
+ && wget -O crictl-v1.20.0-linux.tar.gz https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.20.0/crictl-v1.20.0-linux-"$ENV_ARCH".tar.gz \
+ && tar -zvxf crictl-v1.20.0-linux.tar.gz -C /usr/local/bin/ \
+ && rm -rf crictl-v1.20.0-linux.tar.gz
 
 ADD . /app
 

build/docker/tools/tke-installer/install.sh

@@ -36,8 +36,8 @@ DATA_DIR=$INSTALL_DIR/data
 REGISTRY_DIR=$INSTALL_DIR/registry
 OPTIONS="--name tke-installer -d --privileged --net=host --restart=always
 -v /etc/hosts:/app/hosts
--v /var/run/containerd/containerd.sock:/var/run/containerd/containerd.sock
--v /run/containerd/containerd.sock:/run/containerd/containerd.sock
+-v /var/run/containerd/:/var/run/containerd/
+-v /run/containerd/:/run/containerd/
 -v $DATA_DIR:/app/data
 -v $INSTALL_DIR/conf:/app/conf
 -v registry-certs:/app/certs
@@ -109,8 +109,8 @@ function ensure_containerd() {
 function install_containerd() {
  echo "install containerd [in process]"
 
- tar xvaf "res/containerd.tar.gz" -C /
- tar xvaf "res/nerdctl.tar.gz" -C /usr/local/bin/
+ tar xvaf "res/containerd.tar.gz" -C / >/dev/null 2>&1
+ tar xvaf "res/nerdctl.tar.gz" -C /usr/local/bin/ >/dev/null 2>&1
 
  systemctl daemon-reload
 
@@ -129,6 +129,7 @@ function install_containerd() {
  fi
 
  echo "install containerd [done]"
+ rm -rf /etc/cni
 }
 
 function load_image() {

build/docker/tools/tke-installer/release.sh

@@ -86,7 +86,7 @@ function prepare::tke_installer() {
 function build::installer_image() {
  local -r arch="$1"
 
- docker build --platform="${arch}" --pull -t "${REGISTRY_PREFIX}/tke-installer-${arch}:$VERSION" -f "${SCRIPT_DIR}/Dockerfile" "${DST_DIR}"
+ docker build --platform="${arch}" --build-arg ENV_ARCH="${arch}" --pull -t "${REGISTRY_PREFIX}/tke-installer-${arch}:$VERSION" -f "${SCRIPT_DIR}/Dockerfile" "${DST_DIR}"
 }
 
 function build::installer() {
@@ -143,15 +143,20 @@ function prepare::images() {
 # $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | xargs docker save | gzip -c >"${DST_DIR}"/images.tar.gz
  for((retrynum = 1; retrynum <= 50; retrynum++))
  do
- ctr images pull docker.io/${REGISTRY_PREFIX}/provider-res:${PROVIDER_RES_VERSION} --all-platforms
+ set +e
+ echo "ctr start pulling all images..."
+ $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u | xargs -n1 -I{} sh -c "ctr images pull docker.io/{} --all-platforms"
  if [ $? -eq 0 ]; then
- echo "ctr pull provider-res image succeed!"
+ echo "ctr pull image succeed!"
  break
  else
- echo "ctr pull provider-res image failed retry pull again..."
+ echo "ctr pull image failed retry pull again..."
  fi
  done
- $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u | xargs -n1 -I{} sh -c "ctr images pull docker.io/{} --all-platforms || exit 255"
+ if [ $retrynum -eq 51 ]; then
+ echo "pull image failed after retry"
+ exit 1
+ fi
  $GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u | xargs -n1 -I{} sh -c "ctr images tag docker.io/{} {}" || true
  ctr images export "${DST_DIR}"/images.tar `$GENERATE_IMAGES_BIN | sed "s;^;${REGISTRY_PREFIX}/;" | sed "s;-amd64;;" | sed "s;-arm64;;" | sort -u` --all-platforms || exit 255
 }
@@ -191,5 +196,3 @@ done
 
 rm -rf "${DST_DIR}"
 rm -rf "${INSTALLER_DIR}"
-
-

pkg/platform/provider/baremetal/phases/containerd/containerd.go

@@ -22,7 +22,6 @@ import (
  "bytes"
  "fmt"
  "path"
-
  "tkestack.io/tke/pkg/util/template"
 
  "github.com/pkg/errors"
@@ -79,5 +78,11 @@ func Install(s ssh.Interface, option *Option) error {
  return err
  }
 
+ cmd = "rm -rf /etc/cni/net.d/10-containerd-net.conflist"
+ _, stderr, exit, err = s.Execf(cmd)
+ if err != nil {
+ return fmt.Errorf("exec %q failed:exit %d:stderr %s:error %s", cmd, exit, stderr, err)
+ }
+
  return nil
 }

pkg/util/docker/docker.go

@@ -231,13 +231,8 @@ func (d *Docker) RemoveImage(image string) error {
 
 // RemoveContainers forces to remove one or more running containers.
 func (d *Docker) RemoveContainers(containers ...string) error {
- cmd := fmt.Sprintf("nerdctl stop tke-installer && nerdctl start tke-installer || true")
- err := d.runCmd(cmd)
- if err != nil {
- return pkgerrors.Wrap(err, "nerdctl restart tke-installer error")
- }
  for _, one := range containers {
- cmdString := fmt.Sprintf("nerdctl inspect %s >/dev/null 2>&1 && nerdctl stop %s && nerdctl rm %s || true", one, one, one)
+ cmdString := fmt.Sprintf("nerdctl stop %s && nerdctl rm %s || true", one, one)
  err := d.runCmd(cmdString)
  if err != nil {
  return pkgerrors.Wrap(err, "nerdctl remove containers error")

test/e2e/manifests/tke-platform-controller/tke-platform-controller.yaml

@@ -36,6 +36,8 @@ spec:
  mountPath: /app/provider/baremetal/conf/kubelet/
  - name: kubeadm-volume
  mountPath: /app/provider/baremetal/conf/kubeadm/
+ - name: containerd-volume
+ mountPath: /app/provider/baremetal/conf/containerd/
  ports:
  - containerPort: 9445
  livenessProbe:
@@ -60,6 +62,9 @@ spec:
  - name: docker-volume
  configMap:
  name: docker
+ - name: containerd-volume
+ configMap:
+ name: containerd
  - name: kubelet-volume
  configMap:
  name: kubelet

test/e2e/tke/tke.go

@@ -131,6 +131,10 @@ func (t *TKE) createProviderConfigs(ctx context.Context) error {
  Name: "kubeadm",
  File: e2e.ConfDir + "kubeadm/*",
  },
+ {
+ Name: "containerd",
+ File: e2e.ConfDir + "containerd/*",
+ },
  }
 
  for _, one := range configMaps {