feat: build arm64 images via make image PLATFORM=linux_arm64

1. Update make help 2. Update golang.mk & image.mk 3. Update doc
tkestack · Feb 27, 2020 · 7165264 · 7165264
1 parent 5405688
commit 7165264
Show file tree

Hide file tree

Showing 5 changed files with 132 additions and 48 deletions.
diff --git a/Makefile b/Makefile
@@ -41,9 +41,15 @@ define USAGE_OPTIONS
 
 Options:
  DEBUG Whether to generate debug symbols. Default is 0.
- IMAGES Backend images to make. All by default.
- PLATFORMS The platform to build. Default is host platform and arch.
  BINS The binaries to build. Default is all of cmd.
+ This option is available when using: make build/build.multiarch
+ Example: make build BINS="tke-platform-api tke-platform-controller"
+ IMAGES Backend images to make. Default is all of cmd starting with tke-.
+ This option is available when using: make image/image.multiarch/push/push.multiarch
+ Example: make image.multiarch IMAGES="tke-platform-api tke-platform-controller"
+ PLATFORMS The multiple platforms to build. Default is linux_amd64 and linux_arm64.
+ This option is available when using: make build.multiarch/image.multiarch/push.multiarch
+ Example: make image.multiarch IMAGES="tke-platform-api tke-platform-controller" PLATFORMS="linux_amd64 linux_arm64"
  VERSION The version information compiled into binaries.
  The default is obtained from git.
  V Set to 1 enable verbose build. Default is 0.
@@ -73,21 +79,31 @@ web:
 build:
  @$(MAKE) go.build
 
-## build.all: Build source code for all platforms.
-.PHONY: build.all
-build.all:
- @$(MAKE) go.build.all
+## build.multiarch: Build source code for multiple platforms. See option PLATFORMS.
+.PHONY: build.multiarch
+build.multiarch:
+ @$(MAKE) go.build.multiarch
 
-## image: Build docker images.
+## image: Build docker images for linux_amd64 platform.
 .PHONY: image
 image:
  @$(MAKE) image.build
 
-## push: Build docker images and push to registry.
+## image.multiarch: Build docker images for multiple platforms. See option PLATFORMS.
+.PHONY: image.multiarch
+image.multiarch:
+ @$(MAKE) image.build.multiarch
+
+## push: Build docker images for linux_amd64 platform and push images to registry.
 .PHONY: push
 push:
  @$(MAKE) image.push
 
+## push.multiarch: Build docker images for multiple platforms and push manifests to registry.
+.PHONY: push.multiarch
+push.multiarch:
+ @$(MAKE) image.push.multiarch
+
 ## deploy: Deploy updated components to development env.
 .PHONY: deploy
 deploy:

diff --git a/build/lib/common.mk b/build/lib/common.mk
@@ -47,6 +47,28 @@ ifeq (, $(shell git status --porcelain 2>/dev/null))
 endif
 GIT_COMMIT:=$(shell git rev-parse HEAD)
 
+# The OS must be linux when building docker images
+PLATFORMS ?= linux_amd64 linux_arm64
+# The OS can be linux/windows/darwin when building binaries
+# PLATFORMS ?= darwin_amd64 windows_amd64 linux_amd64 linux_arm64
+
+# Set a specific PLATFORM
+ifeq ($(origin PLATFORM), undefined)
+ ifeq ($(origin GOOS), undefined)
+ GOOS := $(shell go env GOOS)
+ endif
+ ifeq ($(origin GOARCH), undefined)
+ GOARCH := $(shell go env GOARCH)
+ endif
+ PLATFORM := $(GOOS)_$(GOARCH)
+ # Use linux_amd64 as the default platform when building images
+ IMAGE_PLAT := linux_amd64
+else
+ GOOS := $(word 1, $(subst _, ,$(PLATFORM)))
+ GOARCH := $(word 2, $(subst _, ,$(PLATFORM)))
+ IMAGE_PLAT := $(PLATFORM)
+endif
+
 COMMA := ,
 SPACE :=
 SPACE +=

diff --git a/build/lib/golang.mk b/build/lib/golang.mk
@@ -33,25 +33,11 @@ ifeq ($(ROOT_PACKAGE),)
  $(error the variable ROOT_PACKAGE must be set prior to including golang.mk)
 endif
 
-ifeq ($(origin PLATFORM), undefined)
- ifeq ($(origin GOOS), undefined)
- GOOS := $(shell go env GOOS)
- endif
- ifeq ($(origin GOARCH), undefined)
- GOARCH := $(shell go env GOARCH)
- endif
- PLATFORM := $(GOOS)_$(GOARCH)
-else
- GOOS := $(word 1, $(subst _, ,$(PLATFORM)))
- GOARCH := $(word 2, $(subst _, ,$(PLATFORM)))
-endif
-
 GOPATH := $(shell go env GOPATH)
 ifeq ($(origin GOBIN), undefined)
  GOBIN := $(GOPATH)/bin
 endif
 
-PLATFORMS ?= darwin_amd64 windows_amd64 linux_amd64
 COMMANDS ?= $(filter-out %.md, $(wildcard ${ROOT_DIR}/cmd/*))
 BINS ?= $(foreach cmd,${COMMANDS},$(notdir ${cmd}))
 
@@ -81,8 +67,8 @@ go.build.%:
 .PHONY: go.build
 go.build: go.build.verify $(addprefix go.build., $(addprefix $(PLATFORM)., $(BINS)))
 
-.PHONY: go.build.all
-go.build.all: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS))))
+.PHONY: go.build.multiarch
+go.build.multiarch: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS))))
 
 .PHONY: go.clean
 go.clean:

diff --git a/build/lib/image.mk b/build/lib/image.mk
@@ -19,7 +19,7 @@
 #
 
 DOCKER := docker
-DOCKER_SUPPORTED_VERSIONS ?= 17|18|19
+DOCKER_SUPPORTED_VERSIONS ?= 18|19
 
 REGISTRY_PREFIX ?= tkestack
 BASE_IMAGE = alpine:3.10
@@ -52,23 +52,52 @@ endif
  @echo "===========> Docker version verification passed"
 
 .PHONY: image.build
-image.build: image.verify go.build.verify $(addprefix image.build., $(IMAGES))
+image.build: image.verify go.build.verify $(addprefix image.build., $(addprefix $(IMAGE_PLAT)., $(IMAGES)))
 
-.PHONY: image.push
-image.push: image.verify go.build.verify $(addprefix image.push., $(IMAGES))
+.PHONY: image.build.multiarch
+image.build.multiarch: image.verify go.build.verify $(foreach p,$(PLATFORMS),$(addprefix image.build., $(addprefix $(p)., $(IMAGES))))
 
 .PHONY: image.build.%
-image.build.%: go.build.linux_amd64.%
- @echo "===========> Building $* $(VERSION) docker image"
- @mkdir -p $(TMP_DIR)/$*
- @cat $(ROOT_DIR)/build/docker/$*/Dockerfile\
- | sed "s#BASE_IMAGE#$(BASE_IMAGE)#g" >$(TMP_DIR)/$*/Dockerfile
- @cp ${OUTPUT_DIR}/linux/amd64/$* $(TMP_DIR)/$*/
- @DST_DIR=$(TMP_DIR)/$* $(ROOT_DIR)/build/docker/$*/build.sh 2>/dev/null || true
- @$(DOCKER) build $(_DOCKER_BUILD_EXTRA_ARGS) --pull -t $(REGISTRY_PREFIX)/$*:$(VERSION) $(TMP_DIR)/$*
- @rm -rf $(TMP_DIR)/$*
+image.build.%: go.build.%
+ $(eval IMAGE := $(COMMAND))
+ $(eval IMAGE_PLAT := $(subst _,/,$(PLATFORM)))
+ @echo "===========> Building docker image $(IMAGE) $(VERSION) for $(IMAGE_PLAT)"
+ @mkdir -p $(TMP_DIR)/$(IMAGE)
+ @cat $(ROOT_DIR)/build/docker/$(IMAGE)/Dockerfile\
+ | sed "s#BASE_IMAGE#$(BASE_IMAGE)#g" >$(TMP_DIR)/$(IMAGE)/Dockerfile
+ @cp $(OUTPUT_DIR)/$(IMAGE_PLAT)/$(IMAGE) $(TMP_DIR)/$(IMAGE)/
+ @DST_DIR=$(TMP_DIR)/$(IMAGE) $(ROOT_DIR)/build/docker/$(IMAGE)/build.sh 2>/dev/null || true
+ $(DOCKER) build --platform $(IMAGE_PLAT) $(_DOCKER_BUILD_EXTRA_ARGS) --pull \
+ -t $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) $(TMP_DIR)/$(IMAGE)
+ @rm -rf $(TMP_DIR)/$(IMAGE)
+
+.PHONY: image.push
+image.push: image.verify go.build.verify $(addprefix image.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES)))
 
 .PHONY: image.push.%
 image.push.%: image.build.%
- @echo "===========> Pushing $* $(VERSION) image to $(REGISTRY_PREFIX)"
- @$(DOCKER) push $(REGISTRY_PREFIX)/$*:$(VERSION)
+ @echo "===========> Pushing image $(IMAGE) $(VERSION) to $(REGISTRY_PREFIX)"
+ $(DOCKER) push $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION)
+ @$(DOCKER) manifest create --amend $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \
+ $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION)
+ @$(DOCKER) manifest annotate $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \
+ $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) \
+ --os $(OS) --arch ${ARCH}
+
+.PHONY: image.push.multiarch
+image.push.multiarch: image.verify go.build.verify manifest.remove \
+$(foreach p,$(PLATFORMS),$(addprefix image.push., $(addprefix $(p)., $(IMAGES)))) \
+$(addprefix image.push.manifest., $(IMAGES))
+
+# Docker cli has a bug: https://github.com/docker/cli/issues/954
+# If you find your manifests were not updated,
+# Please manually delete them in $HOME/.docker/manifests/
+# and re-run.
+.PHONY: manifest.remove
+manifest.remove:
+ @rm -rf ${HOME}/.docker/manifests/docker.io_$(REGISTRY_PREFIX)_tke-*
+
+.PHONY: image.push.manifest.%
+image.push.manifest.%:
+ @echo "===========> Pushing manifest $* $(VERSION) to $(REGISTRY_PREFIX) and then remove the local manifest list"
+ @$(DOCKER) manifest push --purge $(REGISTRY_PREFIX)/$*:$(VERSION)
diff --git a/docs/devel/development.md b/docs/devel/development.md
@@ -2,12 +2,14 @@
 
 **Table of Contents**
 
-- [Requirements](#requirements)
+- [Development Guide](#development-guide)
+ - [Requirements](#requirements)
  - [Go](#go)
- - [Node.js](#node.js)
-- [Clone source code](#clone-source-code)
-- [Building binary](#building-binary)
-- [Building docker image](#building-docker-image)
+ - [Node.js](#nodejs)
+ - [Clone source code](#clone-source-code)
+ - [Building binary](#building-binary)
+ - [Building docker image](#building-docker-image)
+ - [Releasing docker image](#releasing-docker-image)
 
 This document is the canonical source of truth for things like supported
 toolchain versions for building TKE.
@@ -108,17 +110,17 @@ cd ${working_dir}/tke
 make
 ```
 
-To build binaries for all platforms:
+To build binaries for multiple platforms:
 
 ```sh
-make build.all
+make build.multiarch
 ```
 
 To build a specific os/arch of TKE use the `PLATFORMS` environment variable to
 let the build scripts know you want to build only for os/arch.
 
 ```sh
-make build.all PLATFORMS="linux_amd64 windows_amd64 darwin_amd64"
+make build.multiarch PLATFORMS="linux_amd64 windows_amd64 darwin_amd64"
 ```
 
 ## Building docker image
@@ -146,12 +148,41 @@ make image IMAGES=${package_you_want}
 make image IMAGES="${package_you_want_1} ${package_you_want_2}"
 ```
 
-*Note:* This applies to all top level folders under tke/cmd.
+*Note:* This applies to all top level folders under build/docker 
+(except build/docker/tools).
 
 So for the tke-platform-api, you can run:
 
 ```sh
 make image IMAGES=tke-platform-api
 ```
 
+To build container images for multiple platforms (i.e., linux_amd64 and linux_arm64), type:
+
+```sh
+make image.multiarch
+```
+
+Above command will use experimental features of docker daemon (i.e. docker build --platform). 
+Please refer to [docker build docs](https://docs.docker.com/engine/reference/commandline/build/#--platform) to enable experimental features.
+
+To build a specific os/arch for TKE container images, please use the `PLATFORMS` environment variable to
+let the build scripts know which os/arch you want to build.
+
+```sh
+make image.multiarch PLATFORMS="linux_amd64 linux_arm64"
+```
+
+## Releasing docker image
+
+Below is a quick start on how to push TKE container images to docker hub.
+
+```sh
+make push
+```
+
+TKEStack manages docker images via manifests and manifest lists.
+Please make sure you enable experimental features in the Docker CLI.
+You can find more details in [docker manifest docs](https://docs.docker.com/engine/reference/commandline/manifest/).
+
 For more functions of other components, please see [here](/docs/devel/components.md). To run tke system locally, please see [here](/docs/devel/running-locally.md).