From 7165264bba82e8a910d2454276b1e9f1342148b4 Mon Sep 17 00:00:00 2001
From: Shangru-WU <26450747+Shangru-WU@users.noreply.github.com>
Date: Tue, 25 Feb 2020 22:46:31 +0800
Subject: [PATCH] feat: build arm64 images via make image PLATFORM=linux_arm64

1. Update make help
2. Update golang.mk & image.mk
3. Update doc
---
 Makefile                  | 32 +++++++++++++++------
 build/lib/common.mk       | 22 +++++++++++++++
 build/lib/golang.mk       | 18 ++----------
 build/lib/image.mk        | 59 +++++++++++++++++++++++++++++----------
 docs/devel/development.md | 49 ++++++++++++++++++++++++++------
 5 files changed, 132 insertions(+), 48 deletions(-)

diff --git a/Makefile b/Makefile
index c014bd345..5af8d663a 100644
--- a/Makefile
+++ b/Makefile
@@ -41,9 +41,15 @@ define USAGE_OPTIONS
 
 Options:
   DEBUG        Whether to generate debug symbols. Default is 0.
-  IMAGES       Backend images to make. All by default.
-  PLATFORMS    The platform to build. Default is host platform and arch.
   BINS         The binaries to build. Default is all of cmd.
+               This option is available when using: make build/build.multiarch
+               Example: make build BINS="tke-platform-api tke-platform-controller"
+  IMAGES       Backend images to make. Default is all of cmd starting with tke-.
+               This option is available when using: make image/image.multiarch/push/push.multiarch
+               Example: make image.multiarch IMAGES="tke-platform-api tke-platform-controller"
+  PLATFORMS    The multiple platforms to build. Default is linux_amd64 and linux_arm64.
+               This option is available when using: make build.multiarch/image.multiarch/push.multiarch
+               Example: make image.multiarch IMAGES="tke-platform-api tke-platform-controller" PLATFORMS="linux_amd64 linux_arm64"
   VERSION      The version information compiled into binaries.
                The default is obtained from git.
   V            Set to 1 enable verbose build. Default is 0.
@@ -73,21 +79,31 @@ web:
 build:
 	@$(MAKE) go.build
 
-## build.all: Build source code for all platforms.
-.PHONY: build.all
-build.all:
-	@$(MAKE) go.build.all
+## build.multiarch: Build source code for multiple platforms. See option PLATFORMS.
+.PHONY: build.multiarch
+build.multiarch:
+	@$(MAKE) go.build.multiarch
 
-## image: Build docker images.
+## image: Build docker images for linux_amd64 platform.
 .PHONY: image
 image:
 	@$(MAKE) image.build
 
-## push: Build docker images and push to registry.
+## image.multiarch: Build docker images for multiple platforms. See option PLATFORMS.
+.PHONY: image.multiarch
+image.multiarch:
+	@$(MAKE) image.build.multiarch
+
+## push: Build docker images for linux_amd64 platform and push images to registry.
 .PHONY: push
 push:
 	@$(MAKE) image.push
 
+## push.multiarch: Build docker images for multiple platforms and push manifests to registry.
+.PHONY: push.multiarch
+push.multiarch:
+	@$(MAKE) image.push.multiarch
+
 ## deploy: Deploy updated components to development env.
 .PHONY: deploy
 deploy:
diff --git a/build/lib/common.mk b/build/lib/common.mk
index 47c8e3b3b..76d3e7ee9 100644
--- a/build/lib/common.mk
+++ b/build/lib/common.mk
@@ -47,6 +47,28 @@ ifeq (, $(shell git status --porcelain 2>/dev/null))
 endif
 GIT_COMMIT:=$(shell git rev-parse HEAD)
 
+# The OS must be linux when building docker images
+PLATFORMS ?= linux_amd64 linux_arm64
+# The OS can be linux/windows/darwin when building binaries
+# PLATFORMS ?= darwin_amd64 windows_amd64 linux_amd64 linux_arm64
+
+# Set a specific PLATFORM
+ifeq ($(origin PLATFORM), undefined)
+	ifeq ($(origin GOOS), undefined)
+		GOOS := $(shell go env GOOS)
+	endif
+	ifeq ($(origin GOARCH), undefined)
+		GOARCH := $(shell go env GOARCH)
+	endif
+	PLATFORM := $(GOOS)_$(GOARCH)
+	# Use linux_amd64 as the default platform when building images
+	IMAGE_PLAT := linux_amd64
+else
+	GOOS := $(word 1, $(subst _, ,$(PLATFORM)))
+	GOARCH := $(word 2, $(subst _, ,$(PLATFORM)))
+	IMAGE_PLAT := $(PLATFORM)
+endif
+
 COMMA := ,
 SPACE :=
 SPACE +=
diff --git a/build/lib/golang.mk b/build/lib/golang.mk
index e72886c36..d6d3e9c79 100644
--- a/build/lib/golang.mk
+++ b/build/lib/golang.mk
@@ -33,25 +33,11 @@ ifeq ($(ROOT_PACKAGE),)
 	$(error the variable ROOT_PACKAGE must be set prior to including golang.mk)
 endif
 
-ifeq ($(origin PLATFORM), undefined)
-	ifeq ($(origin GOOS), undefined)
-		GOOS := $(shell go env GOOS)
-	endif
-	ifeq ($(origin GOARCH), undefined)
-		GOARCH := $(shell go env GOARCH)
-	endif
-	PLATFORM := $(GOOS)_$(GOARCH)
-else
-	GOOS := $(word 1, $(subst _, ,$(PLATFORM)))
-	GOARCH := $(word 2, $(subst _, ,$(PLATFORM)))
-endif
-
 GOPATH := $(shell go env GOPATH)
 ifeq ($(origin GOBIN), undefined)
 	GOBIN := $(GOPATH)/bin
 endif
 
-PLATFORMS ?= darwin_amd64 windows_amd64 linux_amd64
 COMMANDS ?= $(filter-out %.md, $(wildcard ${ROOT_DIR}/cmd/*))
 BINS ?= $(foreach cmd,${COMMANDS},$(notdir ${cmd}))
 
@@ -81,8 +67,8 @@ go.build.%:
 .PHONY: go.build
 go.build: go.build.verify $(addprefix go.build., $(addprefix $(PLATFORM)., $(BINS)))
 
-.PHONY: go.build.all
-go.build.all: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS))))
+.PHONY: go.build.multiarch
+go.build.multiarch: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS))))
 
 .PHONY: go.clean
 go.clean:
diff --git a/build/lib/image.mk b/build/lib/image.mk
index 634af92c8..0abf320e1 100644
--- a/build/lib/image.mk
+++ b/build/lib/image.mk
@@ -19,7 +19,7 @@
 #
 
 DOCKER := docker
-DOCKER_SUPPORTED_VERSIONS ?= 17|18|19
+DOCKER_SUPPORTED_VERSIONS ?= 18|19
 
 REGISTRY_PREFIX ?= tkestack
 BASE_IMAGE = alpine:3.10
@@ -52,23 +52,52 @@ endif
 	@echo "===========> Docker version verification passed"
 
 .PHONY: image.build
-image.build: image.verify go.build.verify $(addprefix image.build., $(IMAGES))
+image.build: image.verify go.build.verify $(addprefix image.build., $(addprefix $(IMAGE_PLAT)., $(IMAGES)))
 
-.PHONY: image.push
-image.push: image.verify go.build.verify $(addprefix image.push., $(IMAGES))
+.PHONY: image.build.multiarch
+image.build.multiarch: image.verify go.build.verify $(foreach p,$(PLATFORMS),$(addprefix image.build., $(addprefix $(p)., $(IMAGES))))
 
 .PHONY: image.build.%
-image.build.%: go.build.linux_amd64.%
-	@echo "===========> Building $* $(VERSION) docker image"
-	@mkdir -p $(TMP_DIR)/$*
-	@cat $(ROOT_DIR)/build/docker/$*/Dockerfile\
-		| sed "s#BASE_IMAGE#$(BASE_IMAGE)#g" >$(TMP_DIR)/$*/Dockerfile
-	@cp ${OUTPUT_DIR}/linux/amd64/$* $(TMP_DIR)/$*/
-	@DST_DIR=$(TMP_DIR)/$* $(ROOT_DIR)/build/docker/$*/build.sh 2>/dev/null || true
-	@$(DOCKER) build $(_DOCKER_BUILD_EXTRA_ARGS) --pull -t $(REGISTRY_PREFIX)/$*:$(VERSION) $(TMP_DIR)/$*
-	@rm -rf $(TMP_DIR)/$*
+image.build.%: go.build.%
+	$(eval IMAGE := $(COMMAND))
+	$(eval IMAGE_PLAT := $(subst _,/,$(PLATFORM)))
+	@echo "===========> Building docker image $(IMAGE) $(VERSION) for $(IMAGE_PLAT)"
+	@mkdir -p $(TMP_DIR)/$(IMAGE)
+	@cat $(ROOT_DIR)/build/docker/$(IMAGE)/Dockerfile\
+		| sed "s#BASE_IMAGE#$(BASE_IMAGE)#g" >$(TMP_DIR)/$(IMAGE)/Dockerfile
+	@cp $(OUTPUT_DIR)/$(IMAGE_PLAT)/$(IMAGE) $(TMP_DIR)/$(IMAGE)/
+	@DST_DIR=$(TMP_DIR)/$(IMAGE) $(ROOT_DIR)/build/docker/$(IMAGE)/build.sh 2>/dev/null || true
+	$(DOCKER) build --platform $(IMAGE_PLAT) $(_DOCKER_BUILD_EXTRA_ARGS) --pull \
+	-t $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) $(TMP_DIR)/$(IMAGE)
+	@rm -rf $(TMP_DIR)/$(IMAGE)
+
+.PHONY: image.push
+image.push: image.verify go.build.verify $(addprefix image.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES)))
 
 .PHONY: image.push.%
 image.push.%: image.build.%
-	@echo "===========> Pushing $* $(VERSION) image to $(REGISTRY_PREFIX)"
-	@$(DOCKER) push $(REGISTRY_PREFIX)/$*:$(VERSION)
+	@echo "===========> Pushing image $(IMAGE) $(VERSION) to $(REGISTRY_PREFIX)"
+	$(DOCKER) push $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION)
+	@$(DOCKER) manifest create --amend $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \
+		$(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION)
+	@$(DOCKER) manifest annotate $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \
+		$(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) \
+		--os $(OS) --arch ${ARCH}
+
+.PHONY: image.push.multiarch
+image.push.multiarch: image.verify go.build.verify manifest.remove \
+$(foreach p,$(PLATFORMS),$(addprefix image.push., $(addprefix $(p)., $(IMAGES)))) \
+$(addprefix image.push.manifest., $(IMAGES))
+
+# Docker cli has a bug: https://github.com/docker/cli/issues/954
+# If you find your manifests were not updated,
+# Please manually delete them in $HOME/.docker/manifests/
+# and re-run.
+.PHONY: manifest.remove
+manifest.remove:
+	@rm -rf ${HOME}/.docker/manifests/docker.io_$(REGISTRY_PREFIX)_tke-*
+
+.PHONY: image.push.manifest.%
+image.push.manifest.%:
+	@echo "===========> Pushing manifest $* $(VERSION) to $(REGISTRY_PREFIX) and then remove the local manifest list"
+	@$(DOCKER) manifest push --purge $(REGISTRY_PREFIX)/$*:$(VERSION)
diff --git a/docs/devel/development.md b/docs/devel/development.md
index 070a96059..baf412be3 100644
--- a/docs/devel/development.md
+++ b/docs/devel/development.md
@@ -2,12 +2,14 @@
 
 **Table of Contents**
 
-- [Requirements](#requirements)
+- [Development Guide](#development-guide)
+  - [Requirements](#requirements)
     - [Go](#go)
-    - [Node.js](#node.js)
-- [Clone source code](#clone-source-code)
-- [Building binary](#building-binary)
-- [Building docker image](#building-docker-image)
+    - [Node.js](#nodejs)
+  - [Clone source code](#clone-source-code)
+  - [Building binary](#building-binary)
+  - [Building docker image](#building-docker-image)
+  - [Releasing docker image](#releasing-docker-image)
 
 This document is the canonical source of truth for things like supported
 toolchain versions for building TKE.
@@ -108,17 +110,17 @@ cd ${working_dir}/tke
 make
 ```
 
-To build binaries for all platforms:
+To build binaries for multiple platforms:
 
 ```sh
-make build.all
+make build.multiarch
 ```
 
 To build a specific os/arch of TKE use the `PLATFORMS` environment variable to
 let the build scripts know you want to build only for os/arch.
 
 ```sh
-make build.all PLATFORMS="linux_amd64 windows_amd64 darwin_amd64"
+make build.multiarch PLATFORMS="linux_amd64 windows_amd64 darwin_amd64"
 ```
 
 ## Building docker image
@@ -146,7 +148,8 @@ make image IMAGES=${package_you_want}
 make image IMAGES="${package_you_want_1} ${package_you_want_2}"
 ```
 
-*Note:* This applies to all top level folders under tke/cmd.
+*Note:* This applies to all top level folders under build/docker 
+(except build/docker/tools).
 
 So for the tke-platform-api, you can run:
 
@@ -154,4 +157,32 @@ So for the tke-platform-api, you can run:
 make image IMAGES=tke-platform-api
 ```
 
+To build container images for multiple platforms (i.e., linux_amd64 and linux_arm64), type:
+
+```sh
+make image.multiarch
+```
+
+Above command will use experimental features of docker daemon (i.e. docker build --platform). 
+Please refer to [docker build docs](https://docs.docker.com/engine/reference/commandline/build/#--platform) to enable experimental features.
+
+To build a specific os/arch for TKE container images, please use the `PLATFORMS` environment variable to
+let the build scripts know which os/arch you want to build.
+
+```sh
+make image.multiarch PLATFORMS="linux_amd64 linux_arm64"
+```
+
+## Releasing docker image
+
+Below is a quick start on how to push TKE container images to docker hub.
+
+```sh
+make push
+```
+
+TKEStack manages docker images via manifests and manifest lists.
+Please make sure you enable experimental features in the Docker CLI.
+You can find more details in [docker manifest docs](https://docs.docker.com/engine/reference/commandline/manifest/).
+
 For more functions of other components, please see [here](/docs/devel/components.md). To run tke system locally, please see [here](/docs/devel/running-locally.md).