From a8c1ac48fb022befef30a249d5b08249d2ba5e22 Mon Sep 17 00:00:00 2001 From: leonarliu Date: Thu, 21 Jan 2021 17:00:09 +0800 Subject: [PATCH 1/5] feat(platform): update support k8s to 1.19.7 --- build/docker/tools/provider-res/Makefile | 2 +- build/docker/tools/tke-installer/release.sh | 2 +- cmd/setup-env/main.go | 2 +- cmd/tke-installer/app/installer/images/images.go | 2 +- cmd/tke-installer/app/installer/installer.go | 2 +- cmd/tke-installer/app/installer/prepare.go | 2 +- cmd/tke-installer/app/installer/upgrader.go | 2 +- pkg/platform/provider/baremetal/images/images.go | 2 +- pkg/platform/provider/baremetal/res/res.go | 2 +- pkg/spec/spec.go | 7 +------ pkg/util/apiclient/client_test.go | 2 +- 11 files changed, 11 insertions(+), 16 deletions(-) diff --git a/build/docker/tools/provider-res/Makefile b/build/docker/tools/provider-res/Makefile index f31b86e17..72cb099d9 100644 --- a/build/docker/tools/provider-res/Makefile +++ b/build/docker/tools/provider-res/Makefile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations under the License. IMAGE = provider-res -VERSION = v1.18.3-3 +VERSION = v1.19.7-1 REGISTRY_PREFIX ?= tkestack COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) diff --git a/build/docker/tools/tke-installer/release.sh b/build/docker/tools/tke-installer/release.sh index a400578cd..aec4d1f44 100755 --- a/build/docker/tools/tke-installer/release.sh +++ b/build/docker/tools/tke-installer/release.sh @@ -23,7 +23,7 @@ set -o pipefail REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack} BUILDER=${BUILDER:-default} VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')} -PROVIDER_RES_VERSION=v1.18.3-3 +PROVIDER_RES_VERSION=v1.19.7-1 K8S_VERSION=${PROVIDER_RES_VERSION%-*} DOCKER_VERSION=19.03.14 OSS=(linux) diff --git a/cmd/setup-env/main.go b/cmd/setup-env/main.go index 35b73cf0c..10e15c482 100644 --- a/cmd/setup-env/main.go +++ b/cmd/setup-env/main.go @@ -31,7 +31,7 @@ func main() { env := os.Environ() env = append(env, fmt.Sprintf("ARCHS=%s", strings.Join(spec.Archs, " "))) env = append(env, fmt.Sprintf("OSS=%s", strings.Join(spec.OSs, " "))) - env = append(env, fmt.Sprintf("K8S_VERSIONS=%s", strings.Join(spec.K8sValidVersionsWithV, " "))) + env = append(env, fmt.Sprintf("K8S_VERSIONS=%s", strings.Join(spec.K8sVersionsWithV, " "))) env = append(env, fmt.Sprintf("DOCKER_VERSIONS=%s", strings.Join(spec.DockerVersions, " "))) env = append(env, fmt.Sprintf("CNI_PLUGINS_VERSIONS=%s", strings.Join(spec.CNIPluginsVersions, " "))) env = append(env, fmt.Sprintf("NVIDIA_DRIVER_VERSIONS=%s", strings.Join(spec.NvidiaDriverVersions, " "))) diff --git a/cmd/tke-installer/app/installer/images/images.go b/cmd/tke-installer/app/installer/images/images.go index 7c0b7a8b9..0ad6748f5 100644 --- a/cmd/tke-installer/app/installer/images/images.go +++ b/cmd/tke-installer/app/installer/images/images.go @@ -72,7 +72,7 @@ var components = Components{ Busybox: containerregistry.Image{Name: "busybox", Tag: "1.31.1"}, InfluxDB: containerregistry.Image{Name: "influxdb", Tag: "1.7.9"}, - ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.18.3-3"}, + ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.19.7-1"}, TKEAuthAPI: containerregistry.Image{Name: "tke-auth-api", Tag: Version}, TKEAuthController: containerregistry.Image{Name: "tke-auth-controller", Tag: Version}, diff --git a/cmd/tke-installer/app/installer/installer.go b/cmd/tke-installer/app/installer/installer.go index 409b45fc5..cba58c229 100644 --- a/cmd/tke-installer/app/installer/installer.go +++ b/cmd/tke-installer/app/installer/installer.go @@ -2461,7 +2461,7 @@ func (t *TKE) patchPlatformVersion(ctx context.Context) error { return nil } - versionsByte, err := json.Marshal(spec.K8sValidVersions) + versionsByte, err := json.Marshal(spec.K8sVersions) if err != nil { return err } diff --git a/cmd/tke-installer/app/installer/prepare.go b/cmd/tke-installer/app/installer/prepare.go index 63205d43e..036752f49 100644 --- a/cmd/tke-installer/app/installer/prepare.go +++ b/cmd/tke-installer/app/installer/prepare.go @@ -234,7 +234,7 @@ func (t *TKE) getLatestProviderResTag(ctx context.Context) (tag string, err erro if err != nil { return "", err } - if len(k8sValidVersions) > len(spec.K8sValidVersions) { + if len(k8sValidVersions) > len(spec.K8sVersions) { return k8sValidVersions[len(k8sValidVersions)-1], nil } return images.Get().ProviderRes.Tag, nil diff --git a/cmd/tke-installer/app/installer/upgrader.go b/cmd/tke-installer/app/installer/upgrader.go index 24f175f3a..b14e29d0a 100644 --- a/cmd/tke-installer/app/installer/upgrader.go +++ b/cmd/tke-installer/app/installer/upgrader.go @@ -155,7 +155,7 @@ func (t *TKE) updateTKEPlatformController(ctx context.Context) error { case result < 0: return errors.Errorf("can't upgrade, platform's version %s is higher than installer's version %s", tkeVersion, spec.TKEVersion) case result == 0: - if len(k8sValidVersions) == len(spec.K8sValidVersions) { + if len(k8sValidVersions) == len(spec.K8sVersions) { return errors.Errorf("can't upgrade, platform's version %s is equal to installer's version %s, please prepare your custom upgrade images before upgrade", tkeVersion, spec.TKEVersion) } depl.Spec.Template.Spec.InitContainers[0].Image = containerregistry.GetImagePrefix(images.Get().ProviderRes.Name + ":" + k8sValidVersions[len(k8sValidVersions)-1]) diff --git a/pkg/platform/provider/baremetal/images/images.go b/pkg/platform/provider/baremetal/images/images.go index 3b91cd735..66ec1b37c 100644 --- a/pkg/platform/provider/baremetal/images/images.go +++ b/pkg/platform/provider/baremetal/images/images.go @@ -54,7 +54,7 @@ var kubecomponetNames = []string{"kube-apiserver", "kube-controller-manager", "k var components = Components{ ETCD: containerregistry.Image{Name: "etcd", Tag: "v3.4.7"}, CoreDNS: containerregistry.Image{Name: "coredns", Tag: "1.6.7"}, - Pause: containerregistry.Image{Name: "pause", Tag: "3.1"}, + Pause: containerregistry.Image{Name: "pause", Tag: "3.2"}, NvidiaDevicePlugin: containerregistry.Image{Name: "nvidia-device-plugin", Tag: "1.0.0-beta4"}, Keepalived: containerregistry.Image{Name: "keepalived", Tag: "2.0.16-r0"}, diff --git a/pkg/platform/provider/baremetal/res/res.go b/pkg/platform/provider/baremetal/res/res.go index 9bdbb4291..2f7b6911a 100644 --- a/pkg/platform/provider/baremetal/res/res.go +++ b/pkg/platform/provider/baremetal/res/res.go @@ -46,7 +46,7 @@ var ( } KubernetesNode = Package{ Name: "kubernetes-node", - Versions: spec.K8sValidVersionsWithV, + Versions: spec.K8sVersionsWithV, } NvidiaDriver = Package{ Name: "NVIDIA", diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go index e4af118a2..1f95be6f6 100644 --- a/pkg/spec/spec.go +++ b/pkg/spec/spec.go @@ -31,15 +31,10 @@ var ( OSs = []string{"linux"} K8sVersionConstraint = ">= 1.10" - K8sVersions = []string{"1.18.3", "1.17.13", "1.16.9"} + K8sVersions = []string{"1.19.7", "1.18.3", "1.17.13"} K8sVersionsWithV = funk.Map(K8sVersions, func(s string) string { return "v" + s }).([]string) - // K8sValidVersions for backward compatibility. - K8sValidVersions = append(K8sVersions, []string{"1.16.6", "1.14.10"}...) - K8sValidVersionsWithV = funk.Map(K8sValidVersions, func(s string) string { - return "v" + s - }).([]string) DockerVersions = []string{"19.03.14"} CNIPluginsVersions = []string{"v0.8.6"} diff --git a/pkg/util/apiclient/client_test.go b/pkg/util/apiclient/client_test.go index 65b43bd3b..dffab6be7 100644 --- a/pkg/util/apiclient/client_test.go +++ b/pkg/util/apiclient/client_test.go @@ -39,7 +39,7 @@ func TestPullImageWithPod(t *testing.T) { Containers: []corev1.Container{ { Name: "test", - Image: "tkestack/pause:3.1", + Image: "tkestack/pause:3.2", }, }, }, From c3b1a3147c056c4f7011820901ecb8b8e5a2eb23 Mon Sep 17 00:00:00 2001 From: leonarliu Date: Thu, 28 Jan 2021 10:11:01 +0800 Subject: [PATCH 2/5] feat(platform): modify cni coredns config --- .../provider/baremetal/cluster/create.go | 11 +++++++++++ .../provider/baremetal/cluster/provider.go | 2 ++ .../baremetal/phases/galaxy/template.go | 1 + pkg/util/apiclient/idempotency.go | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go index ada004d1e..04b007a40 100644 --- a/pkg/platform/provider/baremetal/cluster/create.go +++ b/pkg/platform/provider/baremetal/cluster/create.go @@ -920,6 +920,17 @@ func (p *Provider) clusterMachineIPs(c *v1.Cluster) []string { return ips } +func (p *Provider) EnsureCorrectCoreDNSConfig(ctx context.Context, c *v1.Cluster) error { + if c.Status.Phase == platformv1.ClusterUpscaling { + return nil + } + clientset, err := c.ClientsetForBootstrap() + if err != nil { + return err + } + return apiclient.RemoveUnsupportedCoreDNSProperty(ctx, clientset) +} + func (p *Provider) EnsureJoinPhasePreflight(ctx context.Context, c *v1.Cluster) error { machines := map[bool][]platformv1.ClusterMachine{ true: c.Spec.ScalingMachines, diff --git a/pkg/platform/provider/baremetal/cluster/provider.go b/pkg/platform/provider/baremetal/cluster/provider.go index ef2534c95..69b5f49f7 100644 --- a/pkg/platform/provider/baremetal/cluster/provider.go +++ b/pkg/platform/provider/baremetal/cluster/provider.go @@ -110,6 +110,7 @@ func NewProvider() (*Provider, error) { p.EnsureKubeadmInitPhaseAddon, p.EnsureGalaxy, + p.EnsureCorrectCoreDNSConfig, p.EnsureJoinPhasePreflight, p.EnsureJoinPhaseControlPlanePrepare, @@ -144,6 +145,7 @@ func NewProvider() (*Provider, error) { UpgradeHandlers: []clusterprovider.Handler{ p.EnsurePreClusterUpgradeHook, p.EnsureUpgradeControlPlaneNode, + p.EnsureCorrectCoreDNSConfig, p.EnsurePostClusterUpgradeHook, }, ScaleDownHandlers: []clusterprovider.Handler{ diff --git a/pkg/platform/provider/baremetal/phases/galaxy/template.go b/pkg/platform/provider/baremetal/phases/galaxy/template.go index 2e2f3ae4e..eaa3ec6bf 100644 --- a/pkg/platform/provider/baremetal/phases/galaxy/template.go +++ b/pkg/platform/provider/baremetal/phases/galaxy/template.go @@ -142,6 +142,7 @@ metadata: data: 00-galaxy.conf: | { + "name": "galaxy-sdn", "type": "galaxy-sdn", "capabilities": {"portMappings": true}, "cniVersion": "0.2.0" diff --git a/pkg/util/apiclient/idempotency.go b/pkg/util/apiclient/idempotency.go index 58a74126d..228c2acd2 100644 --- a/pkg/util/apiclient/idempotency.go +++ b/pkg/util/apiclient/idempotency.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "math" "path/filepath" + "regexp" "strings" "time" @@ -676,3 +677,21 @@ func GetNodeIPV6Label(ip string) string { lableipv6Tail := fmt.Sprintf("%s=%s", LabelMachineIPV6Tail, strings.Replace(ip[splitLength:], ":", "a", -1)) return lableipv6Head + "," + lableipv6Tail } + +// RemoveUnsupportedCoreDNSProperty will remove unsupported property for old version coreDNS, +// more information please check https://github.com/coredns/coredns/issues/4362 +func RemoveUnsupportedCoreDNSProperty(ctx context.Context, client clientset.Interface) error { + cm, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "coredns", metav1.GetOptions{}) + if err != nil { + return err + } + re, err := regexp.Compile(`\s*max_concurrent\s*\w+`) + if err != nil { + return errors.Wrap(err, "unable to remove unsupported coreDNS property") + } + cm.Data["Corefile"] = re.ReplaceAllString(cm.Data["Corefile"], "") + if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(ctx, cm, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(err, "unable to remove unsupported coreDNS property") + } + return nil +} From 0f16ef8bcba1563dfaba02bce0447e1366ca6b1e Mon Sep 17 00:00:00 2001 From: leonarliu Date: Fri, 29 Jan 2021 16:42:09 +0800 Subject: [PATCH 3/5] feat(platform): add upgrade coredns process --- .github/workflows/e2e.yml | 21 ++++++- .../provider/baremetal/cluster/create.go | 11 ---- .../provider/baremetal/cluster/provider.go | 3 +- .../provider/baremetal/cluster/update.go | 57 +++++++++++++++++++ .../provider/baremetal/images/images.go | 2 +- pkg/util/apiclient/idempotency.go | 19 ------- 6 files changed, 78 insertions(+), 35 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f7b47e9b0..2bc48e686 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,12 +32,12 @@ jobs: run: | make push IMAGES="tke-platform-api tke-platform-controller" VERSION=${{ github.sha }} platform1: - name: platform test - 1.18.3 + name: platform test - 1.19.7 needs: image # if: contains(toJson(github.event.pull_request.labels.*.name), 'platform') runs-on: self-hosted env: - K8SVERSION: 1.18.3 + K8SVERSION: 1.19.7 steps: - name: printversion env: @@ -48,6 +48,23 @@ jobs: if: contains(env.COMMIT_MSG, 'installer') || contains(env.COMMIT_MSG, 'platform') || contains(env.COMMIT_MSG, 'cluster') || contains(env.COMMIT_MSG, 'gateway') || contains(env.COMMIT_MSG, 'addon') || contains(env.COMMIT_MSG, 'registry') run: | go test -timeout 40m -v tkestack.io/tke/test/e2e/platform + # platform1: + # name: platform test - 1.18.3 + # needs: image + # # if: contains(toJson(github.event.pull_request.labels.*.name), 'platform') + # runs-on: self-hosted + # env: + # K8SVERSION: 1.18.3 + # steps: + # - name: printversion + # env: + # LABELNAME_CONTEXT: ${{ toJson(github.event.pull_request.labels.*.name) }} + # run: | + # echo "$LABELNAME_CONTEXT" + # - name: e2e test + # if: contains(env.COMMIT_MSG, 'installer') || contains(env.COMMIT_MSG, 'platform') || contains(env.COMMIT_MSG, 'cluster') || contains(env.COMMIT_MSG, 'gateway') || contains(env.COMMIT_MSG, 'addon') || contains(env.COMMIT_MSG, 'registry') + # run: | + # go test -timeout 40m -v tkestack.io/tke/test/e2e/platform # platform2: # name: platform test - 1.16.9 # needs: image diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go index 04b007a40..ada004d1e 100644 --- a/pkg/platform/provider/baremetal/cluster/create.go +++ b/pkg/platform/provider/baremetal/cluster/create.go @@ -920,17 +920,6 @@ func (p *Provider) clusterMachineIPs(c *v1.Cluster) []string { return ips } -func (p *Provider) EnsureCorrectCoreDNSConfig(ctx context.Context, c *v1.Cluster) error { - if c.Status.Phase == platformv1.ClusterUpscaling { - return nil - } - clientset, err := c.ClientsetForBootstrap() - if err != nil { - return err - } - return apiclient.RemoveUnsupportedCoreDNSProperty(ctx, clientset) -} - func (p *Provider) EnsureJoinPhasePreflight(ctx context.Context, c *v1.Cluster) error { machines := map[bool][]platformv1.ClusterMachine{ true: c.Spec.ScalingMachines, diff --git a/pkg/platform/provider/baremetal/cluster/provider.go b/pkg/platform/provider/baremetal/cluster/provider.go index 69b5f49f7..b10e5781e 100644 --- a/pkg/platform/provider/baremetal/cluster/provider.go +++ b/pkg/platform/provider/baremetal/cluster/provider.go @@ -110,7 +110,6 @@ func NewProvider() (*Provider, error) { p.EnsureKubeadmInitPhaseAddon, p.EnsureGalaxy, - p.EnsureCorrectCoreDNSConfig, p.EnsureJoinPhasePreflight, p.EnsureJoinPhaseControlPlanePrepare, @@ -144,8 +143,8 @@ func NewProvider() (*Provider, error) { }, UpgradeHandlers: []clusterprovider.Handler{ p.EnsurePreClusterUpgradeHook, + p.EnsureUpgradeCoreDNS, p.EnsureUpgradeControlPlaneNode, - p.EnsureCorrectCoreDNSConfig, p.EnsurePostClusterUpgradeHook, }, ScaleDownHandlers: []clusterprovider.Handler{ diff --git a/pkg/platform/provider/baremetal/cluster/update.go b/pkg/platform/provider/baremetal/cluster/update.go index 40f6d25f0..c4b8422d0 100644 --- a/pkg/platform/provider/baremetal/cluster/update.go +++ b/pkg/platform/provider/baremetal/cluster/update.go @@ -20,6 +20,7 @@ package cluster import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -31,13 +32,18 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" certutil "k8s.io/client-go/util/cert" platformv1 "tkestack.io/tke/api/platform/v1" + kubeadmv1beta2 "tkestack.io/tke/pkg/platform/provider/baremetal/apis/kubeadm/v1beta2" "tkestack.io/tke/pkg/platform/provider/baremetal/constants" + "tkestack.io/tke/pkg/platform/provider/baremetal/images" "tkestack.io/tke/pkg/platform/provider/baremetal/phases/kubeadm" "tkestack.io/tke/pkg/platform/provider/baremetal/util" v1 "tkestack.io/tke/pkg/platform/types/v1" "tkestack.io/tke/pkg/util/log" + "tkestack.io/tke/pkg/util/version" ) func (p *Provider) EnsureRenewCerts(ctx context.Context, c *v1.Cluster) error { @@ -140,6 +146,57 @@ func (p *Provider) EnsurePreClusterUpgradeHook(ctx context.Context, c *v1.Cluste return util.ExcuteCustomizedHook(ctx, c, platformv1.HookPreClusterUpgrade, c.Spec.Machines[:1]) } +func (p *Provider) EnsureUpgradeCoreDNS(ctx context.Context, c *v1.Cluster) error { + versionRet, err := version.Parse(c.Spec.Version) + if err != nil { + return errors.Wrap(err, "unable to update coreDNS version") + } + if len(versionRet) < 2 { + return errors.Wrap(fmt.Errorf("invaild version format %s", c.Spec.Version), "unable to update coreDNS version") + } + if versionRet[0] == "1" && versionRet[1] == "19" { + + client, err := c.Clientset() + if err != nil { + return errors.Wrap(err, "unable to update coreDNS version") + } + err = updateCoreDNSVersion(ctx, client, images.Get().CoreDNS.Tag) + if err != nil { + return errors.Wrap(err, "unable to update coreDNS version") + } + } else { + logger := log.FromContext(ctx).WithName("Upgrade coreDNS") + logger.Infof("Target k8s version is %s, skip upgrade coreDNS", c.Spec.Version) + } + return nil +} + +func updateCoreDNSVersion(ctx context.Context, client kubernetes.Interface, version string) error { + cm, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "kubeadm-config", metav1.GetOptions{}) + if err != nil { + return err + } + clsConfigData, err := yaml.ToJSON([]byte(cm.Data["ClusterConfiguration"])) + if err != nil { + return err + } + clsConfig := kubeadmv1beta2.ClusterConfiguration{} + err = json.Unmarshal(clsConfigData, &clsConfig) + if err != nil { + return err + } + + clsConfig.DNS.ImageTag = version + + clsConfigData, err = kubeadm.MarshalToYAML(&clsConfig) + if err != nil { + return err + } + cm.Data["ClusterConfiguration"] = string(clsConfigData) + _, err = client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(ctx, cm, metav1.UpdateOptions{}) + return err +} + func (p *Provider) EnsureUpgradeControlPlaneNode(ctx context.Context, c *v1.Cluster) error { // check all machines are upgraded before upgrade cluster requirement, err := labels.NewRequirement(constants.LabelNodeNeedUpgrade, selection.Exists, []string{}) diff --git a/pkg/platform/provider/baremetal/images/images.go b/pkg/platform/provider/baremetal/images/images.go index 66ec1b37c..a70e51833 100644 --- a/pkg/platform/provider/baremetal/images/images.go +++ b/pkg/platform/provider/baremetal/images/images.go @@ -53,7 +53,7 @@ func (c Components) Get(name string) *containerregistry.Image { var kubecomponetNames = []string{"kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"} var components = Components{ ETCD: containerregistry.Image{Name: "etcd", Tag: "v3.4.7"}, - CoreDNS: containerregistry.Image{Name: "coredns", Tag: "1.6.7"}, + CoreDNS: containerregistry.Image{Name: "coredns", Tag: "1.7.0"}, Pause: containerregistry.Image{Name: "pause", Tag: "3.2"}, NvidiaDevicePlugin: containerregistry.Image{Name: "nvidia-device-plugin", Tag: "1.0.0-beta4"}, Keepalived: containerregistry.Image{Name: "keepalived", Tag: "2.0.16-r0"}, diff --git a/pkg/util/apiclient/idempotency.go b/pkg/util/apiclient/idempotency.go index 228c2acd2..58a74126d 100644 --- a/pkg/util/apiclient/idempotency.go +++ b/pkg/util/apiclient/idempotency.go @@ -25,7 +25,6 @@ import ( "io/ioutil" "math" "path/filepath" - "regexp" "strings" "time" @@ -677,21 +676,3 @@ func GetNodeIPV6Label(ip string) string { lableipv6Tail := fmt.Sprintf("%s=%s", LabelMachineIPV6Tail, strings.Replace(ip[splitLength:], ":", "a", -1)) return lableipv6Head + "," + lableipv6Tail } - -// RemoveUnsupportedCoreDNSProperty will remove unsupported property for old version coreDNS, -// more information please check https://github.com/coredns/coredns/issues/4362 -func RemoveUnsupportedCoreDNSProperty(ctx context.Context, client clientset.Interface) error { - cm, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "coredns", metav1.GetOptions{}) - if err != nil { - return err - } - re, err := regexp.Compile(`\s*max_concurrent\s*\w+`) - if err != nil { - return errors.Wrap(err, "unable to remove unsupported coreDNS property") - } - cm.Data["Corefile"] = re.ReplaceAllString(cm.Data["Corefile"], "") - if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(ctx, cm, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to remove unsupported coreDNS property") - } - return nil -} From c204af1bda1b5e26158b5e1bc0d8fb2b86f5912e Mon Sep 17 00:00:00 2001 From: leonarliu Date: Mon, 1 Feb 2021 14:16:46 +0800 Subject: [PATCH 4/5] feat(platform): change compare func --- pkg/platform/provider/baremetal/cluster/update.go | 14 +++++--------- .../provider/baremetal/constants/constants.go | 2 ++ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/platform/provider/baremetal/cluster/update.go b/pkg/platform/provider/baremetal/cluster/update.go index c4b8422d0..2105a17ce 100644 --- a/pkg/platform/provider/baremetal/cluster/update.go +++ b/pkg/platform/provider/baremetal/cluster/update.go @@ -147,15 +147,12 @@ func (p *Provider) EnsurePreClusterUpgradeHook(ctx context.Context, c *v1.Cluste } func (p *Provider) EnsureUpgradeCoreDNS(ctx context.Context, c *v1.Cluster) error { - versionRet, err := version.Parse(c.Spec.Version) - if err != nil { - return errors.Wrap(err, "unable to update coreDNS version") - } - if len(versionRet) < 2 { - return errors.Wrap(fmt.Errorf("invaild version format %s", c.Spec.Version), "unable to update coreDNS version") + logger := log.FromContext(ctx).WithName("Upgrade coreDNS") + if version.Compare(c.Status.Version, constants.NeedUpgradeCoreDNSK8sVersion) >= 0 { + logger.Infof("Current k8s version is %s, skip upgrade coreDNS", c.Spec.Version) + return nil } - if versionRet[0] == "1" && versionRet[1] == "19" { - + if version.Compare(c.Spec.Version, constants.NeedUpgradeCoreDNSK8sVersion) >= 0 { client, err := c.Clientset() if err != nil { return errors.Wrap(err, "unable to update coreDNS version") @@ -165,7 +162,6 @@ func (p *Provider) EnsureUpgradeCoreDNS(ctx context.Context, c *v1.Cluster) erro return errors.Wrap(err, "unable to update coreDNS version") } } else { - logger := log.FromContext(ctx).WithName("Upgrade coreDNS") logger.Infof("Target k8s version is %s, skip upgrade coreDNS", c.Spec.Version) } return nil diff --git a/pkg/platform/provider/baremetal/constants/constants.go b/pkg/platform/provider/baremetal/constants/constants.go index d10af3424..60277a301 100644 --- a/pkg/platform/provider/baremetal/constants/constants.go +++ b/pkg/platform/provider/baremetal/constants/constants.go @@ -133,4 +133,6 @@ const ( MinNumCPU = 2 APIServerHostName = "api.tke.com" + + NeedUpgradeCoreDNSK8sVersion = "1.19.0" ) From 3b73dcd2ab57d38e1fc43fb5cb960113f3fda980 Mon Sep 17 00:00:00 2001 From: leonarliu Date: Mon, 1 Feb 2021 16:44:57 +0800 Subject: [PATCH 5/5] feat(platform): official coredns version for 1.19 --- pkg/platform/provider/baremetal/cluster/kubeadm.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/platform/provider/baremetal/cluster/kubeadm.go b/pkg/platform/provider/baremetal/cluster/kubeadm.go index 94bae0b9d..fb6027ce4 100644 --- a/pkg/platform/provider/baremetal/cluster/kubeadm.go +++ b/pkg/platform/provider/baremetal/cluster/kubeadm.go @@ -35,6 +35,7 @@ import ( v1 "tkestack.io/tke/pkg/platform/types/v1" "tkestack.io/tke/pkg/util/apiclient" "tkestack.io/tke/pkg/util/json" + "tkestack.io/tke/pkg/util/version" ) func (p *Provider) getKubeadmInitConfig(c *v1.Cluster) *kubeadm.InitConfig { @@ -155,9 +156,6 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste }, DNS: kubeadmv1beta2.DNS{ Type: kubeadmv1beta2.CoreDNS, - ImageMeta: kubeadmv1beta2.ImageMeta{ - ImageTag: images.Get().CoreDNS.Tag, - }, }, ImageRepository: p.config.Registry.Prefix, ClusterName: c.Name, @@ -165,6 +163,11 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste "IPv6DualStack": c.Cluster.Spec.Features.IPv6DualStack}, } + // since k8s 1.19 will use offical coreDNS version + if version.Compare(c.Spec.Version, constants.NeedUpgradeCoreDNSK8sVersion) < 0 { + config.DNS.ImageTag = images.Get().CoreDNS.Tag + } + utilruntime.Must(json.Merge(&config.Etcd, &c.Spec.Etcd)) if config.Etcd.Local != nil { config.Etcd.Local.ImageTag = images.Get().ETCD.Tag