diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f7b47e9b0..2bc48e686 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,12 +32,12 @@ jobs: run: | make push IMAGES="tke-platform-api tke-platform-controller" VERSION=${{ github.sha }} platform1: - name: platform test - 1.18.3 + name: platform test - 1.19.7 needs: image # if: contains(toJson(github.event.pull_request.labels.*.name), 'platform') runs-on: self-hosted env: - K8SVERSION: 1.18.3 + K8SVERSION: 1.19.7 steps: - name: printversion env: @@ -48,6 +48,23 @@ jobs: if: contains(env.COMMIT_MSG, 'installer') || contains(env.COMMIT_MSG, 'platform') || contains(env.COMMIT_MSG, 'cluster') || contains(env.COMMIT_MSG, 'gateway') || contains(env.COMMIT_MSG, 'addon') || contains(env.COMMIT_MSG, 'registry') run: | go test -timeout 40m -v tkestack.io/tke/test/e2e/platform + # platform1: + # name: platform test - 1.18.3 + # needs: image + # # if: contains(toJson(github.event.pull_request.labels.*.name), 'platform') + # runs-on: self-hosted + # env: + # K8SVERSION: 1.18.3 + # steps: + # - name: printversion + # env: + # LABELNAME_CONTEXT: ${{ toJson(github.event.pull_request.labels.*.name) }} + # run: | + # echo "$LABELNAME_CONTEXT" + # - name: e2e test + # if: contains(env.COMMIT_MSG, 'installer') || contains(env.COMMIT_MSG, 'platform') || contains(env.COMMIT_MSG, 'cluster') || contains(env.COMMIT_MSG, 'gateway') || contains(env.COMMIT_MSG, 'addon') || contains(env.COMMIT_MSG, 'registry') + # run: | + # go test -timeout 40m -v tkestack.io/tke/test/e2e/platform # platform2: # name: platform test - 1.16.9 # needs: image diff --git a/build/docker/tools/provider-res/Makefile b/build/docker/tools/provider-res/Makefile index f31b86e17..72cb099d9 100644 --- a/build/docker/tools/provider-res/Makefile +++ b/build/docker/tools/provider-res/Makefile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations under the License. IMAGE = provider-res -VERSION = v1.18.3-3 +VERSION = v1.19.7-1 REGISTRY_PREFIX ?= tkestack COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) diff --git a/build/docker/tools/tke-installer/release.sh b/build/docker/tools/tke-installer/release.sh index a400578cd..aec4d1f44 100755 --- a/build/docker/tools/tke-installer/release.sh +++ b/build/docker/tools/tke-installer/release.sh @@ -23,7 +23,7 @@ set -o pipefail REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack} BUILDER=${BUILDER:-default} VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')} -PROVIDER_RES_VERSION=v1.18.3-3 +PROVIDER_RES_VERSION=v1.19.7-1 K8S_VERSION=${PROVIDER_RES_VERSION%-*} DOCKER_VERSION=19.03.14 OSS=(linux) diff --git a/cmd/setup-env/main.go b/cmd/setup-env/main.go index 35b73cf0c..10e15c482 100644 --- a/cmd/setup-env/main.go +++ b/cmd/setup-env/main.go @@ -31,7 +31,7 @@ func main() { env := os.Environ() env = append(env, fmt.Sprintf("ARCHS=%s", strings.Join(spec.Archs, " "))) env = append(env, fmt.Sprintf("OSS=%s", strings.Join(spec.OSs, " "))) - env = append(env, fmt.Sprintf("K8S_VERSIONS=%s", strings.Join(spec.K8sValidVersionsWithV, " "))) + env = append(env, fmt.Sprintf("K8S_VERSIONS=%s", strings.Join(spec.K8sVersionsWithV, " "))) env = append(env, fmt.Sprintf("DOCKER_VERSIONS=%s", strings.Join(spec.DockerVersions, " "))) env = append(env, fmt.Sprintf("CNI_PLUGINS_VERSIONS=%s", strings.Join(spec.CNIPluginsVersions, " "))) env = append(env, fmt.Sprintf("NVIDIA_DRIVER_VERSIONS=%s", strings.Join(spec.NvidiaDriverVersions, " "))) diff --git a/cmd/tke-installer/app/installer/images/images.go b/cmd/tke-installer/app/installer/images/images.go index 7c0b7a8b9..0ad6748f5 100644 --- a/cmd/tke-installer/app/installer/images/images.go +++ b/cmd/tke-installer/app/installer/images/images.go @@ -72,7 +72,7 @@ var components = Components{ Busybox: containerregistry.Image{Name: "busybox", Tag: "1.31.1"}, InfluxDB: containerregistry.Image{Name: "influxdb", Tag: "1.7.9"}, - ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.18.3-3"}, + ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.19.7-1"}, TKEAuthAPI: containerregistry.Image{Name: "tke-auth-api", Tag: Version}, TKEAuthController: containerregistry.Image{Name: "tke-auth-controller", Tag: Version}, diff --git a/cmd/tke-installer/app/installer/installer.go b/cmd/tke-installer/app/installer/installer.go index 409b45fc5..cba58c229 100644 --- a/cmd/tke-installer/app/installer/installer.go +++ b/cmd/tke-installer/app/installer/installer.go @@ -2461,7 +2461,7 @@ func (t *TKE) patchPlatformVersion(ctx context.Context) error { return nil } - versionsByte, err := json.Marshal(spec.K8sValidVersions) + versionsByte, err := json.Marshal(spec.K8sVersions) if err != nil { return err } diff --git a/cmd/tke-installer/app/installer/prepare.go b/cmd/tke-installer/app/installer/prepare.go index 63205d43e..036752f49 100644 --- a/cmd/tke-installer/app/installer/prepare.go +++ b/cmd/tke-installer/app/installer/prepare.go @@ -234,7 +234,7 @@ func (t *TKE) getLatestProviderResTag(ctx context.Context) (tag string, err erro if err != nil { return "", err } - if len(k8sValidVersions) > len(spec.K8sValidVersions) { + if len(k8sValidVersions) > len(spec.K8sVersions) { return k8sValidVersions[len(k8sValidVersions)-1], nil } return images.Get().ProviderRes.Tag, nil diff --git a/cmd/tke-installer/app/installer/upgrader.go b/cmd/tke-installer/app/installer/upgrader.go index 24f175f3a..b14e29d0a 100644 --- a/cmd/tke-installer/app/installer/upgrader.go +++ b/cmd/tke-installer/app/installer/upgrader.go @@ -155,7 +155,7 @@ func (t *TKE) updateTKEPlatformController(ctx context.Context) error { case result < 0: return errors.Errorf("can't upgrade, platform's version %s is higher than installer's version %s", tkeVersion, spec.TKEVersion) case result == 0: - if len(k8sValidVersions) == len(spec.K8sValidVersions) { + if len(k8sValidVersions) == len(spec.K8sVersions) { return errors.Errorf("can't upgrade, platform's version %s is equal to installer's version %s, please prepare your custom upgrade images before upgrade", tkeVersion, spec.TKEVersion) } depl.Spec.Template.Spec.InitContainers[0].Image = containerregistry.GetImagePrefix(images.Get().ProviderRes.Name + ":" + k8sValidVersions[len(k8sValidVersions)-1]) diff --git a/pkg/platform/provider/baremetal/cluster/kubeadm.go b/pkg/platform/provider/baremetal/cluster/kubeadm.go index 94bae0b9d..fb6027ce4 100644 --- a/pkg/platform/provider/baremetal/cluster/kubeadm.go +++ b/pkg/platform/provider/baremetal/cluster/kubeadm.go @@ -35,6 +35,7 @@ import ( v1 "tkestack.io/tke/pkg/platform/types/v1" "tkestack.io/tke/pkg/util/apiclient" "tkestack.io/tke/pkg/util/json" + "tkestack.io/tke/pkg/util/version" ) func (p *Provider) getKubeadmInitConfig(c *v1.Cluster) *kubeadm.InitConfig { @@ -155,9 +156,6 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste }, DNS: kubeadmv1beta2.DNS{ Type: kubeadmv1beta2.CoreDNS, - ImageMeta: kubeadmv1beta2.ImageMeta{ - ImageTag: images.Get().CoreDNS.Tag, - }, }, ImageRepository: p.config.Registry.Prefix, ClusterName: c.Name, @@ -165,6 +163,11 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste "IPv6DualStack": c.Cluster.Spec.Features.IPv6DualStack}, } + // since k8s 1.19 will use offical coreDNS version + if version.Compare(c.Spec.Version, constants.NeedUpgradeCoreDNSK8sVersion) < 0 { + config.DNS.ImageTag = images.Get().CoreDNS.Tag + } + utilruntime.Must(json.Merge(&config.Etcd, &c.Spec.Etcd)) if config.Etcd.Local != nil { config.Etcd.Local.ImageTag = images.Get().ETCD.Tag diff --git a/pkg/platform/provider/baremetal/cluster/provider.go b/pkg/platform/provider/baremetal/cluster/provider.go index ef2534c95..b10e5781e 100644 --- a/pkg/platform/provider/baremetal/cluster/provider.go +++ b/pkg/platform/provider/baremetal/cluster/provider.go @@ -143,6 +143,7 @@ func NewProvider() (*Provider, error) { }, UpgradeHandlers: []clusterprovider.Handler{ p.EnsurePreClusterUpgradeHook, + p.EnsureUpgradeCoreDNS, p.EnsureUpgradeControlPlaneNode, p.EnsurePostClusterUpgradeHook, }, diff --git a/pkg/platform/provider/baremetal/cluster/update.go b/pkg/platform/provider/baremetal/cluster/update.go index 40f6d25f0..2105a17ce 100644 --- a/pkg/platform/provider/baremetal/cluster/update.go +++ b/pkg/platform/provider/baremetal/cluster/update.go @@ -20,6 +20,7 @@ package cluster import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -31,13 +32,18 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" certutil "k8s.io/client-go/util/cert" platformv1 "tkestack.io/tke/api/platform/v1" + kubeadmv1beta2 "tkestack.io/tke/pkg/platform/provider/baremetal/apis/kubeadm/v1beta2" "tkestack.io/tke/pkg/platform/provider/baremetal/constants" + "tkestack.io/tke/pkg/platform/provider/baremetal/images" "tkestack.io/tke/pkg/platform/provider/baremetal/phases/kubeadm" "tkestack.io/tke/pkg/platform/provider/baremetal/util" v1 "tkestack.io/tke/pkg/platform/types/v1" "tkestack.io/tke/pkg/util/log" + "tkestack.io/tke/pkg/util/version" ) func (p *Provider) EnsureRenewCerts(ctx context.Context, c *v1.Cluster) error { @@ -140,6 +146,53 @@ func (p *Provider) EnsurePreClusterUpgradeHook(ctx context.Context, c *v1.Cluste return util.ExcuteCustomizedHook(ctx, c, platformv1.HookPreClusterUpgrade, c.Spec.Machines[:1]) } +func (p *Provider) EnsureUpgradeCoreDNS(ctx context.Context, c *v1.Cluster) error { + logger := log.FromContext(ctx).WithName("Upgrade coreDNS") + if version.Compare(c.Status.Version, constants.NeedUpgradeCoreDNSK8sVersion) >= 0 { + logger.Infof("Current k8s version is %s, skip upgrade coreDNS", c.Spec.Version) + return nil + } + if version.Compare(c.Spec.Version, constants.NeedUpgradeCoreDNSK8sVersion) >= 0 { + client, err := c.Clientset() + if err != nil { + return errors.Wrap(err, "unable to update coreDNS version") + } + err = updateCoreDNSVersion(ctx, client, images.Get().CoreDNS.Tag) + if err != nil { + return errors.Wrap(err, "unable to update coreDNS version") + } + } else { + logger.Infof("Target k8s version is %s, skip upgrade coreDNS", c.Spec.Version) + } + return nil +} + +func updateCoreDNSVersion(ctx context.Context, client kubernetes.Interface, version string) error { + cm, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "kubeadm-config", metav1.GetOptions{}) + if err != nil { + return err + } + clsConfigData, err := yaml.ToJSON([]byte(cm.Data["ClusterConfiguration"])) + if err != nil { + return err + } + clsConfig := kubeadmv1beta2.ClusterConfiguration{} + err = json.Unmarshal(clsConfigData, &clsConfig) + if err != nil { + return err + } + + clsConfig.DNS.ImageTag = version + + clsConfigData, err = kubeadm.MarshalToYAML(&clsConfig) + if err != nil { + return err + } + cm.Data["ClusterConfiguration"] = string(clsConfigData) + _, err = client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(ctx, cm, metav1.UpdateOptions{}) + return err +} + func (p *Provider) EnsureUpgradeControlPlaneNode(ctx context.Context, c *v1.Cluster) error { // check all machines are upgraded before upgrade cluster requirement, err := labels.NewRequirement(constants.LabelNodeNeedUpgrade, selection.Exists, []string{}) diff --git a/pkg/platform/provider/baremetal/constants/constants.go b/pkg/platform/provider/baremetal/constants/constants.go index d10af3424..60277a301 100644 --- a/pkg/platform/provider/baremetal/constants/constants.go +++ b/pkg/platform/provider/baremetal/constants/constants.go @@ -133,4 +133,6 @@ const ( MinNumCPU = 2 APIServerHostName = "api.tke.com" + + NeedUpgradeCoreDNSK8sVersion = "1.19.0" ) diff --git a/pkg/platform/provider/baremetal/images/images.go b/pkg/platform/provider/baremetal/images/images.go index 3b91cd735..a70e51833 100644 --- a/pkg/platform/provider/baremetal/images/images.go +++ b/pkg/platform/provider/baremetal/images/images.go @@ -53,8 +53,8 @@ func (c Components) Get(name string) *containerregistry.Image { var kubecomponetNames = []string{"kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"} var components = Components{ ETCD: containerregistry.Image{Name: "etcd", Tag: "v3.4.7"}, - CoreDNS: containerregistry.Image{Name: "coredns", Tag: "1.6.7"}, - Pause: containerregistry.Image{Name: "pause", Tag: "3.1"}, + CoreDNS: containerregistry.Image{Name: "coredns", Tag: "1.7.0"}, + Pause: containerregistry.Image{Name: "pause", Tag: "3.2"}, NvidiaDevicePlugin: containerregistry.Image{Name: "nvidia-device-plugin", Tag: "1.0.0-beta4"}, Keepalived: containerregistry.Image{Name: "keepalived", Tag: "2.0.16-r0"}, diff --git a/pkg/platform/provider/baremetal/phases/galaxy/template.go b/pkg/platform/provider/baremetal/phases/galaxy/template.go index 2e2f3ae4e..eaa3ec6bf 100644 --- a/pkg/platform/provider/baremetal/phases/galaxy/template.go +++ b/pkg/platform/provider/baremetal/phases/galaxy/template.go @@ -142,6 +142,7 @@ metadata: data: 00-galaxy.conf: | { + "name": "galaxy-sdn", "type": "galaxy-sdn", "capabilities": {"portMappings": true}, "cniVersion": "0.2.0" diff --git a/pkg/platform/provider/baremetal/res/res.go b/pkg/platform/provider/baremetal/res/res.go index 9bdbb4291..2f7b6911a 100644 --- a/pkg/platform/provider/baremetal/res/res.go +++ b/pkg/platform/provider/baremetal/res/res.go @@ -46,7 +46,7 @@ var ( } KubernetesNode = Package{ Name: "kubernetes-node", - Versions: spec.K8sValidVersionsWithV, + Versions: spec.K8sVersionsWithV, } NvidiaDriver = Package{ Name: "NVIDIA", diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go index e4af118a2..1f95be6f6 100644 --- a/pkg/spec/spec.go +++ b/pkg/spec/spec.go @@ -31,15 +31,10 @@ var ( OSs = []string{"linux"} K8sVersionConstraint = ">= 1.10" - K8sVersions = []string{"1.18.3", "1.17.13", "1.16.9"} + K8sVersions = []string{"1.19.7", "1.18.3", "1.17.13"} K8sVersionsWithV = funk.Map(K8sVersions, func(s string) string { return "v" + s }).([]string) - // K8sValidVersions for backward compatibility. - K8sValidVersions = append(K8sVersions, []string{"1.16.6", "1.14.10"}...) - K8sValidVersionsWithV = funk.Map(K8sValidVersions, func(s string) string { - return "v" + s - }).([]string) DockerVersions = []string{"19.03.14"} CNIPluginsVersions = []string{"v0.8.6"} diff --git a/pkg/util/apiclient/client_test.go b/pkg/util/apiclient/client_test.go index 65b43bd3b..dffab6be7 100644 --- a/pkg/util/apiclient/client_test.go +++ b/pkg/util/apiclient/client_test.go @@ -39,7 +39,7 @@ func TestPullImageWithPod(t *testing.T) { Containers: []corev1.Container{ { Name: "test", - Image: "tkestack/pause:3.1", + Image: "tkestack/pause:3.2", }, }, },