feat(platform): allow cluster to set hostname as nodename

tkestack · Sep 11, 2020 · 5dcd1c2 · 5dcd1c2
1 parent d7b2b86
commit 5dcd1c2
Show file tree

Hide file tree

Showing 25 changed files with 572 additions and 425 deletions.
diff --git a/api/openapi/zz_generated.openapi.go b/api/openapi/zz_generated.openapi.go
diff --git a/api/platform/types.go b/api/platform/types.go
@@ -122,6 +122,9 @@ type ClusterSpec struct {
  // Upgrade control upgrade process.
  // +optional
  Upgrade Upgrade
+ // If true will use hostname as nodename, if false will use machine IP as nodename.
+ // +optional
+ HostnameAsNodename bool
 }
 
 // ClusterStatus represents information about the status of a cluster.

diff --git a/api/platform/v1/generated.pb.go b/api/platform/v1/generated.pb.go
diff --git a/api/platform/v1/generated.proto b/api/platform/v1/generated.proto
diff --git a/api/platform/v1/types.go b/api/platform/v1/types.go
@@ -127,6 +127,9 @@ type ClusterSpec struct {
  // Upgrade control upgrade process.
  // +optional
  Upgrade Upgrade `json:"upgrade,omitempty" protobuf:"bytes,22,opt,name=upgrade"`
+ // If true will use hostname as nodename, if false will use machine IP as nodename.
+ // +optional
+ HostnameAsNodename bool `json:"hostnameAsNodename,omitempty" protobuf:"bytes,23,opt,name=hostnameAsNodename"`
 }
 
 // ClusterStatus represents information about the status of a cluster.

diff --git a/api/platform/v1/types_swagger_doc_generated.go b/api/platform/v1/types_swagger_doc_generated.go
diff --git a/api/platform/v1/zz_generated.conversion.go b/api/platform/v1/zz_generated.conversion.go
diff --git a/cmd/tke-installer/app/installer/installer.go b/cmd/tke-installer/app/installer/installer.go
@@ -1779,8 +1779,9 @@ func (t *TKE) installTKEBusinessController(ctx context.Context) error {
 func (t *TKE) installInfluxDB(ctx context.Context) error {
  err := apiclient.CreateResourceWithDir(ctx, t.globalClient, "manifests/influxdb/*.yaml",
  map[string]interface{}{
- "Image": images.Get().InfluxDB.FullName(),
- "NodeName": t.servers[0],
+ "Image": images.Get().InfluxDB.FullName(),
+ "NodeSelectorKey": apiclient.LabelMachineIP,
+ "NodeSelectorValue": t.servers[0],
  })
  if err != nil {
  return err
@@ -1947,15 +1948,16 @@ func (t *TKE) installTKENotifyController(ctx context.Context) error {
 
 func (t *TKE) installTKERegistryAPI(ctx context.Context) error {
  options := map[string]interface{}{
- "Replicas": t.Config.Replicas,
- "Image": images.Get().TKERegistryAPI.FullName(),
- "NodeName": t.servers[0],
- "AdminUsername": t.Para.Config.Registry.TKERegistry.Username,
- "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password),
- "EnableAuth": t.Para.Config.Auth.TKEAuth != nil,
- "EnableBusiness": t.businessEnabled(),
- "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain,
- "EnableAudit": t.auditEnabled(),
+ "Replicas": t.Config.Replicas,
+ "Image": images.Get().TKERegistryAPI.FullName(),
+ "NodeSelectorKey": apiclient.LabelMachineIP,
+ "NodeSelectorValue": t.servers[0],
+ "AdminUsername": t.Para.Config.Registry.TKERegistry.Username,
+ "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password),
+ "EnableAuth": t.Para.Config.Auth.TKEAuth != nil,
+ "EnableBusiness": t.businessEnabled(),
+ "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain,
+ "EnableAudit": t.auditEnabled(),
  }
  if t.Para.Config.Auth.OIDCAuth != nil {
  options["OIDCClientID"] = t.Para.Config.Auth.OIDCAuth.ClientID
@@ -1979,14 +1981,15 @@ func (t *TKE) installTKERegistryAPI(ctx context.Context) error {
 func (t *TKE) installTKERegistryController(ctx context.Context) error {
  err := apiclient.CreateResourceWithDir(ctx, t.globalClient, "manifests/tke-registry-controller/*.yaml",
  map[string]interface{}{
- "Replicas": t.Config.Replicas,
- "Image": images.Get().TKERegistryController.FullName(),
- "NodeName": t.servers[0],
- "AdminUsername": t.Para.Config.Registry.TKERegistry.Username,
- "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password),
- "EnableAuth": t.Para.Config.Auth.TKEAuth != nil,
- "EnableBusiness": t.businessEnabled(),
- "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain,
+ "Replicas": t.Config.Replicas,
+ "Image": images.Get().TKERegistryController.FullName(),
+ "NodeSelectorKey": apiclient.LabelMachineIP,
+ "NodeSelectorValue": t.servers[0],
+ "AdminUsername": t.Para.Config.Registry.TKERegistry.Username,
+ "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password),
+ "EnableAuth": t.Para.Config.Auth.TKEAuth != nil,
+ "EnableBusiness": t.businessEnabled(),
+ "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain,
  })
  if err != nil {
  return err

diff --git a/cmd/tke-installer/app/installer/manifests/influxdb/influxdb.yaml b/cmd/tke-installer/app/installer/manifests/influxdb/influxdb.yaml
@@ -33,7 +33,8 @@ spec:
  - name: data
  mountPath: /var/lib/influxdb
  subPath:
- nodeName: {{ .NodeName }}
+ nodeSelector:
+ {{ .NodeSelectorKey }}: {{ .NodeSelectorValue }}
  hostNetwork: true
  volumes:
  - name: data

diff --git a/cmd/tke-installer/app/installer/manifests/influxdb/manifest_test.go b/cmd/tke-installer/app/installer/manifests/influxdb/manifest_test.go
@@ -29,8 +29,9 @@ import (
 func TestManifest(t *testing.T) {
  data, err := template.ParseFile("influxdb.yaml",
  map[string]interface{}{
- "Image": "Image",
- "NodeName": "NodeName",
+ "Image": "Image",
+ "NodeSelectorKey": "LabelMachineIP",
+ "NodeSelectorValue": "MachineIP",
  })
  if !assert.Nil(t, err) {
  t.FailNow()

diff --git a/cmd/tke-installer/app/installer/manifests/tke-registry-api/manifest_test.go b/cmd/tke-installer/app/installer/manifests/tke-registry-api/manifest_test.go
@@ -29,14 +29,15 @@ import (
 func TestManifest(t *testing.T) {
  data, err := template.ParseFile("tke-registry-api.yaml",
  map[string]interface{}{
- "Replicas": 1,
- "Image": "Image",
- "NodeName": "NodeName",
- "AdminUsername": "AdminUsername",
- "AdminPassword": "AdminPassword",
- "EnableAuth": true,
- "EnableBusiness": true,
- "DomainSuffix": "DomainSuffix",
+ "Replicas": 1,
+ "Image": "Image",
+ "NodeSelectorKey": "LabelMachineIP",
+ "NodeSelectorValue": "MachineIP",
+ "AdminUsername": "AdminUsername",
+ "AdminPassword": "AdminPassword",
+ "EnableAuth": true,
+ "EnableBusiness": true,
+ "DomainSuffix": "DomainSuffix",
  })
  if !assert.Nil(t, err) {
  t.FailNow()

diff --git a/cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml b/cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml
@@ -73,7 +73,8 @@ spec:
  requests:
  cpu: 50m
  memory: 128Mi
- nodeName: {{ .NodeName }}
+ nodeSelector:
+ {{ .NodeSelectorKey }}: {{ .NodeSelectorValue }}
  volumes:
  - name: certs-volume
  configMap:

diff --git a/cmd/tke-installer/app/installer/manifests/tke-registry-controller/manifest_test.go b/cmd/tke-installer/app/installer/manifests/tke-registry-controller/manifest_test.go
@@ -29,14 +29,15 @@ import (
 func TestManifest(t *testing.T) {
  data, err := template.ParseFile("tke-registry-controller.yaml",
  map[string]interface{}{
- "Replicas": 1,
- "Image": "Image",
- "NodeName": "NodeName",
- "AdminUsername": "AdminUsername",
- "AdminPassword": "AdminPassword",
- "EnableBusiness": true,
- "EnableAuth": true,
- "DomainSuffix": "DomainSuffix",
+ "Replicas": 1,
+ "Image": "Image",
+ "NodeSelectorKey": "LabelMachineIP",
+ "NodeSelectorValue": "MachineIP",
+ "AdminUsername": "AdminUsername",
+ "AdminPassword": "AdminPassword",
+ "EnableBusiness": true,
+ "EnableAuth": true,
+ "DomainSuffix": "DomainSuffix",
  })
  if !assert.Nil(t, err) {
  t.FailNow()

diff --git a/...ke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml b/...ke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml
@@ -44,7 +44,8 @@ spec:
  requests:
  cpu: 50m
  memory: 128Mi
- nodeName: {{ .NodeName }}
+ nodeSelector:
+ {{ .NodeSelectorKey }}: {{ .NodeSelectorValue }}
  volumes:
  - name: certs-volume
  configMap:

diff --git a/cmd/tke-upgrade/app/options/options.go b/cmd/tke-upgrade/app/options/options.go
@@ -4,12 +4,14 @@ import (
  "encoding/json"
  "fmt"
  "io/ioutil"
+
  platformv1 "tkestack.io/tke/api/platform/v1"
  v1 "tkestack.io/tke/api/platform/v1"
  "tkestack.io/tke/cmd/tke-installer/app/config"
  "tkestack.io/tke/cmd/tke-installer/app/installer/constants"
  "tkestack.io/tke/cmd/tke-installer/app/installer/images"
  "tkestack.io/tke/cmd/tke-installer/app/installer/types"
+ "tkestack.io/tke/pkg/util/apiclient"
  "tkestack.io/tke/pkg/util/containerregistry"
 )
 
@@ -214,8 +216,9 @@ func (t *TKE) TKEBusinessController() (option Options) {
 
 func (t *TKE) InfluxDB() (option Options) {
  option = map[string]interface{}{
- "Image": images.Get().InfluxDB.FullName(),
- "NodeName": t.Servers[0],
+ "Image": images.Get().InfluxDB.FullName(),
+ "NodeSelectorKey": apiclient.LabelMachineIP,
+ "NodeSelectorValue": t.Servers[0],
  }
  return
 }
@@ -306,12 +309,13 @@ func (t *TKE) TKENotifyController() (option Options) {
 
 func (t *TKE) TKERegistryAPI() (option Options) {
  option = map[string]interface{}{
- "Image": t.GetFullName(images.Get().TKERegistryAPI.Name),
- "NodeName": t.Servers[0],
- "AdminUsername": t.Para.Config.Registry.TKERegistry.Username,
- "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password),
- "EnableAuth": t.Para.Config.Auth.TKEAuth != nil,
- "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain,
+ "Image": t.GetFullName(images.Get().TKERegistryAPI.Name),
+ "NodeSelectorKey": apiclient.LabelMachineIP,
+ "NodeSelectorValue": t.Servers[0],
+ "AdminUsername": t.Para.Config.Registry.TKERegistry.Username,
+ "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password),
+ "EnableAuth": t.Para.Config.Auth.TKEAuth != nil,
+ "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain,
  }
  if t.Para.Config.Auth.OIDCAuth != nil {
  option["OIDCClientID"] = t.Para.Config.Auth.OIDCAuth.ClientID

diff --git a/pkg/platform/controller/machine/deletion/machine_deleter.go b/pkg/platform/controller/machine/deletion/machine_deleter.go
@@ -31,6 +31,7 @@ import (
  v1 "tkestack.io/tke/api/platform/v1"
  machineprovider "tkestack.io/tke/pkg/platform/provider/machine"
  typesv1 "tkestack.io/tke/pkg/platform/types/v1"
+ "tkestack.io/tke/pkg/util/apiclient"
  "tkestack.io/tke/pkg/util/log"
 )
 
@@ -302,7 +303,16 @@ func deleteNode(ctx context.Context, deleter *machineDeleter, machine *v1.Machin
  return err
  }
 
- err = clientset.CoreV1().Nodes().Delete(context.Background(), machine.Spec.IP, metav1.DeleteOptions{})
+ node, err := apiclient.GetNodeByMachineIP(ctx, clientset, machine.Spec.IP)
+ if err != nil {
+ if !errors.IsNotFound(err) {
+ return err
+ }
+ log.FromContext(ctx).Info("deleteNode done")
+ return nil
+ }
+
+ err = clientset.CoreV1().Nodes().Delete(context.Background(), node.Name, metav1.DeleteOptions{})
  if err != nil {
  if !errors.IsNotFound(err) {
  return err

diff --git a/pkg/platform/controller/machine/machine_controller.go b/pkg/platform/controller/machine/machine_controller.go
@@ -39,6 +39,7 @@ import (
  machineprovider "tkestack.io/tke/pkg/platform/provider/machine"
  typesv1 "tkestack.io/tke/pkg/platform/types/v1"
  "tkestack.io/tke/pkg/platform/util"
+ "tkestack.io/tke/pkg/util/apiclient"
  "tkestack.io/tke/pkg/util/log"
  "tkestack.io/tke/pkg/util/metrics"
 )
@@ -305,7 +306,7 @@ func (c *Controller) checkHealth(ctx context.Context, machine *platformv1.Machin
  healthCheckCondition.Reason = failedHealthCheckReason
  healthCheckCondition.Message = err.Error()
  } else {
- _, err = clientset.CoreV1().Nodes().Get(ctx, machine.Spec.IP, metav1.GetOptions{})
+ _, err = apiclient.GetNodeByMachineIP(ctx, clientset, machine.Spec.IP)
  if err != nil {
  machine.Status.Phase = platformv1.MachineFailed
 

diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go
@@ -697,8 +697,11 @@ func (p *Provider) EnsureKubeadmInitPhaseKubeletStart(ctx context.Context, c *v1
  if err != nil {
  return err
  }
- return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c),
- fmt.Sprintf("kubelet-start --node-name=%s", c.Spec.Machines[0].IP))
+ phase := "kubelet-start"
+ if !c.Spec.HostnameAsNodename {
+ phase += fmt.Sprintf(" --node-name=%s", c.Spec.Machines[0].IP)
+ }
+ return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), phase)
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseCerts(ctx context.Context, c *v1.Cluster) error {
@@ -1000,7 +1003,11 @@ func (p *Provider) EnsureMarkControlPlane(ctx context.Context, c *v1.Cluster) er
  machine.Taints = append(machine.Taints, taint)
  }
  }
- err := apiclient.MarkNode(ctx, clientset, machine.IP, machine.Labels, machine.Taints)
+ node, err := apiclient.GetNodeByMachineIP(ctx, clientset, machine.IP)
+ if err != nil {
+ return errors.Wrap(err, machine.IP)
+ }
+ err = apiclient.MarkNode(ctx, clientset, node.Name, machine.Labels, machine.Taints)
  if err != nil {
  return errors.Wrap(err, machine.IP)
  }