fix(cluster): allocate galaxy ipam service ip

Signed-off-by: Tengfei Wang <[email protected]>
k8ssphere · Jun 30, 2020 · a5bdb73 · a5bdb73
1 parent c3012eb
commit a5bdb73
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 11 deletions.
diff --git a/pkg/platform/controller/addon/ipam/ipam_controller.go b/pkg/platform/controller/addon/ipam/ipam_controller.go
@@ -42,6 +42,7 @@ import (
  v1 "tkestack.io/tke/api/platform/v1"
  controllerutil "tkestack.io/tke/pkg/controller"
  "tkestack.io/tke/pkg/platform/controller/addon/ipam/images"
+ "tkestack.io/tke/pkg/platform/provider/baremetal/constants"
  "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/log"
  "tkestack.io/tke/pkg/util/metrics"
@@ -398,7 +399,11 @@ func (c *Controller) installIPAM(ctx context.Context, ipam *v1.IPAM) error {
  return err
  }
  // Service IPAM
- if _, err := kubeClient.CoreV1().Services(metav1.NamespaceSystem).Create(ctx, serviceIPAM(), metav1.CreateOptions{}); err != nil {
+ clusterIP := ""
+ if cluster.Annotations != nil {
+ clusterIP = cluster.Annotations[constants.GalaxyIPAMIPIndexAnnotaion]
+ }
+ if _, err := kubeClient.CoreV1().Services(metav1.NamespaceSystem).Create(ctx, serviceIPAM(clusterIP), metav1.CreateOptions{}); err != nil {
  return err
  }
  log.Info("ipam installed")
@@ -640,7 +645,7 @@ func cmFloatingIP() *corev1.ConfigMap {
  }
 }
 
-func serviceIPAM() *corev1.Service {
+func serviceIPAM(clusterIP string) *corev1.Service {
  return &corev1.Service{
  TypeMeta: metav1.TypeMeta{
  Kind: "Service",
@@ -652,7 +657,8 @@ func serviceIPAM() *corev1.Service {
  Labels: map[string]string{"app": "galaxy-ipam"},
  },
  Spec: corev1.ServiceSpec{
- Selector: map[string]string{"app": "galaxy-ipam"},
+ ClusterIP: clusterIP,
+ Selector: map[string]string{"app": "galaxy-ipam"},
  Ports: []corev1.ServicePort{
  {Name: "scheduler-port", Port: 9040, TargetPort: intstr.FromInt(9040)},
  {Name: "api-port", Port: 9041, TargetPort: intstr.FromInt(9041)},

diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go
@@ -259,7 +259,7 @@ func (p *Provider) EnsureClusterComplete(ctx context.Context, cluster *v1.Cluste
  funcs := []func(cluster *v1.Cluster) error{
  completeNetworking,
  completeDNS,
- completeGPU,
+ completeServiceIP,
  completeAddresses,
  completeCredential,
  }
@@ -315,15 +315,20 @@ func completeDNS(cluster *v1.Cluster) error {
  return nil
 }
 
-func completeGPU(cluster *v1.Cluster) error {
- ip, err := GetIndexedIP(cluster.Status.ServiceCIDR, constants.GPUQuotaAdmissionIPIndex)
- if err != nil {
- return errors.Wrap(err, "get gpu quota admission IP error")
- }
+func completeServiceIP(cluster *v1.Cluster) error {
  if cluster.Annotations == nil {
  cluster.Annotations = make(map[string]string)
  }
- cluster.Annotations[constants.GPUQuotaAdmissionIPAnnotaion] = ip.String()
+ for index, name := range map[int]string{
+ constants.GPUQuotaAdmissionIPIndex: constants.GPUQuotaAdmissionIPAnnotaion,
+ constants.GalaxyIPAMIPIndex: constants.GalaxyIPAMIPIndexAnnotaion,
+ } {
+ ip, err := GetIndexedIP(cluster.Status.ServiceCIDR, index)
+ if err != nil {
+ return errors.Wrap(err, "get service IP error")
+ }
+ cluster.Annotations[name] = ip.String()
+ }
 
  return nil
 }
@@ -572,8 +577,13 @@ func (p *Provider) EnsurePrepareForControlplane(ctx context.Context, c *v1.Clust
  if GPUQuotaAdmissionHost == "" {
  GPUQuotaAdmissionHost = "gpu-quota-admission"
  }
+ GalaxyIPAMHost := c.Annotations[constants.GalaxyIPAMIPIndexAnnotaion]
+ if GalaxyIPAMHost == "" {
+ GalaxyIPAMHost = "galaxy-ipam"
+ }
  schedulerPolicyConfig, err := template.ParseString(schedulerPolicyConfig, map[string]interface{}{
  "GPUQuotaAdmissionHost": GPUQuotaAdmissionHost,
+ "GalaxyIPAMHost": GalaxyIPAMHost,
  })
  if err != nil {
  return errors.Wrap(err, "parse schedulerPolicyConfig error")

diff --git a/pkg/platform/provider/baremetal/cluster/manifests.go b/pkg/platform/provider/baremetal/cluster/manifests.go
@@ -52,7 +52,7 @@ const (
  }
  ],
  "nodeCacheCapable" : false,
- "urlPrefix" : "https://galaxy-ipam:9040/v1"
+ "urlPrefix" : "https://{{.GalaxyIPAMHost}}:9040/v1"
  }
  ],
  "kind" : "Policy"

diff --git a/pkg/platform/provider/baremetal/constants/constants.go b/pkg/platform/provider/baremetal/constants/constants.go
@@ -104,7 +104,9 @@ const (
  KUBERNETES = 1
  DNSIPIndex = 10
  GPUQuotaAdmissionIPIndex = 9
+ GalaxyIPAMIPIndex = 8
  GPUQuotaAdmissionIPAnnotaion = platformv1.GroupName + "/gpu-quota-admission-ip"
+ GalaxyIPAMIPIndexAnnotaion = platformv1.GroupName + "/galaxy-ipam-ip"
 
  // RenewCertsTimeThreshold control how long time left to renew certs
  RenewCertsTimeThreshold = 30 * 24 * time.Hour