feat(platform): support custom credential getter (tkestack#1401)

* feat(platform): support custom credential getter * feat(platform): add credential provider * feat(platform): remove util credential methods * feat(platform): add username for interface
eshao731 · Jul 12, 2021 · 99c0633 · 99c0633
1 parent aa6098d
commit 99c0633
Show file tree

Hide file tree

Showing 30 changed files with 292 additions and 182 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -59,4 +59,4 @@ jobs:
  echo "$LABELNAME_CONTEXT"
  - name: e2e test
  run: |
- mkdir -p provider/baremetal/ && cp -r pkg/platform/provider/baremetal/conf provider/baremetal/ && ginkgo -nodes=3 -v test/e2e/platform
+ mkdir -p test/e2e/platform/provider/baremetal/ && cp -r pkg/platform/provider/baremetal/conf test/e2e/platform/provider/baremetal/ && ginkgo -nodes=3 -v test/e2e/platform
diff --git a/cmd/tke-installer/app/installer/upgrader.go b/cmd/tke-installer/app/installer/upgrader.go
@@ -41,7 +41,7 @@ import (
  "tkestack.io/tke/cmd/tke-installer/app/installer/types"
  cronhpaimage "tkestack.io/tke/pkg/platform/controller/addon/cronhpa/images"
  tappimage "tkestack.io/tke/pkg/platform/controller/addon/tappcontroller/images"
- typesv1 "tkestack.io/tke/pkg/platform/types/v1"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  "tkestack.io/tke/pkg/platform/util"
  configv1 "tkestack.io/tke/pkg/registry/apis/config/v1"
  "tkestack.io/tke/pkg/spec"
@@ -316,7 +316,7 @@ func (t *TKE) prepareForUpgrade(ctx context.Context) error {
  if err != nil {
  return err
  }
- t.Cluster, err = typesv1.GetClusterByName(ctx, t.platformClient, "global")
+ t.Cluster, err = clusterprovider.GetV1ClusterByName(ctx, t.platformClient, "global", clusterprovider.AdminUsername)
  if err != nil {
  return err
  }

diff --git a/pkg/logagent/util/cluster.go b/pkg/logagent/util/cluster.go
@@ -36,6 +36,8 @@ import (
  platformversionedclient "tkestack.io/tke/api/client/clientset/versioned/typed/platform/v1"
  platformv1 "tkestack.io/tke/api/platform/v1"
  v1platform "tkestack.io/tke/api/platform/v1"
+ "tkestack.io/tke/pkg/apiserver/authentication"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/log"
 )
@@ -81,7 +83,12 @@ func APIServerLocationByCluster(ctx context.Context, clusterName string, platfor
  if cluster.Status.Phase != v1platform.ClusterRunning {
  return nil, nil, "", errors.NewServiceUnavailable(fmt.Sprintf("cluster %s status is abnormal", cluster.ObjectMeta.Name))
  }
- credential, err := util.GetClusterCredentialV1(ctx, platformClient, cluster)
+ provider, err := clusterprovider.GetProvider(cluster.Spec.Type)
+ if err != nil {
+ return nil, nil, "", err
+ }
+ username, _ := authentication.UsernameAndTenantID(ctx)
+ credential, err := provider.GetClusterCredentialV1(ctx, platformClient, cluster, username)
  if err != nil {
  log.Errorf("unable to get credential %v", err)
  return nil, nil, "", err

diff --git a/pkg/monitor/controller/prometheus/controller.go b/pkg/monitor/controller/prometheus/controller.go
@@ -60,6 +60,7 @@ import (
  "tkestack.io/tke/pkg/monitor/controller/prometheus/images"
  esutil "tkestack.io/tke/pkg/monitor/storage/es/client"
  monitorutil "tkestack.io/tke/pkg/monitor/util"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  platformutil "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/apiclient"
  containerregistryutil "tkestack.io/tke/pkg/util/containerregistry"
@@ -736,8 +737,12 @@ func (c *Controller) installPrometheus(ctx context.Context, prometheus *v1.Prome
  prometheus.Status.SubVersion[AlertManagerService] = components.AlertManagerService.Tag
 
  log.Infof("Start to create prometheus")
+ provider, err := clusterprovider.GetProvider(cluster.Spec.Type)
+ if err != nil {
+ return fmt.Errorf("get provider failed: %v", err)
+ }
  // Secret for prometheus-etcd
- credential, err := platformutil.GetClusterCredentialV1(ctx, c.platformClient, cluster)
+ credential, err := provider.GetClusterCredentialV1(ctx, c.platformClient, cluster, clusterprovider.AdminUsername)
  if err != nil {
  return fmt.Errorf("get credential failed: %v", err)
  }

diff --git a/pkg/monitor/util/cache/cache.go b/pkg/monitor/util/cache/cache.go
@@ -31,6 +31,7 @@ import (
  "tkestack.io/tke/api/monitor"
  platformv1 "tkestack.io/tke/api/platform/v1"
  "tkestack.io/tke/pkg/monitor/util"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  platformutil "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/log"
 
@@ -280,7 +281,11 @@ func (c *cacher) getClusters(ctx context.Context) {
 }
 
 func (c *cacher) getMetricServerClientSet(ctx context.Context, cls *platformv1.Cluster) (*metricsv.Clientset, error) {
- cc, err := platformutil.GetClusterCredentialV1(ctx, c.platformClient, cls)
+ provider, err := clusterprovider.GetProvider(cls.Spec.Type)
+ if err != nil {
+ return nil, err
+ }
+ cc, err := provider.GetClusterCredentialV1(ctx, c.platformClient, cls, clusterprovider.AdminUsername)
  if err != nil {
  log.Error("query cluster credential failed", log.Any("cluster", cls.GetName()), log.Err(err))
  return nil, err

diff --git a/pkg/platform/controller/addon/lbcf/controller.go b/pkg/platform/controller/addon/lbcf/controller.go
@@ -52,6 +52,7 @@ import (
  clientset "tkestack.io/tke/api/client/clientset/versioned"
  platformv1informer "tkestack.io/tke/api/client/informers/externalversions/platform/v1"
  platformv1lister "tkestack.io/tke/api/client/listers/platform/v1"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  "tkestack.io/tke/pkg/util/log"
 )
 
@@ -453,7 +454,11 @@ func (c *Controller) uninstallLBCF(ctx context.Context, lbcf *v1.LBCF) error {
  if err != nil {
  return err
  }
- credential, err := util.GetClusterCredentialV1(ctx, c.client.PlatformV1(), cluster)
+ provider, err := clusterprovider.GetProvider(cluster.Spec.Type)
+ if err != nil {
+ return err
+ }
+ credential, err := provider.GetClusterCredentialV1(ctx, c.client.PlatformV1(), cluster, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }

diff --git a/pkg/platform/controller/addon/prometheus/controller.go b/pkg/platform/controller/addon/prometheus/controller.go
@@ -58,6 +58,7 @@ import (
  esutil "tkestack.io/tke/pkg/monitor/storage/es/client"
  monitorutil "tkestack.io/tke/pkg/monitor/util"
  "tkestack.io/tke/pkg/platform/controller/addon/prometheus/images"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/apiclient"
  containerregistryutil "tkestack.io/tke/pkg/util/containerregistry"
@@ -734,8 +735,12 @@ func (c *Controller) installPrometheus(ctx context.Context, prometheus *v1.Prome
  prometheus.Status.SubVersion[AlertManagerService] = components.AlertManagerService.Tag
 
  log.Infof("Start to create prometheus")
+ provider, err := clusterprovider.GetProvider(cluster.Spec.Type)
+ if err != nil {
+ return fmt.Errorf("get provider failed: %v", err)
+ }
  // Secret for prometheus-etcd
- credential, err := util.GetClusterCredentialV1(ctx, c.client.PlatformV1(), cluster)
+ credential, err := provider.GetClusterCredentialV1(ctx, c.client.PlatformV1(), cluster, clusterprovider.AdminUsername)
  if err != nil {
  return fmt.Errorf("get credential failed: %v", err)
  }

diff --git a/pkg/platform/controller/cluster/cluster_controller.go b/pkg/platform/controller/cluster/cluster_controller.go
@@ -301,7 +301,7 @@ func (c *Controller) onCreate(ctx context.Context, cluster *platformv1.Cluster)
  if err != nil {
  return err
  }
- clusterWrapper, err := typesv1.GetCluster(ctx, c.platformClient, cluster)
+ clusterWrapper, err := clusterprovider.GetV1Cluster(ctx, c.platformClient, cluster, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }
@@ -332,7 +332,7 @@ func (c *Controller) onUpdate(ctx context.Context, cluster *platformv1.Cluster)
  if err != nil {
  return err
  }
- clusterWrapper, err := typesv1.GetCluster(ctx, c.platformClient, cluster)
+ clusterWrapper, err := clusterprovider.GetV1Cluster(ctx, c.platformClient, cluster, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }

diff --git a/pkg/platform/controller/cluster/deletion/cluster_deleter.go b/pkg/platform/controller/cluster/deletion/cluster_deleter.go
@@ -30,7 +30,6 @@ import (
  v1clientset "tkestack.io/tke/api/client/clientset/versioned/typed/platform/v1"
  platformv1 "tkestack.io/tke/api/platform/v1"
  clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
- typesv1 "tkestack.io/tke/pkg/platform/types/v1"
  "tkestack.io/tke/pkg/util/log"
 )
 
@@ -389,7 +388,7 @@ func deleteClusterProvider(ctx context.Context, deleter *clusterDeleter, cluster
  if err != nil {
  panic(err)
  }
- clusterWrapper, err := typesv1.GetCluster(ctx, deleter.platformClient, cluster)
+ clusterWrapper, err := clusterprovider.GetV1Cluster(ctx, deleter.platformClient, cluster, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }

diff --git a/pkg/platform/controller/machine/deletion/machine_deleter.go b/pkg/platform/controller/machine/deletion/machine_deleter.go
@@ -29,8 +29,8 @@ import (
  v1clientset "tkestack.io/tke/api/client/clientset/versioned/typed/platform/v1"
  platformv1 "tkestack.io/tke/api/platform/v1"
  v1 "tkestack.io/tke/api/platform/v1"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  machineprovider "tkestack.io/tke/pkg/platform/provider/machine"
- typesv1 "tkestack.io/tke/pkg/platform/types/v1"
  "tkestack.io/tke/pkg/util/apiclient"
  "tkestack.io/tke/pkg/util/log"
 )
@@ -273,7 +273,7 @@ func deleteMachineProvider(ctx context.Context, deleter *machineDeleter, machine
  if err != nil {
  panic(err)
  }
- cluster, err := typesv1.GetClusterByName(context.Background(), deleter.platformClient, machine.Spec.ClusterName)
+ cluster, err := clusterprovider.GetV1ClusterByName(context.Background(), deleter.platformClient, machine.Spec.ClusterName, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }
@@ -291,7 +291,7 @@ func deleteMachineProvider(ctx context.Context, deleter *machineDeleter, machine
 func deleteNode(ctx context.Context, deleter *machineDeleter, machine *v1.Machine) error {
  log.FromContext(ctx).Info("deleteNode doing")
 
- cluster, err := typesv1.GetClusterByName(context.Background(), deleter.platformClient, machine.Spec.ClusterName)
+ cluster, err := clusterprovider.GetV1ClusterByName(context.Background(), deleter.platformClient, machine.Spec.ClusterName, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }

diff --git a/pkg/platform/controller/machine/machine_controller.go b/pkg/platform/controller/machine/machine_controller.go
@@ -40,8 +40,8 @@ import (
  platformv1 "tkestack.io/tke/api/platform/v1"
  machineconfig "tkestack.io/tke/pkg/platform/controller/machine/config"
  "tkestack.io/tke/pkg/platform/controller/machine/deletion"
+ clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  machineprovider "tkestack.io/tke/pkg/platform/provider/machine"
- typesv1 "tkestack.io/tke/pkg/platform/types/v1"
  "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/apiclient"
  "tkestack.io/tke/pkg/util/log"
@@ -253,7 +253,7 @@ func (c *Controller) onCreate(ctx context.Context, machine *platformv1.Machine)
  if err != nil {
  return err
  }
- cluster, err := typesv1.GetClusterByName(ctx, c.platformClient, machine.Spec.ClusterName)
+ cluster, err := clusterprovider.GetV1ClusterByName(ctx, c.platformClient, machine.Spec.ClusterName, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }
@@ -280,7 +280,7 @@ func (c *Controller) onUpdate(ctx context.Context, machine *platformv1.Machine)
  return err
  }
 
- cluster, err := typesv1.GetClusterByName(ctx, c.platformClient, machine.Spec.ClusterName)
+ cluster, err := clusterprovider.GetV1ClusterByName(ctx, c.platformClient, machine.Spec.ClusterName, clusterprovider.AdminUsername)
  if err != nil {
  return err
  }
@@ -340,7 +340,7 @@ func (c *Controller) checkHealth(ctx context.Context, machine *platformv1.Machin
 
 func (c *Controller) ensureSyncMachineNodeLabel(ctx context.Context, machine *platformv1.Machine) {
 
- cluster, err := typesv1.GetClusterByName(ctx, c.platformClient, machine.Spec.ClusterName)
+ cluster, err := clusterprovider.GetV1ClusterByName(ctx, c.platformClient, machine.Spec.ClusterName, clusterprovider.AdminUsername)
  if err != nil {
  log.FromContext(ctx).Error(err, "sync Machine node label error")
  return

diff --git a/pkg/platform/provider/cluster/cluster.go b/pkg/platform/provider/cluster/cluster.go
@@ -19,18 +19,29 @@
 package cluster
 
 import (
+ "context"
  "fmt"
  "sort"
  "sync"
 
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apiserver/pkg/server/mux"
+ "tkestack.io/tke/api/client/clientset/internalversion/typed/platform/internalversion"
+ platformversionedclient "tkestack.io/tke/api/client/clientset/versioned/typed/platform/v1"
+ "tkestack.io/tke/api/platform"
+ platformv1 "tkestack.io/tke/api/platform/v1"
+ "tkestack.io/tke/pkg/platform/types"
+ v1 "tkestack.io/tke/pkg/platform/types/v1"
 )
 
 var (
  providersMu sync.RWMutex
  providers = make(map[string]Provider)
 )
 
+const AdminUsername = "admin"
+
 // Register makes a provider available by the provided name.
 // If Register is called twice with the same name or if provider is nil,
 // it panics.
@@ -99,3 +110,52 @@ func GetProvider(name string) (Provider, error) {
 
  return provider, nil
 }
+
+func GetCluster(ctx context.Context, platformClient internalversion.PlatformInterface, cluster *platform.Cluster, username string) (*types.Cluster, error) {
+ result := new(types.Cluster)
+ result.Cluster = cluster
+ provider, err := GetProvider(cluster.Spec.Type)
+ if err != nil {
+ return nil, err
+ }
+ clusterCredential, err := provider.GetClusterCredential(ctx, platformClient, cluster, username)
+ if err != nil && !apierrors.IsNotFound(err) {
+ return result, err
+ }
+ result.ClusterCredential = clusterCredential
+
+ return result, nil
+}
+
+func GetClusterByName(ctx context.Context, platformClient internalversion.PlatformInterface, clsname, username string) (*types.Cluster, error) {
+ cluster, err := platformClient.Clusters().Get(ctx, clsname, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+ return GetCluster(ctx, platformClient, cluster, username)
+}
+
+func GetV1Cluster(ctx context.Context, platformClient platformversionedclient.PlatformV1Interface, cluster *platformv1.Cluster, username string) (*v1.Cluster, error) {
+ result := new(v1.Cluster)
+ result.Cluster = cluster
+ result.IsCredentialChanged = false
+ provider, err := GetProvider(cluster.Spec.Type)
+ if err != nil {
+ return nil, err
+ }
+ clusterCredential, err := provider.GetClusterCredentialV1(ctx, platformClient, cluster, username)
+ if err != nil && !apierrors.IsNotFound(err) {
+ return result, err
+ }
+ result.ClusterCredential = clusterCredential
+
+ return result, nil
+}
+
+func GetV1ClusterByName(ctx context.Context, platformClient platformversionedclient.PlatformV1Interface, clsname, username string) (*v1.Cluster, error) {
+ cluster, err := platformClient.Clusters().Get(ctx, clsname, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+ return GetV1Cluster(ctx, platformClient, cluster, username)
+}