feat: platform and auth support webhook authentication

cmd/tke-auth-api/app/config/config.go

@@ -42,6 +42,8 @@ import (
  "k8s.io/apiserver/pkg/authentication/request/bearertoken"
  "k8s.io/apiserver/pkg/authentication/request/union"
  "k8s.io/apiserver/pkg/authentication/request/websocket"
+ tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
+ tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
  "k8s.io/apiserver/pkg/authorization/authorizer"
  genericapiserver "k8s.io/apiserver/pkg/server"
  serverstorage "k8s.io/apiserver/pkg/server/storage"
@@ -144,8 +146,13 @@ func CreateConfigFromOptions(serverName string, opts *options.Options) (*Config,
  }
 
  authClient := authinternalclient.NewForConfigOrDie(genericAPIServerConfig.LoopbackClientConfig)
+ apiKeyAuth, err := authenticator.NewAPIKeyAuthenticator(authClient)
+ if err != nil {
+ return nil, err
+ }
+
  tokenAuth := authenticator.NewTokenAuthenticator(authClient)
- if err := setupAuthentication(genericAPIServerConfig, opts.Authentication, tokenAuth); err != nil {
+ if err := setupAuthentication(genericAPIServerConfig, opts.Authentication, []genericauthenticator.Token{tokenAuth, apiKeyAuth}); err != nil {
  return nil, err
  }
 
@@ -160,11 +167,6 @@ func CreateConfigFromOptions(serverName string, opts *options.Options) (*Config,
  return nil, err
  }
 
- apiKeyAuth, err := authenticator.NewAPIKeyAuthenticator(authClient)
- if err != nil {
- return nil, err
- }
-
  local.SetupRestClient(authClient)
  log.Info("init tenant type", log.String("type", opts.Auth.InitTenantType))
  switch opts.Auth.InitTenantType {
@@ -204,7 +206,7 @@ func CreateConfigFromOptions(serverName string, opts *options.Options) (*Config,
  }, nil
 }
 
-func setupAuthentication(genericAPIServerConfig *genericapiserver.Config, opts *apiserveroptions.AuthenticationWithAPIOptions, tokenAuth *authenticator.TokenAuthenticator) error {
+func setupAuthentication(genericAPIServerConfig *genericapiserver.Config, opts *apiserveroptions.AuthenticationWithAPIOptions, tokenAuthenticators []genericauthenticator.Token) error {
  if err := authentication.SetupAuthentication(genericAPIServerConfig, opts); err != nil {
  return nil
  }
@@ -215,6 +217,11 @@ func setupAuthentication(genericAPIServerConfig *genericapiserver.Config, opts *
  configAuthenticators,
  }
  defs := *configDefs
+ tokenAuth := tokenunion.New(tokenAuthenticators...)
+ if opts.TokenSuccessCacheTTL > 0 || opts.TokenFailureCacheTTL > 0 {
+ tokenAuth = tokencache.New(tokenAuth, true, opts.TokenSuccessCacheTTL, opts.TokenFailureCacheTTL)
+ }
+
  authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth))
  defs["BearerToken"] = &spec.SecurityScheme{
  SecuritySchemeProps: spec.SecuritySchemeProps{

cmd/tke-installer/app/installer/manifests/tke-platform-api/tke-platform-api.yaml

@@ -152,6 +152,11 @@ data:
  {{- end}}
 {{- end }}
 
+{{- if .EnableAuth }}
+ [authentication.webhook]
+ config_file = "/app/conf/tke-authn-webhook.yaml"
+{{- end }}
+
  [authentication.requestheader]
  client_ca_file = "/app/certs/front-proxy-ca.crt"
  username_headers = "X-Remote-User"
@@ -161,10 +166,30 @@ data:
  [authorization]
 {{- if .EnableAuth }}
  mode = "Webhook"
- webhook_config_file = "/app/conf/tke-auth-webhook.yaml"
+ webhook_config_file = "/app/conf/tke-authz-webhook.yaml"
 {{- end }}
 
- tke-auth-webhook.yaml: |
+ tke-authn-webhook.yaml: |
+ apiVersion: v1
+ kind: Config
+ clusters:
+ - name: tke
+ cluster:
+ certificate-authority: /app/certs/ca.crt
+ server: https://tke-auth-api/auth/authn
+ users:
+ - name: admin-cert
+ user:
+ client-certificate: /app/certs/admin.crt
+ client-key: /app/certs/admin.key
+ current-context: tke
+ contexts:
+ - context:
+ cluster: tke
+ user: admin-cert
+ name: tke
+
+ tke-authz-webhook.yaml: |
  apiVersion: v1
  kind: Config
  clusters:

pkg/apiserver/authentication/authentication.go

@@ -20,6 +20,7 @@ package authentication
 
 import (
  "fmt"
+
  "github.com/go-openapi/spec"
  "k8s.io/apiserver/pkg/authentication/authenticator"
  genericapiserver "k8s.io/apiserver/pkg/server"
@@ -94,6 +95,12 @@ func buildAuthenticator(o *options.AuthenticationOptions, apiAudiences []string)
  ret.OIDCRequiredClaims = o.OIDC.RequiredClaims
  }
 
+ if o.WebHook != nil {
+ ret.WebhookTokenAuthnConfigFile = o.WebHook.ConfigFile
+ ret.WebhookTokenAuthnVersion = o.WebHook.Version
+ ret.WebhookTokenAuthnCacheTTL = o.WebHook.CacheTTL
+ }
+
  if o.RequestHeader != nil {
  var err error
  ret.RequestHeaderConfig, err = o.RequestHeader.ToAuthenticationRequestHeaderConfig()

pkg/apiserver/authentication/config.go

@@ -33,6 +33,7 @@ import (
  tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
  "k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile"
  "k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth"
+ "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
  certutil "k8s.io/client-go/util/cert"
  "time"
  "tkestack.io/tke/pkg/apiserver/authentication/authenticator/localtrust"
@@ -41,24 +42,28 @@ import (
 
 // Config contains the data on how to authenticate a request to the Kube API Server
 type Config struct {
- ClientCAFile string
- TokenAuthFile string
- OIDCIssuerURL string
- OIDCExternalIssuerURL string
- OIDCClientID string
- OIDCCAFile string
- OIDCUsernameClaim string
- OIDCUsernamePrefix string
- OIDCGroupsClaim string
- OIDCGroupsPrefix string
- OIDCTenantIDClaim string
- OIDCTenantIDPrefix string
- OIDCSigningAlgs []string
- OIDCRequiredClaims map[string]string
- APIAudiences authenticator.Audiences
- TokenSuccessCacheTTL time.Duration
- TokenFailureCacheTTL time.Duration
- RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig
+ ClientCAFile string
+ TokenAuthFile string
+ OIDCIssuerURL string
+ OIDCExternalIssuerURL string
+ OIDCClientID string
+ OIDCCAFile string
+ OIDCUsernameClaim string
+ OIDCUsernamePrefix string
+ OIDCGroupsClaim string
+ OIDCGroupsPrefix string
+ OIDCTenantIDClaim string
+ OIDCTenantIDPrefix string
+ OIDCSigningAlgs []string
+ OIDCRequiredClaims map[string]string
+ APIAudiences authenticator.Audiences
+ WebhookTokenAuthnConfigFile string
+ WebhookTokenAuthnVersion string
+ WebhookTokenAuthnCacheTTL time.Duration
+
+ TokenSuccessCacheTTL time.Duration
+ TokenFailureCacheTTL time.Duration
+ RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig
 }
 
 // New returns an authenticator.Request or an error that supports the standard
@@ -123,6 +128,14 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
  tokenAuthenticators = append(tokenAuthenticators, oidcAuth)
  }
 
+ if len(config.WebhookTokenAuthnConfigFile) > 0 {
+ webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnVersion, config.WebhookTokenAuthnCacheTTL, config.APIAudiences)
+ if err != nil {
+ return nil, nil, err
+ }
+ tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
+ }
+
  if len(tokenAuthenticators) > 0 {
  // Union the token authenticators
  tokenAuth := tokenunion.New(tokenAuthenticators...)
@@ -188,3 +201,12 @@ func newAuthenticatorFromOIDCIssuerURL(opts *oidc.Options) (authenticator.Token,
 
  return tokenAuthenticator, nil
 }
+
+func newWebhookTokenAuthenticator(webhookConfigFile string, version string, ttl time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) {
+ webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, version, implicitAuds)
+ if err != nil {
+ return nil, err
+ }
+
+ return tokencache.New(webhookTokenAuthenticator, false, ttl, ttl), nil
+}

pkg/apiserver/options/authentication.go

@@ -20,6 +20,7 @@ package options
 
 import (
  "fmt"
+ "k8s.io/klog"
  "time"
 
  "github.com/spf13/pflag"
@@ -51,6 +52,7 @@ const (
 type AuthenticationOptions struct {
  ClientCert *genericoptions.ClientCertAuthenticationOptions
  OIDC *OIDCOptions
+ WebHook *WebHookOptions
  RequestHeader *genericoptions.RequestHeaderAuthenticationOptions
  TokenFile *TokenFileAuthenticationOptions
  TokenSuccessCacheTTL time.Duration
@@ -74,6 +76,7 @@ func NewAuthenticationOptions() *AuthenticationOptions {
  return &AuthenticationOptions{
  ClientCert: &genericoptions.ClientCertAuthenticationOptions{},
  OIDC: NewOIDCOptions(),
+ WebHook: NewWebhookOptions(),
  RequestHeader: &genericoptions.RequestHeaderAuthenticationOptions{},
  TokenFile: &TokenFileAuthenticationOptions{},
  TokenSuccessCacheTTL: 10 * time.Second,
@@ -93,6 +96,8 @@ func (o *AuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 
  o.OIDC.AddFlags(fs)
 
+ o.WebHook.AddFlags(fs)
+
  o.RequestHeader.AddFlags(fs)
  _ = viper.BindPFlag(configAuthnRequestHeaderUsernameHeaders, fs.Lookup(flagAuthnRequestHeaderUsernameHeaders))
  _ = viper.BindPFlag(configAuthnRequestHeaderGroupHeaders, fs.Lookup(flagAuthnRequestHeaderGroupHeaders))
@@ -110,6 +115,7 @@ func (o *AuthenticationOptions) ApplyFlags() []error {
  o.TokenFile.TokenFile = viper.GetString(configAuthnTokenFile)
 
  errs = append(errs, o.OIDC.ApplyFlags()...)
+ errs = append(errs, o.WebHook.ApplyFlags()...)
 
  o.RequestHeader.AllowedNames = viper.GetStringSlice(configAuthnRequestHeaderAllowedNames)
  o.RequestHeader.ClientCAFile = viper.GetString(configAuthnRequestHeaderClientCAFile)
@@ -125,5 +131,16 @@ func (o *AuthenticationOptions) ApplyFlags() []error {
  o.OIDC.ExternalIssuerURL = o.OIDC.IssuerURL
  }
 
+ if o.WebHook != nil {
+ if len(o.WebHook.ConfigFile) > 0 && o.WebHook.CacheTTL > 0 {
+ if o.TokenSuccessCacheTTL > 0 && o.WebHook.CacheTTL < o.TokenSuccessCacheTTL {
+ klog.Warningf("the webhook cache ttl of %s is shorter than the overall cache ttl of %s for successful token authentication attempts.", o.WebHook.CacheTTL, o.TokenSuccessCacheTTL)
+ }
+ if o.TokenFailureCacheTTL > 0 && o.WebHook.CacheTTL < o.TokenFailureCacheTTL {
+ klog.Warningf("the webhook cache ttl of %s is shorter than the overall cache ttl of %s for failed token authentication attempts.", o.WebHook.CacheTTL, o.TokenFailureCacheTTL)
+ }
+ }
+ }
+
  return errs
 }

pkg/apiserver/options/webhook.go

@@ -0,0 +1,81 @@
+/*
+ * Tencent is pleased to support the open source community by making TKEStack
+ * available.
+ *
+ * Copyright (C) 2012-2019 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * https://opensource.org/licenses/Apache-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS, WITHOUT
+ * WARRANTIES OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package options
+
+import (
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+ "time"
+)
+
+const (
+ flagTokenWebhookConfigFile = "token-webhook-config-file"
+ flagTokenWebhookVersion = "token-webhook-version"
+ flagTokenWebhookCacheTTL = "token-webhook-cache-ttl"
+)
+
+const (
+ configTokenWebhookConfigFile = "authentication.webhook.config_file"
+ configTokenWehookVersion = "authentication.webhook.version"
+ configTokenWehookCacheTTL = "authentication.webhook.cache_ttl"
+)
+
+type WebHookOptions struct {
+ ConfigFile string
+ Version string
+ CacheTTL time.Duration
+}
+
+
+// NewWebhookOptions creates the default WebHookOptions object.
+func NewWebhookOptions() *WebHookOptions {
+ return &WebHookOptions{
+ Version: "v1beta1",
+ CacheTTL: 2 * time.Minute,
+ }
+}
+
+// AddFlags adds flags for log to the specified FlagSet object.
+func (w *WebHookOptions) AddFlags(fs *pflag.FlagSet) {
+ fs.StringVar(&w.ConfigFile, flagTokenWebhookConfigFile, w.ConfigFile, ""+
+ "File with webhook configuration for token authentication in kubeconfig format. "+
+ "The API server will query the remote service to determine authentication for bearer tokens.")
+ _ = viper.BindPFlag(configTokenWebhookConfigFile, fs.Lookup(flagTokenWebhookConfigFile))
+
+ fs.StringVar(&w.Version, flagTokenWebhookVersion, w.Version, ""+
+ "The API version of the authentication.k8s.io TokenReview to send to and expect from the webhook.")
+ _ = viper.BindPFlag(configTokenWehookVersion, fs.Lookup(flagTokenWebhookVersion))
+
+ fs.DurationVar(&w.CacheTTL, flagTokenWebhookCacheTTL, w.CacheTTL,
+ "The duration to cache responses from the webhook token authenticator.")
+ _ = viper.BindPFlag(configTokenWehookCacheTTL, fs.Lookup(flagTokenWebhookCacheTTL))
+
+}
+
+// ApplyFlags parsing parameters from the command line or configuration file
+// to the options instance.
+func (w *WebHookOptions) ApplyFlags() []error {
+ var errs []error
+ w.ConfigFile = viper.GetString(configTokenWebhookConfigFile)
+ w.Version = viper.GetString(configTokenWehookVersion)
+ w.CacheTTL = viper.GetDuration(configTokenWehookCacheTTL)
+
+ return errs
+}
+

pkg/auth/authentication/authenticator/apikey.go

@@ -63,6 +63,7 @@ func (h *APIKeyAuthenticator) AuthenticateToken(ctx context.Context, token strin
  fields.OneTermEqualSelector("spec.tenantID", tokenInfo.TenantID),
  fields.OneTermEqualSelector("spec.apiKey", token))
 
+ log.Info("apikey selector", log.String("selector", selector.String()))
  apiKeyList, err := h.authClient.APIKeys().List(metav1.ListOptions{FieldSelector: selector.String()})
  if err != nil {
  log.Error("List api keys failed", log.String("api key", token), log.Err(err))

pkg/auth/registry/apikey/storage/storage.go

@@ -22,8 +22,6 @@ import (
  "context"
  "fmt"
 
- "k8s.io/apimachinery/pkg/fields"
-
  "tkestack.io/tke/pkg/apiserver/authentication"
 
  apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -125,11 +123,7 @@ func ValidateExportObjectAndTenantID(ctx context.Context, store *registry.Store,
 // ValidateListObject validate if list by admin, if false, filter deleted apikey.
 func ValidateListObjectAndTenantID(ctx context.Context, store *registry.Store, options *metainternal.ListOptions) (runtime.Object, error) {
  wrappedOptions := apiserverutil.PredicateListOptions(ctx, options)
-
- username, tenantID := authentication.GetUsernameAndTenantID(ctx)
- if tenantID != "" {
- wrappedOptions.FieldSelector = fields.AndSelectors(wrappedOptions.FieldSelector, fields.OneTermEqualSelector("spec.username", username))
- }
+ wrappedOptions = util.PredicateUserNameListOptions(ctx, wrappedOptions)
 
  obj, err := store.List(ctx, wrappedOptions)
  if err != nil {