Support certificates configuration in TLSStore CRD

Co-authored-by: Romain <[email protected]>

docs/content/https/tls.md

@@ -364,8 +364,9 @@ spec:
 
 ### Strict SNI Checking
 
-With strict SNI checking enabled, Traefik won't allow connections from clients
-that do not specify a server_name extension or don't match any certificate configured on the tlsOption.
+With strict SNI checking enabled, Traefik won't allow connections from clients that do not specify a server_name extension
+or don't match any of the configured certificates.
+The default certificate is irrelevant on that matter.
 
 ```yaml tab="File (YAML)"
 # Dynamic configuration

docs/content/reference/dynamic-configuration/traefik.containo.us_tlsstores.yaml

@@ -36,9 +36,23 @@ spec:
  spec:
  description: TLSStoreSpec configures a TLSStore resource.
  properties:
+ certificates:
+ description: Certificates is a list of secret names, each secret holding
+ a key/certificate pair to add to the store.
+ items:
+ description: Certificate holds a secret name for the TLSStore resource.
+ properties:
+ secretName:
+ description: SecretName is the name of the referenced Kubernetes
+ Secret to specify the certificate details.
+ type: string
+ required:
+ - secretName
+ type: object
+ type: array
  defaultCertificate:
- description: DefaultCertificate holds a secret name for the TLSOption
- resource.
+ description: DefaultCertificate is the name of the secret holding
+ the default key/certificate pair for the store.
  properties:
  secretName:
  description: SecretName is the name of the referenced Kubernetes
@@ -47,8 +61,6 @@ spec:
  required:
  - secretName
  type: object
- required:
- - defaultCertificate
  type: object
  required:
  - metadata

docs/content/routing/providers/kubernetes-crd.md

@@ -1618,25 +1618,27 @@ or referencing TLS stores in the [`IngressRoute`](#kind-ingressroute) / [`Ingres
  Traefik currently only uses the [TLS Store named "default"](../../https/tls.md#certificates-stores).
  This means that if you have two stores that are named default in different kubernetes namespaces,
  they may be randomly chosen.
- For the time being, please only configure one TLSSTore named default.
+ For the time being, please only configure one TLSStore named default.
 
 !!! info "TLSStore Attributes"
-
  ```yaml tab="TLSStore"
  apiVersion: traefik.containo.us/v1alpha1
  kind: TLSStore
  metadata:
  name: default
  namespace: default
-
  spec:
-  defaultCertificate:
-  secretName: my-secret # [1]
+ certificates: # [1]
+ - secretName: foo 
+ - secretName: bar
+ defaultCertificate: # [2]
+ secretName: secret 
  ```
 
-| Ref | Attribute | Purpose |
-|-----|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [1] | `secretName` | The name of the referenced Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) that holds the default certificate for the store. |
+| Ref | Attribute | Purpose |
+|-----|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [1] | `certificates` | List of Kubernetes [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), each of them holding a key/certificate pair to add to the store. |
+| [2] | `defaultCertificate` | Name of a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) that holds the default key/certificate pair for the store. |
 
 ??? example "Declaring and referencing a TLSStore"
 

integration/fixtures/k8s/01-traefik-crd.yml

@@ -1356,9 +1356,23 @@ spec:
  spec:
  description: TLSStoreSpec configures a TLSStore resource.
  properties:
+ certificates:
+ description: Certificates is a list of secret names, each secret holding
+ a key/certificate pair to add to the store.
+ items:
+ description: Certificate holds a secret name for the TLSStore resource.
+ properties:
+ secretName:
+ description: SecretName is the name of the referenced Kubernetes
+ Secret to specify the certificate details.
+ type: string
+ required:
+ - secretName
+ type: object
+ type: array
  defaultCertificate:
- description: DefaultCertificate holds a secret name for the TLSOption
- resource.
+ description: DefaultCertificate is the name of the secret holding
+ the default key/certificate pair for the store.
  properties:
  secretName:
  description: SecretName is the name of the referenced Kubernetes
@@ -1367,8 +1381,6 @@ spec:
  required:
  - secretName
  type: object
- required:
- - defaultCertificate
  type: object
  required:
  - metadata

pkg/provider/kubernetes/crd/fixtures/with_tls_store_certificates.yml

@@ -0,0 +1,43 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: TLSStore
+metadata:
+ name: default
+ namespace: default
+
+spec:
+ certificates:
+ - secretName: supersecret
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: supersecret
+ namespace: default
+
+data:
+ tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0=
+ tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0=
+
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: test.route
+ namespace: default
+
+spec:
+ entryPoints:
+ - web
+
+ routes:
+ - match: Host(`foo.com`) && PathPrefix(`/bar`)
+ kind: Rule
+ priority: 12
+ services:
+ - name: whoami
+ port: 80
+
+ tls:
+ store:
+ name: default

pkg/provider/kubernetes/crd/kubernetes.go

@@ -179,18 +179,25 @@ func (p *Provider) Provide(configurationChan chan<- dynamic.Message, pool *safe.
 }
 
 func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client) *dynamic.Configuration {
- tlsConfigs := make(map[string]*tls.CertAndStores)
+ stores, tlsConfigs := buildTLSStores(ctx, client)
+ if tlsConfigs == nil {
+ tlsConfigs = make(map[string]*tls.CertAndStores)
+ }
+
  conf := &dynamic.Configuration{
+ // TODO: choose between mutating and returning tlsConfigs
  HTTP: p.loadIngressRouteConfiguration(ctx, client, tlsConfigs),
  TCP: p.loadIngressRouteTCPConfiguration(ctx, client, tlsConfigs),
  UDP: p.loadIngressRouteUDPConfiguration(ctx, client),
  TLS: &dynamic.TLSConfiguration{
- Certificates: getTLSConfig(tlsConfigs),
- Options: buildTLSOptions(ctx, client),
- Stores: buildTLSStores(ctx, client),
+ Options: buildTLSOptions(ctx, client),
+ Stores: stores,
  },
  }
 
+ // Done after because tlsConfigs is mutated by the others above.
+ conf.TLS.Certificates = getTLSConfig(tlsConfigs)
+
  for _, middleware := range client.GetMiddlewares() {
  id := provider.Normalize(makeID(middleware.Namespace, middleware.Name))
  ctxMid := log.With(ctx, log.Str(log.MiddlewareName, id))
@@ -828,57 +835,86 @@ func buildTLSOptions(ctx context.Context, client Client) map[string]tls.Options
  return tlsOptions
 }
 
-func buildTLSStores(ctx context.Context, client Client) map[string]tls.Store {
+func buildTLSStores(ctx context.Context, client Client) (map[string]tls.Store, map[string]*tls.CertAndStores) {
  tlsStoreCRD := client.GetTLSStores()
- var tlsStores map[string]tls.Store
-
  if len(tlsStoreCRD) == 0 {
- return tlsStores
+ return nil, nil
  }
- tlsStores = make(map[string]tls.Store)
+
  var nsDefault []string
+ tlsStores := make(map[string]tls.Store)
+ tlsConfigs := make(map[string]*tls.CertAndStores)
 
- for _, tlsStore := range tlsStoreCRD {
- namespace := tlsStore.Namespace
- secretName := tlsStore.Spec.DefaultCertificate.SecretName
- logger := log.FromContext(log.With(ctx, log.Str("tlsStore", tlsStore.Name), log.Str("namespace", namespace), log.Str("secretName", secretName)))
+ for _, t := range tlsStoreCRD {
+ logger := log.FromContext(log.With(ctx, log.Str("TLSStore", t.Name), log.Str("namespace", t.Namespace)))
 
- secret, exists, err := client.GetSecret(namespace, secretName)
- if err != nil {
- logger.Errorf("Failed to fetch secret %s/%s: %v", namespace, secretName, err)
- continue
- }
- if !exists {
- logger.Errorf("Secret %s/%s does not exist", namespace, secretName)
- continue
- }
-
- cert, key, err := getCertificateBlocks(secret, namespace, secretName)
- if err != nil {
- logger.Errorf("Could not get certificate blocks: %v", err)
- continue
- }
+ id := makeID(t.Namespace, t.Name)
 
- id := makeID(tlsStore.Namespace, tlsStore.Name)
  // If the name is default, we override the default config.
- if tlsStore.Name == tls.DefaultTLSStoreName {
- id = tlsStore.Name
- nsDefault = append(nsDefault, tlsStore.Namespace)
+ if t.Name == tls.DefaultTLSStoreName {
+ id = t.Name
+ nsDefault = append(nsDefault, t.Namespace)
  }
- tlsStores[id] = tls.Store{
- DefaultCertificate: &tls.Certificate{
+
+ var tlsStore tls.Store
+
+ if t.Spec.DefaultCertificate != nil {
+ secretName := t.Spec.DefaultCertificate.SecretName
+
+ secret, exists, err := client.GetSecret(t.Namespace, secretName)
+ if err != nil {
+ logger.Errorf("Failed to fetch secret %s/%s: %v", t.Namespace, secretName, err)
+ continue
+ }
+ if !exists {
+ logger.Errorf("Secret %s/%s does not exist", t.Namespace, secretName)
+ continue
+ }
+
+ cert, key, err := getCertificateBlocks(secret, t.Namespace, secretName)
+ if err != nil {
+ logger.Errorf("Could not get certificate blocks: %v", err)
+ continue
+ }
+
+ tlsStore.DefaultCertificate = &tls.Certificate{
  CertFile: tls.FileOrContent(cert),
  KeyFile: tls.FileOrContent(key),
- },
+ }
  }
+
+ if err := buildCertificates(client, id, t.Namespace, t.Spec.Certificates, tlsConfigs); err != nil {
+ logger.Errorf("Failed to load certificates: %v", err)
+ continue
+ }
+
+ tlsStores[id] = tlsStore
  }
 
  if len(nsDefault) > 1 {
  delete(tlsStores, tls.DefaultTLSStoreName)
  log.FromContext(ctx).Errorf("Default TLS Stores defined in multiple namespaces: %v", nsDefault)
  }
 
- return tlsStores
+ return tlsStores, tlsConfigs
+}
+
+// buildCertificates loads TLSStore certificates from secrets and sets them into tlsConfigs.
+func buildCertificates(client Client, tlsStore, namespace string, certificates []v1alpha1.Certificate, tlsConfigs map[string]*tls.CertAndStores) error {
+ for _, c := range certificates {
+ configKey := namespace + "/" + c.SecretName
+ if _, tlsExists := tlsConfigs[configKey]; !tlsExists {
+ certAndStores, err := getTLS(client, c.SecretName, namespace)
+ if err != nil {
+ return fmt.Errorf("unable to read secret %s: %w", configKey, err)
+ }
+
+ certAndStores.Stores = []string{tlsStore}
+ tlsConfigs[configKey] = certAndStores
+ }
+ }
+
+ return nil
 }
 
 func makeServiceKey(rule, ingressName string) (string, error) {

pkg/provider/kubernetes/crd/kubernetes_http.go

@@ -495,6 +495,7 @@ func namespaceOrFallback(lb v1alpha1.LoadBalancerSpec, fallback string) string {
  return fallback
 }
 
+// getTLSHTTP mutates tlsConfigs.
 func getTLSHTTP(ctx context.Context, ingressRoute *v1alpha1.IngressRoute, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error {
  if ingressRoute.Spec.TLS == nil {
  return nil

pkg/provider/kubernetes/crd/kubernetes_tcp.go

@@ -269,6 +269,7 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc v1alpha1.
  return servers, nil
 }
 
+// getTLSTCP mutates tlsConfigs.
 func getTLSTCP(ctx context.Context, ingressRoute *v1alpha1.IngressRouteTCP, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error {
  if ingressRoute.Spec.TLS == nil {
  return nil

pkg/provider/kubernetes/crd/kubernetes_test.go

@@ -3480,6 +3480,63 @@ func TestLoadIngressRoutes(t *testing.T) {
  },
  },
  },
+ {
+ desc: "TLS with tls store containing certificates",
+ paths: []string{"services.yml", "with_tls_store_certificates.yml"},
+ expected: &dynamic.Configuration{
+ TLS: &dynamic.TLSConfiguration{
+ Certificates: []*tls.CertAndStores{
+ {
+ Certificate: tls.Certificate{
+ CertFile: tls.FileOrContent("-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"),
+ KeyFile: tls.FileOrContent("-----BEGIN PRIVATE KEY-----\n-----END PRIVATE KEY-----"),
+ },
+ Stores: []string{"default"},
+ },
+ },
+ Stores: map[string]tls.Store{
+ "default": {},
+ },
+ },
+ UDP: &dynamic.UDPConfiguration{
+ Routers: map[string]*dynamic.UDPRouter{},
+ Services: map[string]*dynamic.UDPService{},
+ },
+ TCP: &dynamic.TCPConfiguration{
+ Routers: map[string]*dynamic.TCPRouter{},
+ Middlewares: map[string]*dynamic.TCPMiddleware{},
+ Services: map[string]*dynamic.TCPService{},
+ },
+ HTTP: &dynamic.HTTPConfiguration{
+ Routers: map[string]*dynamic.Router{
+ "default-test-route-6b204d94623b3df4370c": {
+ EntryPoints: []string{"web"},
+ Service: "default-test-route-6b204d94623b3df4370c",
+ Rule: "Host(`foo.com`) && PathPrefix(`/bar`)",
+ Priority: 12,
+ TLS: &dynamic.RouterTLSConfig{},
+ },
+ },
+ Middlewares: map[string]*dynamic.Middleware{},
+ Services: map[string]*dynamic.Service{
+ "default-test-route-6b204d94623b3df4370c": {
+ LoadBalancer: &dynamic.ServersLoadBalancer{
+ Servers: []dynamic.Server{
+ {
+ URL: "http:https://10.10.0.1:80",
+ },
+ {
+ URL: "http:https://10.10.0.2:80",
+ },
+ },
+ PassHostHeader: Bool(true),
+ },
+ },
+ },
+ ServersTransports: map[string]*dynamic.ServersTransport{},
+ },
+ },
+ },
  {
  desc: "TLS with tls store default two times",
  paths: []string{"services.yml", "with_tls_store.yml", "with_default_tls_store.yml"},