From 3b7340b91d0f910d5e8cb6e40d9b9fc781a62995 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 26 Apr 2024 17:43:48 +0100 Subject: [PATCH] Add support for named ports in ingresses which use-cluster-ip (#5456) --- internal/configs/ingress_test.go | 132 +++++++++++++++++++++++++++++++ internal/k8s/controller.go | 16 ++++ 2 files changed, 148 insertions(+) diff --git a/internal/configs/ingress_test.go b/internal/configs/ingress_test.go index 07b6a3d881c..2810fbaac90 100644 --- a/internal/configs/ingress_test.go +++ b/internal/configs/ingress_test.go @@ -2,6 +2,7 @@ package configs import ( "errors" + "fmt" "reflect" "strings" "testing" @@ -794,6 +795,93 @@ func createExpectedConfigForMergeableCafeIngressWithUseClusterIP() version1.Ingr return expected } +func createExpectedConfigForCafeIngressWithUseClusterIPNamedPorts() version1.IngressNginxConfig { + upstreamZoneSize := "256k" + + coffeeUpstream := version1.Upstream{ + Name: "default-cafe-ingress-cafe.example.com-coffee-svc-custom-port-name", + LBMethod: "random two least_conn", + UpstreamZoneSize: upstreamZoneSize, + UpstreamServers: []version1.UpstreamServer{ + { + Address: "10.109.204.250:3000", + MaxFails: 1, + MaxConns: 0, + FailTimeout: "10s", + }, + }, + } + + teaUpstream := version1.Upstream{ + Name: "default-cafe-ingress-cafe.example.com-tea-svc-80", + LBMethod: "random two least_conn", + UpstreamZoneSize: upstreamZoneSize, + UpstreamServers: []version1.UpstreamServer{ + { + Address: "10.109.204.250:80", + MaxFails: 1, + MaxConns: 0, + FailTimeout: "10s", + }, + }, + } + + expected := version1.IngressNginxConfig{ + Upstreams: []version1.Upstream{ + coffeeUpstream, + teaUpstream, + }, + Servers: []version1.Server{ + { + Name: "cafe.example.com", + ServerTokens: "on", + Locations: []version1.Location{ + { + Path: "/coffee", + ServiceName: "coffee-svc", + Upstream: coffeeUpstream, + ProxyConnectTimeout: "60s", + ProxyReadTimeout: "60s", + ProxySendTimeout: "60s", + ClientMaxBodySize: "1m", + ProxyBuffering: true, + ProxySSLName: "coffee-svc.default.svc", + }, + { + Path: "/tea", + ServiceName: "tea-svc", + Upstream: teaUpstream, + ProxyConnectTimeout: "60s", + ProxyReadTimeout: "60s", + ProxySendTimeout: "60s", + ClientMaxBodySize: "1m", + ProxyBuffering: true, + ProxySSLName: "tea-svc.default.svc", + }, + }, + SSL: true, + SSLCertificate: "/etc/nginx/secrets/default-cafe-secret", + SSLCertificateKey: "/etc/nginx/secrets/default-cafe-secret", + StatusZone: "cafe.example.com", + HSTSMaxAge: 2592000, + Ports: []int{80}, + SSLPorts: []int{443}, + SSLRedirect: true, + HealthChecks: make(map[string]version1.HealthCheck), + }, + }, + Ingress: version1.Ingress{ + Name: "cafe-ingress", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "nginx.org/use-cluster-ip": "true", + }, + }, + } + return expected +} + func createExpectedConfigForCafeIngressWithUseClusterIP() version1.IngressNginxConfig { upstreamZoneSize := "256k" @@ -910,6 +998,50 @@ func TestGenerateNginxCfgWithUseClusterIP(t *testing.T) { } } +func TestGenerateNginxCfgWithUseClusterIPWithNamedPorts(t *testing.T) { + t.Parallel() + customPort := 3000 + customPortName := "custom-port-name" + clusterIP := "10.109.204.250" + cafeIngressEx := createCafeIngressEx() + cafeIngressEx.Ingress.Annotations["nginx.org/use-cluster-ip"] = "true" + cafeIngressEx.Endpoints["coffee-svccustom-port-name"] = make([]string, 1) + + // coffee will use a named port + cafeIngressEx.Endpoints["coffee-svccustom-port-name"][0] = fmt.Sprintf("%s:%d", clusterIP, customPort) + + // tea will not use a named port + cafeIngressEx.Endpoints["tea-svc80"][0] = fmt.Sprintf("%s:%d", clusterIP, 80) + + // unset the port number and set the port name for the /coffee path + cafeIngressEx.Ingress.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Port.Number = 0 + cafeIngressEx.Ingress.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Port.Name = customPortName + + isPlus := false + configParams := NewDefaultConfigParams(isPlus) + + expected := createExpectedConfigForCafeIngressWithUseClusterIPNamedPorts() + + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: false, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) + + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff) + } + if len(warnings) != 0 { + t.Errorf("generateNginxCfg() returned warnings: %v", warnings) + } +} + func TestGenerateNginxCfgForLimitReq(t *testing.T) { t.Parallel() cafeIngressEx := createCafeIngressEx() diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 843f94fe9da..fa881f7ca45 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -2970,6 +2970,14 @@ func (lbc *LoadBalancerController) createIngressEx(ing *networking.Ingress, vali } if svc != nil && !external && hasUseClusterIP { + if ing.Spec.DefaultBackend.Service.Port.Number == 0 { + for _, port := range svc.Spec.Ports { + if port.Name == ing.Spec.DefaultBackend.Service.Port.Name { + ing.Spec.DefaultBackend.Service.Port.Number = port.Port + break + } + } + } endps = []string{ipv6SafeAddrPort(svc.Spec.ClusterIP, ing.Spec.DefaultBackend.Service.Port.Number)} } else { endps = getIPAddressesFromEndpoints(podEndps) @@ -3030,6 +3038,14 @@ func (lbc *LoadBalancerController) createIngressEx(ing *networking.Ingress, vali } if svc != nil && !external && hasUseClusterIP { + if path.Backend.Service.Port.Number == 0 { + for _, port := range svc.Spec.Ports { + if port.Name == path.Backend.Service.Port.Name { + path.Backend.Service.Port.Number = port.Port + break + } + } + } endps = []string{ipv6SafeAddrPort(svc.Spec.ClusterIP, path.Backend.Service.Port.Number)} } else { endps = getIPAddressesFromEndpoints(podEndps)