From 1dd811d79b84a0dcf66539b4898b28594951cc75 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Wed, 19 Jun 2024 16:20:50 +0530 Subject: [PATCH 1/4] Add request redirect and mirror filters to config deployer --- .../ballerina/APIClient.bal | 203 +++++++++++++----- .../ballerina/modules/model/HttpRoute.bal | 4 +- .../ballerina/resources/apk-conf-schema.yaml | 33 ++- .../ballerina/types.bal | 97 +++++---- .../config-deployer/conf/apk-schema.json | 46 +++- 5 files changed, 290 insertions(+), 93 deletions(-) diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index 0a6d3d8a5..7c919c728 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -787,14 +787,17 @@ public class APIClient { return e909022("Provided Type currently not supported for GraphQL APIs.", error("Provided Type currently not supported for GraphQL APIs.")); } } else if apkConf.'type == API_TYPE_REST { - { - model:HTTPRouteRule httpRouteRule = { - matches: self.retrieveHTTPMatches(apkConf, operation, organization), - backendRefs: self.retrieveGeneratedBackend(apkConf, endpointToUse, endpointType), - filters: self.generateFilters(apiArtifact, apkConf, endpointToUse, operation, endpointType, organization) - }; - return httpRouteRule; + model:HTTPRouteFilter[] filters = []; + boolean hasRedirectPolicy = false; + [filters, hasRedirectPolicy] = self.generateFilters(apiArtifact, apkConf, endpointToUse, operation, endpointType, organization); + model:HTTPRouteRule httpRouteRule = { + matches: self.retrieveHTTPMatches(apkConf, operation, organization), + filters: filters + }; + if !hasRedirectPolicy { + httpRouteRule.backendRefs = self.retrieveGeneratedBackend(apkConf, endpointToUse, endpointType); } + return httpRouteRule; } else { return e909018("Invalid API Type specified"); } @@ -808,19 +811,9 @@ public class APIClient { } } - private isolated function generateFilters(model:APIArtifact apiArtifact, APKConf apkConf, model:Endpoint endpoint, APKOperations operation, string endpointType, commons:Organization organization) returns model:HTTPRouteFilter[] { + private isolated function generateFilters(model:APIArtifact apiArtifact, APKConf apkConf, model:Endpoint endpoint, APKOperations operation, string endpointType, commons:Organization organization) returns [model:HTTPRouteFilter[], boolean] { model:HTTPRouteFilter[] routeFilters = []; - string generatedPath = self.generatePrefixMatch(endpoint, operation); - model:HTTPRouteFilter replacePathFilter = { - 'type: "URLRewrite", - urlRewrite: { - path: { - 'type: "ReplaceFullPath", - replaceFullPath: generatedPath - } - } - }; - routeFilters.push(replacePathFilter); + boolean hasRedirectPolicy = false; APIOperationPolicies? operationPoliciesToUse = (); APIOperationPolicies? operationPolicies = apkConf.apiPolicies; if (operationPolicies is APIOperationPolicies && operationPolicies != {}) { @@ -833,61 +826,151 @@ public class APIClient { if operationPoliciesToUse is APIOperationPolicies { APKOperationPolicy[]? requestPolicies = operationPoliciesToUse.request; APKOperationPolicy[]? responsePolicies = operationPoliciesToUse.response; - if requestPolicies is APKOperationPolicy[] && requestPolicies.length() > 0 { - model:HTTPRouteFilter headerModifierFilter = {'type: "RequestHeaderModifier"}; - headerModifierFilter.requestHeaderModifier = self.extractHttpHeaderFilterData(requestPolicies, organization); - routeFilters.push(headerModifierFilter); + model:HTTPRouteFilter[] requestHttpRouteFilters = []; + [requestHttpRouteFilters, hasRedirectPolicy] = self.extractHttpRouteFilter(apiArtifact, apkConf, operation, endpoint, requestPolicies, organization, true); + routeFilters.push(...requestHttpRouteFilters); } if responsePolicies is APKOperationPolicy[] && responsePolicies.length() > 0 { - model:HTTPRouteFilter headerModifierFilter = {'type: "ResponseHeaderModifier"}; - headerModifierFilter.responseHeaderModifier = self.extractHttpHeaderFilterData(responsePolicies, organization); - routeFilters.push(headerModifierFilter); + model:HTTPRouteFilter[] responseHttpRouteFilters = []; + [responseHttpRouteFilters, _] = self.extractHttpRouteFilter(apiArtifact, apkConf, operation, endpoint, responsePolicies, organization, false); + routeFilters.push(...responseHttpRouteFilters); } } - return routeFilters; + + if !hasRedirectPolicy { + string generatedPath = self.generatePrefixMatch(endpoint, operation); + model:HTTPRouteFilter replacePathFilter = { + 'type: "URLRewrite", + urlRewrite: { + path: { + 'type: "ReplaceFullPath", + replaceFullPath: generatedPath + } + } + }; + routeFilters.push(replacePathFilter); + } + return [routeFilters, hasRedirectPolicy]; } - isolated function extractHttpHeaderFilterData(APKOperationPolicy[] operationPolicy, commons:Organization organization) returns model:HTTPHeaderFilter { - model:HTTPHeader[] addPolicies = []; - model:HTTPHeader[] setPolicies = []; - string[] removePolicies = []; - foreach APKOperationPolicy policy in operationPolicy { + isolated function extractHttpRouteFilter(model:APIArtifact apiArtifact, APKConf apkConf, APKOperations apiOperation, model:Endpoint endpoint, APKOperationPolicy[] operationPolicies, commons:Organization organization, boolean isRequest) returns [model:HTTPRouteFilter[], boolean] { + model:HTTPRouteFilter[] httpRouteFilters = []; + model:HTTPHeader[] addHeaders = []; + model:HTTPHeader[] setHeaders = []; + string[] removeHeaders = []; + boolean hasRedirectPolicy = false; + model:HTTPRouteFilter headerModifierFilter = {'type: "RequestHeaderModifier"}; + if !isRequest { + headerModifierFilter.'type = "ResponseHeaderModifier"; + } + foreach APKOperationPolicy policy in operationPolicies { if policy is HeaderModifierPolicy { HeaderModifierPolicyParameters policyParameters = policy.parameters; match policy.policyName { AddHeaders => { - ModifierHeader[] addHeaders = policyParameters.headers; - foreach ModifierHeader header in addHeaders { - addPolicies.push(header); + ModifierHeader[] headers = policyParameters.headers; + foreach ModifierHeader header in headers { + headers.push(header); } } SetHeaders => { - ModifierHeader[] setHeaders = policyParameters.headers; - foreach ModifierHeader header in setHeaders { - setPolicies.push(header); + ModifierHeader[] headers = policyParameters.headers; + foreach ModifierHeader header in headers { + headers.push(header); } } RemoveHeaders => { - string[] removeHeaders = policyParameters.headers; - foreach string header in removeHeaders { - removePolicies.push(header); + string[] headers = policyParameters.headers; + foreach string header in headers { + headers.push(header); } } } + } else if policy is RequestMirrorPolicy { + RequestMirrorPolicyParameters policyParameters = policy.parameters; + string[] urls = policyParameters.urls; + foreach string url in urls { + model:HTTPRouteFilter mirrorFilter = {'type: "RequestMirror"}; + if !isRequest { + log:printError("Mirror filter cannot be appended as a response policy."); + } + string host = self.getHost(url); + int|error port = self.getPort(url); + if port is int { + model:Backend backendService = { + metadata: { + name: self.getBackendServiceUid(apkConf, apiOperation, "", organization), + labels: self.getLabels(apkConf, organization) + }, + spec: { + services: [ + { + host: host, + port: port + } + ], + basePath: getPath(url), + protocol: self.getProtocol(url) + } + }; + apiArtifact.backendServices[backendService.metadata.name] = backendService; + model:Endpoint mirrorEndpoint = { + url: url, + name: backendService.metadata.name + }; + model:BackendRef backendRef = self.retrieveGeneratedBackend(apkConf, mirrorEndpoint, "")[0]; + mirrorFilter.requestMirror = { + backendRef: { + name: backendRef.name, + namespace: backendRef.namespace, + group: backendRef.group, + kind: backendRef.kind, + port: backendRef.port + } + }; + } + httpRouteFilters.push(mirrorFilter); + } + } else if policy is RequestRedirectPolicy { + hasRedirectPolicy = true; + if !isRequest { + log:printError("Redirect filter cannot be appended as a response policy."); + } + RequestRedirectPolicyParameters policyParameters = policy.parameters; + string url = policyParameters.url; + int statusCode = policyParameters.statusCode; + model:HTTPRouteFilter redirectFilter = {'type: "RequestRedirect"}; + int|error port = self.getPort(url); + if port is int { + redirectFilter.requestRedirect = { + hostname: self.getHost(url), + scheme: self.getProtocol(url), + statusCode: statusCode, + path: { + 'type: "ReplaceFullPath", + replaceFullPath: self.getPath(url) + } + }; + } + httpRouteFilters.push(redirectFilter); } } - model:HTTPHeaderFilter headerModifier = {}; - if addPolicies != [] { - headerModifier.add = addPolicies; + + if addHeaders != [] { + headerModifierFilter.requestHeaderModifier.add = addHeaders; + } + if setHeaders != [] { + headerModifierFilter.requestHeaderModifier.set = setHeaders; } - if setPolicies != [] { - headerModifier.set = setPolicies; + if removeHeaders != [] { + headerModifierFilter.requestHeaderModifier.remove = removeHeaders; } - if removePolicies != [] { - headerModifier.remove = removePolicies; + if addHeaders.length() > 0 || setHeaders.length() > 0 || removeHeaders.length() > 0 { + httpRouteFilters.push(headerModifierFilter); } - return headerModifier; + + return [httpRouteFilters, hasRedirectPolicy]; } isolated function generatePrefixMatch(model:Endpoint endpoint, APKOperations operation) returns string { @@ -923,6 +1006,23 @@ public class APIClient { return generatedPath; } + isolated function getPath(string url) returns string { + string host = ""; + if url.startsWith("https://") { + host = url.substring(8, url.length()); + } else if url.startsWith("http://") { + host = url.substring(7, url.length()); + } else { + return ""; + } + int? indexOfSlash = host.indexOf("/", 0); + if indexOfSlash is int { + return host.substring(indexOfSlash); + } else { + return ""; + } + } + public isolated function retrievePathPrefix(string basePath, string 'version, string operation, commons:Organization organization) returns string { string[] splitValues = regex:split(operation, "/"); string generatedPath = ""; @@ -980,7 +1080,8 @@ public class APIClient { } private isolated function retrieveGQLRouteMatch(APKOperations apiOperation) returns model:GQLRouteMatch|error { - model:GQLType? routeMatch = model:getGQLRouteMatch(apiOperation.verb); + model:GQLType + ? routeMatch = model:getGQLRouteMatch(apiOperation.verb); if routeMatch is model:GQLType { return {'type: routeMatch, path: apiOperation.target}; } else { @@ -1273,7 +1374,7 @@ public class APIClient { model:BackendJWT backendJwt = self.retrieveBackendJWTPolicy(apkConf, apiArtifact, backendJWTPolicy, operations, organization); apiArtifact.backendJwt = backendJwt; policyReferences.push({name: backendJwt.metadata.name}); - } else if policyName != AddHeaders && policyName != SetHeaders && policyName != RemoveHeaders { + } else if policyName != AddHeaders && policyName != SetHeaders && policyName != RemoveHeaders && policyName != RequestMirror && policyName != RequestRedirect { return e909052(error("Incorrect API Policy name provided.")); } } diff --git a/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal b/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal index 38ff5974c..382f8540f 100644 --- a/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal +++ b/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal @@ -38,7 +38,6 @@ public type HTTPQueryParamMatch record { }; public type HTTPRouteMatch record { - HTTPPathMatch path?; HTTPHeaderMatch headers?; HTTPQueryParamMatch queryParams?; @@ -79,14 +78,13 @@ public type HTTPRequestRedirectFilter record { string scheme?; string hostname?; HTTPPathModifier path?; - string port?; + int port?; int statusCode?; }; public type HTTPURLRewriteFilter record { string hostname?; HTTPPathModifier path?; - }; public type LocalObjectReference record { diff --git a/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml b/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml index 1a8814ad9..2adc6185e 100644 --- a/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml +++ b/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml @@ -257,6 +257,8 @@ components: - $ref: "#/components/schemas/InterceptorPolicy" - $ref: "#/components/schemas/BackendJWTPolicy" - $ref: "#/components/schemas/HeaderModifierPolicy" + - $ref: "#/components/schemas/RequestMirrorPolicy" + - $ref: "#/components/schemas/RequestRedirectPolicy" discriminator: propertyName: "policyName" mapping: @@ -264,7 +266,9 @@ components: Interceptor: "#/components/schemas/InterceptorPolicy" AddHeaders: "#/components/schemas/HeaderModifierPolicy" SetHeaders: "#/components/schemas/HeaderModifierPolicy" - RemoveHeadersHeaders: "#/components/schemas/HeaderModifierPolicy" + RemoveHeaders: "#/components/schemas/HeaderModifierPolicy" + RequestMirror: "#/components/schemas/RequestMirrorPolicy" + RequstRedirect: "#/components/schemas/RequestRedirectPolicy" BaseOperationPolicy: title: API Operation Policy required: @@ -521,6 +525,33 @@ components: type: string value: type: string + RequestMirrorPolicy: + title: Request Mirror Parameters + type: object + properties: + urls: + type: array + items: + - type: string + additionalProperties: false + RequestRedirectPolicy: + title: Request Redirect Parameters + type: object + properties: + url: + type: string + description: The URL to redirect the request to. + statusCode: + type: integer + description: The status code to show upon redirecting the request. + default: 301 + enum: + - 301 + - 302 + - 303 + - 307 + - 308 + additionalProperties: false CustomClaims: type: object required: diff --git a/runtime/config-deployer-service/ballerina/types.bal b/runtime/config-deployer-service/ballerina/types.bal index 1f59fd5d3..eecc9c4e9 100644 --- a/runtime/config-deployer-service/ballerina/types.bal +++ b/runtime/config-deployer-service/ballerina/types.bal @@ -127,7 +127,63 @@ public type APKOperations record { }; # Common type for operation policies. -public type APKOperationPolicy InterceptorPolicy|BackendJWTPolicy|HeaderModifierPolicy; +public type APKOperationPolicy InterceptorPolicy|BackendJWTPolicy|HeaderModifierPolicy|RequestMirrorPolicy|RequestRedirectPolicy; + +# Header modification configuration for an operation. +# +# + parameters - Contains header name and value of the header. +public type HeaderModifierPolicy record { + *BaseOperationPolicy; + HeaderModifierPolicyParameters parameters; +}; + +# Configuration for header modifiers as received from the apk-conf file. +# +# + headers - Headers to be added, set or removed. +public type HeaderModifierPolicyParameters record {| + ModifierHeader[]|string[] headers; +|}; + +# Configuration for headers. +# +# + name - The name of the header. +# + value - The value of the header. +public type ModifierHeader record {| + string name; + string value; +|}; + +# Request mirror configuration for an operation. +# +# + parameters - Contains the urls to request the mirror to. +public type RequestMirrorPolicy record { + *BaseOperationPolicy; + RequestMirrorPolicyParameters parameters; +}; + +# Configuration containing the different headers. +# +# + urls - The urls to mirror the filters to. +public type RequestMirrorPolicyParameters record {| + string[] urls; +|}; + +# Request redirect configuration for an operation. +# +# + parameters - Contains the url to redirect the request to. +public type RequestRedirectPolicy record { + *BaseOperationPolicy; + RequestRedirectPolicyParameters parameters; +}; + +# Configuration containing the different headers. +# +# + url - The url to redirect the filters to. +# + statusCode - The status code to be sent as response to the client. +public type RequestRedirectPolicyParameters record {| + string url; + int statusCode; +|}; # Configuration for API deployment using the apk-conf file. # @@ -319,7 +375,9 @@ public enum PolicyName { Interceptor, AddHeaders, SetHeaders, - RemoveHeaders + RemoveHeaders, + RequestMirror, + RequestRedirect } # Configuration for authentication types. @@ -331,14 +389,6 @@ public type Authentication record {| boolean enabled = true; |}; -# Header modification configuration for an operation. -# -# + parameters - Contains header name and value of the header. -public type HeaderModifierPolicy record { - *BaseOperationPolicy; - HeaderModifierPolicyParameters parameters; -}; - # Interceptor policy configuration for an operation. # # + parameters - Contains interceptor policy parameters @@ -387,33 +437,6 @@ public type APKConf record { CORSConfiguration corsConfiguration?; }; -# Configuration for header modifiers as received from the apk-conf file. -# -# + headers - Headers to be added, set or removed. -public type HeaderModifierPolicyParameters record {| - ModifierHeader[]|string[] headers; -|}; - -# Configuration containing the different headers. -# -# + addHeaders - Headers to be added. -# + setHeaders - Headers to be set. -# + removeHeaders - Headers to be removed. -public type HeaderModifierFilterParameters record {| - ModifierHeader[] addHeaders; - ModifierHeader[] setHeaders; - string[] removeHeaders; -|}; - -# Configuration for headers. -# -# + name - The name of the header. -# + value - The value of the header. -public type ModifierHeader record {| - string name; - string value; -|}; - # Configuration for Interceptor Policy parameters. # # + backendUrl - Backend URL of the interceptor service. diff --git a/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json b/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json index 09149e9f5..9f202e8d1 100644 --- a/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json +++ b/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json @@ -362,7 +362,9 @@ "RemoveHeaders", "SetHeaders", "Interceptor", - "BackendJwt" + "BackendJwt", + "RequestMirror", + "RequestRedirect" ] }, "policyVersion": { @@ -385,6 +387,12 @@ }, { "$ref": "#/schemas/HeaderModifierProperties" + }, + { + "$ref": "#/schemas/RequestMirrorProperties" + }, + { + "$ref": "#/schemas/RequestRedirectProperties" } ] } @@ -736,6 +744,42 @@ "type": "string" } }, + "RequestMirrorProperties": { + "title": "Request Mirror Parameters", + "type": "object", + "properties": { + "urls": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "RequestRedirectProperties": { + "title": "Request Redirect Parameters", + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to redirect the request to." + }, + "statusCode": { + "type": "integer", + "description": "The status code to show upon redirecting the request.", + "default": 301, + "enum": [ + 301, + 302, + 303, + 307, + 308 + ] + } + }, + "additionalProperties": false + }, "CustomClaims": { "type": "object", "required": [ From 81423abb65deec4c560e2a9dd70e509336f93e70 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Thu, 20 Jun 2024 15:23:05 +0530 Subject: [PATCH 2/4] Add tests for HTTPRoute filters --- .../oasparser/envoyconf/routes_configs.go | 14 ++-- .../envoyconf/routes_with_clusters.go | 5 +- .../ballerina/APIClient.bal | 66 +++++++++++++------ .../ballerina/resources/apk-conf-schema.yaml | 7 +- .../ballerina/types.bal | 2 +- .../config-deployer/conf/apk-schema.json | 7 +- .../utils/clients/SimpleHTTPClient.java | 5 ++ ...k-conf => header-modifier-filter.apk-conf} | 4 +- .../apk-confs/request-mirror-filter.apk-conf | 35 ++++++++++ .../request-redirect-filter.apk-conf | 34 ++++++++++ ...Filters.feature => HeaderModifier.feature} | 6 +- .../resources/tests/api/RequestMirror.feature | 21 ++++++ .../tests/api/RequestRedirect.feature | 21 ++++++ 13 files changed, 180 insertions(+), 47 deletions(-) rename test/cucumber-tests/src/test/resources/artifacts/apk-confs/{request_and_response_filters.apk-conf => header-modifier-filter.apk-conf} (94%) create mode 100644 test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-mirror-filter.apk-conf create mode 100644 test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-redirect-filter.apk-conf rename test/cucumber-tests/src/test/resources/tests/api/{HTTPRouteFilters.feature => HeaderModifier.feature} (84%) create mode 100644 test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature diff --git a/adapter/internal/oasparser/envoyconf/routes_configs.go b/adapter/internal/oasparser/envoyconf/routes_configs.go index dbafbec47..585ab47ba 100644 --- a/adapter/internal/oasparser/envoyconf/routes_configs.go +++ b/adapter/internal/oasparser/envoyconf/routes_configs.go @@ -128,7 +128,7 @@ func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, rate return action } -func generateRequestRedirectRoute(route string, policyParams interface{}) (action *routev3.Route_Redirect) { +func generateRequestRedirectRoute(route string, policyParams interface{}) (*routev3.Route_Redirect, error) { policyParameters, _ := policyParams.(map[string]interface{}) scheme, _ := policyParameters[constants.RedirectScheme].(string) hostname, _ := policyParameters[constants.RedirectHostname].(string) @@ -137,10 +137,10 @@ func generateRequestRedirectRoute(route string, policyParams interface{}) (actio replaceFullPath, _ := policyParameters[constants.RedirectPath].(string) redirectActionStatusCode := mapStatusCodeToEnum(statusCode) if redirectActionStatusCode == -1 { - _ = fmt.Errorf("Invalid status code provided") + return nil, fmt.Errorf("Invalid status code provided") } - action = &routev3.Route_Redirect{ + action := &routev3.Route_Redirect{ Redirect: &routev3.RedirectAction{ SchemeRewriteSpecifier: &routev3.RedirectAction_HttpsRedirect{ HttpsRedirect: scheme == "https", @@ -153,7 +153,7 @@ func generateRequestRedirectRoute(route string, policyParams interface{}) (actio ResponseCode: routev3.RedirectAction_RedirectResponseCode(redirectActionStatusCode), }, } - return action + return action, nil } func mapStatusCodeToEnum(statusCode int) int { @@ -162,12 +162,6 @@ func mapStatusCodeToEnum(statusCode int) int { return 0 case 302: return 1 - case 303: - return 2 - case 307: - return 3 - case 308: - return 4 default: return -1 } diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 869246b19..17ab9f110 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -998,7 +998,10 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error case constants.ActionRedirectRequest: logger.LoggerOasparser.Debugf("Adding %s policy to request flow for %s %s", constants.ActionRedirectRequest, resourcePath, operation.GetMethod()) - requestRedirectAction = generateRequestRedirectRoute(resourcePath, requestPolicy.Parameters) + requestRedirectAction, err = generateRequestRedirectRoute(resourcePath, requestPolicy.Parameters) + if err != nil { + return nil, err + } } } diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index 7c919c728..237a433b3 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -860,10 +860,7 @@ public class APIClient { model:HTTPHeader[] setHeaders = []; string[] removeHeaders = []; boolean hasRedirectPolicy = false; - model:HTTPRouteFilter headerModifierFilter = {'type: "RequestHeaderModifier"}; - if !isRequest { - headerModifierFilter.'type = "ResponseHeaderModifier"; - } + foreach APKOperationPolicy policy in operationPolicies { if policy is HeaderModifierPolicy { HeaderModifierPolicyParameters policyParameters = policy.parameters; @@ -871,19 +868,19 @@ public class APIClient { AddHeaders => { ModifierHeader[] headers = policyParameters.headers; foreach ModifierHeader header in headers { - headers.push(header); + addHeaders.push(header); } } SetHeaders => { ModifierHeader[] headers = policyParameters.headers; foreach ModifierHeader header in headers { - headers.push(header); + setHeaders.push(header); } } RemoveHeaders => { string[] headers = policyParameters.headers; foreach string header in headers { - headers.push(header); + removeHeaders.push(header); } } } @@ -939,35 +936,64 @@ public class APIClient { } RequestRedirectPolicyParameters policyParameters = policy.parameters; string url = policyParameters.url; - int statusCode = policyParameters.statusCode; model:HTTPRouteFilter redirectFilter = {'type: "RequestRedirect"}; int|error port = self.getPort(url); + if port is int { redirectFilter.requestRedirect = { hostname: self.getHost(url), scheme: self.getProtocol(url), - statusCode: statusCode, path: { 'type: "ReplaceFullPath", replaceFullPath: self.getPath(url) } }; + if policyParameters.statusCode is int { + int statusCode = policyParameters.statusCode; + redirectFilter.requestRedirect.statusCode = statusCode; + } } + httpRouteFilters.push(redirectFilter); } } + if isRequest { + model:HTTPHeaderFilter requestHeaderModifier = {}; + if addHeaders != [] { + requestHeaderModifier.add = addHeaders; + } + if setHeaders != [] { + requestHeaderModifier.set = setHeaders; + } + if removeHeaders != [] { + requestHeaderModifier.remove = removeHeaders; + } - if addHeaders != [] { - headerModifierFilter.requestHeaderModifier.add = addHeaders; - } - if setHeaders != [] { - headerModifierFilter.requestHeaderModifier.set = setHeaders; - } - if removeHeaders != [] { - headerModifierFilter.requestHeaderModifier.remove = removeHeaders; - } - if addHeaders.length() > 0 || setHeaders.length() > 0 || removeHeaders.length() > 0 { - httpRouteFilters.push(headerModifierFilter); + if addHeaders.length() > 0 || setHeaders.length() > 0 || removeHeaders.length() > 0 { + model:HTTPRouteFilter headerModifierFilter = { + 'type: "RequestHeaderModifier", + requestHeaderModifier: requestHeaderModifier + }; + httpRouteFilters.push(headerModifierFilter); + } + } else { + model:HTTPHeaderFilter responseHeaderModifier = {}; + if addHeaders != [] { + responseHeaderModifier.add = addHeaders; + } + if setHeaders != [] { + responseHeaderModifier.set = setHeaders; + } + if removeHeaders != [] { + responseHeaderModifier.remove = removeHeaders; + } + if addHeaders.length() > 0 || setHeaders.length() > 0 || removeHeaders.length() > 0 { + model:HTTPRouteFilter headerModifierFilter = { + 'type: "ResponseHeaderModifier", + responseHeaderModifier: responseHeaderModifier + }; + httpRouteFilters.push(headerModifierFilter); + } } return [httpRouteFilters, hasRedirectPolicy]; diff --git a/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml b/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml index 2adc6185e..6285131a6 100644 --- a/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml +++ b/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml @@ -268,7 +268,7 @@ components: SetHeaders: "#/components/schemas/HeaderModifierPolicy" RemoveHeaders: "#/components/schemas/HeaderModifierPolicy" RequestMirror: "#/components/schemas/RequestMirrorPolicy" - RequstRedirect: "#/components/schemas/RequestRedirectPolicy" + RequestRedirect: "#/components/schemas/RequestRedirectPolicy" BaseOperationPolicy: title: API Operation Policy required: @@ -544,13 +544,10 @@ components: statusCode: type: integer description: The status code to show upon redirecting the request. - default: 301 + default: 302 enum: - 301 - 302 - - 303 - - 307 - - 308 additionalProperties: false CustomClaims: type: object diff --git a/runtime/config-deployer-service/ballerina/types.bal b/runtime/config-deployer-service/ballerina/types.bal index eecc9c4e9..edd5e4362 100644 --- a/runtime/config-deployer-service/ballerina/types.bal +++ b/runtime/config-deployer-service/ballerina/types.bal @@ -182,7 +182,7 @@ public type RequestRedirectPolicy record { # + statusCode - The status code to be sent as response to the client. public type RequestRedirectPolicyParameters record {| string url; - int statusCode; + int statusCode?; |}; # Configuration for API deployment using the apk-conf file. diff --git a/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json b/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json index 9f202e8d1..0ad4df6c2 100644 --- a/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json +++ b/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json @@ -768,13 +768,10 @@ "statusCode": { "type": "integer", "description": "The status code to show upon redirecting the request.", - "default": 301, + "default": 302, "enum": [ 301, - 302, - 303, - 307, - 308 + 302 ] } }, diff --git a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/utils/clients/SimpleHTTPClient.java b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/utils/clients/SimpleHTTPClient.java index d06cd0705..eb591864b 100644 --- a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/utils/clients/SimpleHTTPClient.java +++ b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/utils/clients/SimpleHTTPClient.java @@ -26,6 +26,7 @@ import javax.net.ssl.TrustManager; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; @@ -87,7 +88,11 @@ public SimpleHTTPClient() throws NoSuchAlgorithmException, KeyStoreException, Ke .build(); final SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslcontext); + RequestConfig requestConfig = RequestConfig.custom() + .setRedirectsEnabled(false) // Disable redirects + .build(); this.client = HttpClients.custom() + .setDefaultRequestConfig(requestConfig) .setSSLSocketFactory(csf) .evictExpiredConnections() .setMaxConnPerRoute(100) diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request_and_response_filters.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/header-modifier-filter.apk-conf similarity index 94% rename from test/cucumber-tests/src/test/resources/artifacts/apk-confs/request_and_response_filters.apk-conf rename to test/cucumber-tests/src/test/resources/artifacts/apk-confs/header-modifier-filter.apk-conf index a41f67efc..a9d08d90f 100644 --- a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request_and_response_filters.apk-conf +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/header-modifier-filter.apk-conf @@ -1,7 +1,7 @@ --- -id: "api-with-request-and-response-filters" +id: "api-with-header-modifier-filters" name: "EmployeeServiceAPI" -basePath: "/request-and-response-filters" +basePath: "/header-modifier-filters" version: "3.14" type: "REST" defaultVersion: false diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-mirror-filter.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-mirror-filter.apk-conf new file mode 100644 index 000000000..46523900a --- /dev/null +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-mirror-filter.apk-conf @@ -0,0 +1,35 @@ +--- +id: "api-with-request-mirror-filter" +name: "EmployeeServiceAPI" +basePath: "/request-mirror-filter" +version: "3.14" +type: "REST" +defaultVersion: false +endpointConfigurations: + production: + endpoint: "http://backend:80/anything" +operations: + - target: "/employee" + verb: "GET" + secured: false + scopes: [] + operationPolicies: + request: + - policyName: RequestMirror + policyVersion: v1 + parameters: + urls: + - "http://backend:80/anything" + - "http://backend:80/anything" + - target: "/employee" + verb: "POST" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "PUT" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "DELETE" + secured: true + scopes: [] diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-redirect-filter.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-redirect-filter.apk-conf new file mode 100644 index 000000000..c972aeea2 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-redirect-filter.apk-conf @@ -0,0 +1,34 @@ +--- +id: "api-with-request-redirect-filter" +name: "EmployeeServiceAPI" +basePath: "/request-redirect-filter" +version: "3.14" +type: "REST" +defaultVersion: false +endpointConfigurations: + production: + endpoint: "http://backend:80/anything" +operations: + - target: "/employee" + verb: "GET" + secured: false + scopes: [] + operationPolicies: + request: + - policyName: RequestRedirect + policyVersion: v1 + parameters: + url: "http://backend:80/anything" + statusCode: 301 + - target: "/employee" + verb: "POST" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "PUT" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "DELETE" + secured: true + scopes: [] diff --git a/test/cucumber-tests/src/test/resources/tests/api/HTTPRouteFilters.feature b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature similarity index 84% rename from test/cucumber-tests/src/test/resources/tests/api/HTTPRouteFilters.feature rename to test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature index e8138c5b5..f6482ceca 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/HTTPRouteFilters.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature @@ -2,13 +2,13 @@ Feature: Test HTTPRoute Filter Header Modifier functionality Scenario: Test request and response header modification functionality Given The system is ready And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/request_and_response_filters.apk-conf" + When I use the APK Conf file "artifacts/apk-confs/header-modifier-filter.apk-conf" And the definition file "artifacts/definitions/employees_api.json" And make the API deployment request Then the response status code should be 200 Then I set headers | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/request-and-response-filters/3.14/employee/" with body "" + And I send "GET" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" And I eventually receive 200 response code, not accepting | 401 | And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" @@ -22,7 +22,7 @@ Feature: Test HTTPRoute Filter Header Modifier functionality Scenario: Undeploy the API Given The system is ready And I have a valid subscription - When I undeploy the API whose ID is "api-with-request-and-response-filters" + When I undeploy the API whose ID is "api-with-header-modifier-filters" Then the response status code should be 202 \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature new file mode 100644 index 000000000..0c4ed706f --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature @@ -0,0 +1,21 @@ +Feature: Test HTTPRoute Filter Request Mirror functionality + Scenario: Test request mirror functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/request-mirror-filter.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/request-mirror-filter/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-request-mirror-filter" + Then the response status code should be 202 + + \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature new file mode 100644 index 000000000..85b393f90 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature @@ -0,0 +1,21 @@ +Feature: Test HTTPRoute Filter Request Redirect functionality + Scenario: Test request redirect functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/request-redirect-filter.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-request-redirect-filter" + Then the response status code should be 202 + + \ No newline at end of file From e4280a44c20e2f3ee9674d1a3d3567396cc4907b Mon Sep 17 00:00:00 2001 From: sgayangi Date: Fri, 5 Jul 2024 14:19:35 +0530 Subject: [PATCH 3/4] Add config deployer changes from code review --- .../ballerina/APIClient.bal | 53 ++++++----- .../ballerina/resources/apk-conf-schema.yaml | 49 ++++++----- .../ballerina/tests/resources/apk-schema.json | 6 +- .../ballerina/types.bal | 34 ++++---- .../config-deployer/conf/apk-schema.json | 87 +++++++++++++------ .../api-level-header.apk-conf | 58 +++++++++++++ .../api-level-mirror.apk-conf | 34 ++++++++ .../api-level-redirect.apk-conf | 34 ++++++++ .../header-modifier-filter.apk-conf | 38 ++++---- .../request-mirror-filter.apk-conf | 0 .../request-redirect-filter.apk-conf | 0 .../tests/api/HeaderModifier.feature | 58 ++++++++++++- .../resources/tests/api/RequestMirror.feature | 2 +- .../tests/api/RequestRedirect.feature | 32 ++++++- 14 files changed, 364 insertions(+), 121 deletions(-) create mode 100644 test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-header.apk-conf create mode 100644 test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-mirror.apk-conf create mode 100644 test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf rename test/cucumber-tests/src/test/resources/artifacts/apk-confs/{ => httproute-filters}/header-modifier-filter.apk-conf (56%) rename test/cucumber-tests/src/test/resources/artifacts/apk-confs/{ => httproute-filters}/request-mirror-filter.apk-conf (100%) rename test/cucumber-tests/src/test/resources/artifacts/apk-confs/{ => httproute-filters}/request-redirect-filter.apk-conf (100%) diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index 237a433b3..4331101e8 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -185,14 +185,14 @@ public class APIClient { } isolated function isPolicyEmpty(APIOperationPolicies? policies) returns boolean { if policies is APIOperationPolicies { - APKOperationPolicy[]? request = policies.request; - if request is APKOperationPolicy[] { + APKRequestOperationPolicy[]? request = policies.request; + if request is APKRequestOperationPolicy[] { if (request.length() > 0) { return false; } } - APKOperationPolicy[]? response = policies.response; - if response is APKOperationPolicy[] { + APKResponseOperationPolicy[]? response = policies.response; + if response is APKResponseOperationPolicy[] { if (response.length() > 0) { return false; } @@ -694,7 +694,7 @@ public class APIClient { private isolated function generateAPIPolicyAndBackendCR(model:APIArtifact apiArtifact, APKConf apkConf, APKOperations? operations, APIOperationPolicies? policies, commons:Organization organization, string targetRefName) returns model:APIPolicy?|error { model:APIPolicyData defaultSpecData = {}; - APKOperationPolicy[]? request = policies?.request; + APKRequestOperationPolicy[]? request = policies?.request; any[] requestPolicy = check self.retrieveAPIPolicyDetails(apiArtifact, apkConf, operations, organization, request, "request"); foreach any item in requestPolicy { if item is model:InterceptorReference { @@ -703,7 +703,7 @@ public class APIClient { defaultSpecData.backendJwtPolicy = item; } } - APKOperationPolicy[]? response = policies?.response; + APKResponseOperationPolicy[]? response = policies?.response; any[] responseInterceptor = check self.retrieveAPIPolicyDetails(apiArtifact, apkConf, operations, organization, response, "response"); foreach any item in responseInterceptor { if item is model:InterceptorReference { @@ -817,21 +817,21 @@ public class APIClient { APIOperationPolicies? operationPoliciesToUse = (); APIOperationPolicies? operationPolicies = apkConf.apiPolicies; if (operationPolicies is APIOperationPolicies && operationPolicies != {}) { - if operationPolicies.request is APKOperationPolicy[] || operationPolicies.response is APKOperationPolicy[] { + if operationPolicies.request is APKRequestOperationPolicy[] || operationPolicies.response is APKResponseOperationPolicy[] { operationPoliciesToUse = apkConf.apiPolicies; } } else { operationPoliciesToUse = operation.operationPolicies; } if operationPoliciesToUse is APIOperationPolicies { - APKOperationPolicy[]? requestPolicies = operationPoliciesToUse.request; - APKOperationPolicy[]? responsePolicies = operationPoliciesToUse.response; - if requestPolicies is APKOperationPolicy[] && requestPolicies.length() > 0 { + APKRequestOperationPolicy[]? requestPolicies = operationPoliciesToUse.request; + APKResponseOperationPolicy[]? responsePolicies = operationPoliciesToUse.response; + if requestPolicies is APKRequestOperationPolicy[] && requestPolicies.length() > 0 { model:HTTPRouteFilter[] requestHttpRouteFilters = []; [requestHttpRouteFilters, hasRedirectPolicy] = self.extractHttpRouteFilter(apiArtifact, apkConf, operation, endpoint, requestPolicies, organization, true); routeFilters.push(...requestHttpRouteFilters); } - if responsePolicies is APKOperationPolicy[] && responsePolicies.length() > 0 { + if responsePolicies is APKResponseOperationPolicy[] && responsePolicies.length() > 0 { model:HTTPRouteFilter[] responseHttpRouteFilters = []; [responseHttpRouteFilters, _] = self.extractHttpRouteFilter(apiArtifact, apkConf, operation, endpoint, responsePolicies, organization, false); routeFilters.push(...responseHttpRouteFilters); @@ -865,23 +865,22 @@ public class APIClient { if policy is HeaderModifierPolicy { HeaderModifierPolicyParameters policyParameters = policy.parameters; match policy.policyName { - AddHeaders => { - ModifierHeader[] headers = policyParameters.headers; - foreach ModifierHeader header in headers { - addHeaders.push(header); - } + AddHeader => { + model:HTTPHeader addHeader = { + name: policyParameters.headerName, + value: policyParameters.headerValue + }; + addHeaders.push(addHeader); } - SetHeaders => { - ModifierHeader[] headers = policyParameters.headers; - foreach ModifierHeader header in headers { - setHeaders.push(header); - } + SetHeader => { + model:HTTPHeader setHeader = { + name: policyParameters.headerName, + value: policyParameters.headerValue + }; + setHeaders.push(setHeader); } - RemoveHeaders => { - string[] headers = policyParameters.headers; - foreach string header in headers { - removeHeaders.push(header); - } + RemoveHeader => { + removeHeaders.push(policyParameters.headerName); } } } else if policy is RequestMirrorPolicy { @@ -1400,7 +1399,7 @@ public class APIClient { model:BackendJWT backendJwt = self.retrieveBackendJWTPolicy(apkConf, apiArtifact, backendJWTPolicy, operations, organization); apiArtifact.backendJwt = backendJwt; policyReferences.push({name: backendJwt.metadata.name}); - } else if policyName != AddHeaders && policyName != SetHeaders && policyName != RemoveHeaders && policyName != RequestMirror && policyName != RequestRedirect { + } else if policyName != AddHeader && policyName != SetHeader && policyName != RemoveHeader && policyName != RequestMirror && policyName != RequestRedirect { return e909052(error("Incorrect API Policy name provided.")); } } diff --git a/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml b/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml index 6285131a6..b32f902c1 100644 --- a/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml +++ b/runtime/config-deployer-service/ballerina/resources/apk-conf-schema.yaml @@ -245,13 +245,13 @@ components: request: type: array items: - $ref: "#/components/schemas/APKOperationPolicy" + $ref: "#/components/schemas/APKRequestOperationPolicy" response: type: array items: - $ref: "#/components/schemas/APKOperationPolicy" + $ref: "#/components/schemas/APKResponseOperationPolicy" additionalProperties: false - APKOperationPolicy: + APKRequestOperationPolicy: title: API Operation Policy oneOf: - $ref: "#/components/schemas/InterceptorPolicy" @@ -264,11 +264,25 @@ components: mapping: BackendJwt: "#/components/schemas/BackendJWTPolicy" Interceptor: "#/components/schemas/InterceptorPolicy" - AddHeaders: "#/components/schemas/HeaderModifierPolicy" - SetHeaders: "#/components/schemas/HeaderModifierPolicy" - RemoveHeaders: "#/components/schemas/HeaderModifierPolicy" + AddHeader: "#/components/schemas/HeaderModifierPolicy" + SetHeader: "#/components/schemas/HeaderModifierPolicy" + RemoveHeader: "#/components/schemas/HeaderModifierPolicy" RequestMirror: "#/components/schemas/RequestMirrorPolicy" RequestRedirect: "#/components/schemas/RequestRedirectPolicy" + APKResponseOperationPolicy: + title: API Operation Policy + oneOf: + - $ref: "#/components/schemas/InterceptorPolicy" + - $ref: "#/components/schemas/BackendJWTPolicy" + - $ref: "#/components/schemas/HeaderModifierPolicy" + discriminator: + propertyName: "policyName" + mapping: + BackendJwt: "#/components/schemas/BackendJWTPolicy" + Interceptor: "#/components/schemas/InterceptorPolicy" + AddHeader: "#/components/schemas/HeaderModifierPolicy" + SetHeader: "#/components/schemas/HeaderModifierPolicy" + RemoveHeader: "#/components/schemas/HeaderModifierPolicy" BaseOperationPolicy: title: API Operation Policy required: @@ -278,9 +292,9 @@ components: policyName: type: string enum: - - AddHeaders - - RemoveHeaders - - SetHeaders + - AddHeader + - RemoveHeader + - SetHeader - Interceptor - BackendJwt policyVersion: @@ -511,20 +525,13 @@ components: title: Header Modifier Parameters type: object properties: - headers: - type: array - items: - oneOf: - - $ref: "#/components/schemas/Header" - - type: string - additionalProperties: false - Header: - type: object - properties: - name: + headerName: type: string - value: + headerValue: type: string + required: + - headerName + additionalProperties: false RequestMirrorPolicy: title: Request Mirror Parameters type: object diff --git a/runtime/config-deployer-service/ballerina/tests/resources/apk-schema.json b/runtime/config-deployer-service/ballerina/tests/resources/apk-schema.json index d7c57998f..5a65a7355 100644 --- a/runtime/config-deployer-service/ballerina/tests/resources/apk-schema.json +++ b/runtime/config-deployer-service/ballerina/tests/resources/apk-schema.json @@ -313,9 +313,9 @@ "type": "string", "description": "The name of the operation policy.", "enum": [ - "AddHeaders", - "RemoveHeaders", - "SetHeaders", + "AddHeader", + "RemoveHeader", + "SetHeader", "Interceptor", "BackendJwt" ] diff --git a/runtime/config-deployer-service/ballerina/types.bal b/runtime/config-deployer-service/ballerina/types.bal index edd5e4362..ab89f24f5 100644 --- a/runtime/config-deployer-service/ballerina/types.bal +++ b/runtime/config-deployer-service/ballerina/types.bal @@ -126,8 +126,13 @@ public type APKOperations record { string[] scopes?; }; -# Common type for operation policies. -public type APKOperationPolicy InterceptorPolicy|BackendJWTPolicy|HeaderModifierPolicy|RequestMirrorPolicy|RequestRedirectPolicy; +public type APKOperationPolicy APKRequestOperationPolicy|APKResponseOperationPolicy; + +# Common type for request operation policies. +public type APKRequestOperationPolicy InterceptorPolicy|BackendJWTPolicy|HeaderModifierPolicy|RequestMirrorPolicy|RequestRedirectPolicy; + +# Common type for response operation policies. +public type APKResponseOperationPolicy InterceptorPolicy|BackendJWTPolicy|HeaderModifierPolicy; # Header modification configuration for an operation. # @@ -139,18 +144,11 @@ public type HeaderModifierPolicy record { # Configuration for header modifiers as received from the apk-conf file. # -# + headers - Headers to be added, set or removed. +# + headerName - Header name to be added, set or removed. +# + headerValue - Header value to be added, set or removed. public type HeaderModifierPolicyParameters record {| - ModifierHeader[]|string[] headers; -|}; - -# Configuration for headers. -# -# + name - The name of the header. -# + value - The value of the header. -public type ModifierHeader record {| - string name; - string value; + string headerName; + string headerValue?; |}; # Request mirror configuration for an operation. @@ -199,8 +197,8 @@ public type DeployApiBody record { # + request - List of policies to be applied on the request. # + response - List of policies to be applied on the response. public type APIOperationPolicies record { - APKOperationPolicy[] request?; - APKOperationPolicy[] response?; + APKRequestOperationPolicy[] request?; + APKResponseOperationPolicy[] response?; }; # Additional properties for APK configuration. @@ -373,9 +371,9 @@ public type BaseOperationPolicy record { public enum PolicyName { BackendJwt, Interceptor, - AddHeaders, - SetHeaders, - RemoveHeaders, + AddHeader, + SetHeader, + RemoveHeader, RequestMirror, RequestRedirect } diff --git a/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json b/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json index 0ad4df6c2..684bd4d88 100644 --- a/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json +++ b/runtime/config-deployer-service/docker/config-deployer/conf/apk-schema.json @@ -333,21 +333,21 @@ "request": { "type": "array", "items": { - "$ref": "#/schemas/APKOperationPolicy" + "$ref": "#/schemas/APKRequestOperationPolicy" }, "description": "Policies applied to request operations." }, "response": { "type": "array", "items": { - "$ref": "#/schemas/APKOperationPolicy" + "$ref": "#/schemas/APKResponseOperationPolicy" }, "description": "Policies applied to response operations." } }, "additionalProperties": false }, - "APKOperationPolicy": { + "APKRequestOperationPolicy": { "title": "API Operation Policy", "required": [ "policyName" @@ -358,9 +358,9 @@ "type": "string", "description": "The name of the operation policy.", "enum": [ - "AddHeaders", - "RemoveHeaders", - "SetHeaders", + "AddHeader", + "RemoveHeader", + "SetHeader", "Interceptor", "BackendJwt", "RequestMirror", @@ -399,6 +399,50 @@ }, "additionalProperties": false }, + "APKResponseOperationPolicy": { + "title": "API Operation Policy", + "required": [ + "policyName" + ], + "type": "object", + "properties": { + "policyName": { + "type": "string", + "description": "The name of the operation policy.", + "enum": [ + "AddHeader", + "RemoveHeader", + "SetHeader", + "Interceptor", + "BackendJwt" + ] + }, + "policyVersion": { + "type": "string", + "default": "v1", + "description": "The version of the operation policy." + }, + "policyId": { + "type": "string", + "description": "The ID of the operation policy." + }, + "parameters": { + "type": "object", + "oneOf": [ + { + "$ref": "#/schemas/InterceptorProperties" + }, + { + "$ref": "#/schemas/BackendJWTProperties" + }, + { + "$ref": "#/schemas/HeaderModifierProperties" + } + ] + } + }, + "additionalProperties": false + }, "RateLimit": { "title": "API Rate Limit Details", "type": "object", @@ -719,31 +763,20 @@ "title": "Header Modifier Parameters", "type": "object", "properties": { - "headers": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/schemas/Header" - }, - { - "type": "string" - } - ] - } + "headerName": { + "type": "string", + "description": "The name of the header." + }, + "headerValue": { + "type": "string", + "description": "The value of the header." } }, + "required": [ + "headerName" + ], "additionalProperties": false }, - "Header": { - "type": "object", - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, "RequestMirrorProperties": { "title": "Request Mirror Parameters", "type": "object", diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-header.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-header.apk-conf new file mode 100644 index 000000000..75067f2f6 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-header.apk-conf @@ -0,0 +1,58 @@ +--- +id: "api-with-header-modifier-filters" +name: "EmployeeServiceAPI" +basePath: "/header-modifier-filters" +version: "3.14" +type: "REST" +defaultVersion: false +endpointConfigurations: + production: + endpoint: "http://backend:80/anything" +operations: + - target: "/employee" + verb: "GET" + secured: false + scopes: [] + - target: "/employee" + verb: "POST" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "PUT" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "DELETE" + secured: true + scopes: [] +apiPolicies: + request: + - policyName: AddHeader + policyVersion: v1 + parameters: + headerName: "Test-Request-Header" + headerValue: "Test-Value" + - policyName: SetHeader + policyVersion: v1 + parameters: + headerName: "Set-Request-Header" + headerValue: "Test-Value" + - policyName: RemoveHeader + policyVersion: v1 + parameters: + headerName: "Authorization" + response: + - policyName: AddHeader + policyVersion: v1 + parameters: + headerName: "Test-Response-Header" + headerValue: "Test-Value" + - policyName: SetHeader + policyVersion: v1 + parameters: + headerName: "Set-Response-Header" + headerValue: "Test-Value" + - policyName: RemoveHeader + policyVersion: v1 + parameters: + headerName: "content-type" diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-mirror.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-mirror.apk-conf new file mode 100644 index 000000000..3555f72c8 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-mirror.apk-conf @@ -0,0 +1,34 @@ +--- +id: "api-with-request-mirror-filter" +name: "EmployeeServiceAPI" +basePath: "/request-mirror-filter" +version: "3.14" +type: "REST" +defaultVersion: false +endpointConfigurations: + production: + endpoint: "http://backend:80/anything" +operations: + - target: "/employee" + verb: "GET" + secured: false + scopes: [] + - target: "/employee" + verb: "POST" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "PUT" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "DELETE" + secured: true + scopes: [] +apiPolicies: + request: + - policyName: RequestMirror + policyVersion: v1 + parameters: + urls: + - "http://backend:80/anything" diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf new file mode 100644 index 000000000..2b2c4985f --- /dev/null +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf @@ -0,0 +1,34 @@ +--- +id: "api-with-request-redirect-filter" +name: "EmployeeServiceAPI" +basePath: "/request-redirect-filter" +version: "3.14" +type: "REST" +defaultVersion: false +endpointConfigurations: + production: + endpoint: "http://backend:80/anything" +operations: + - target: "/employee" + verb: "GET" + secured: false + scopes: [] + - target: "/employee" + verb: "POST" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "PUT" + secured: true + scopes: [] + - target: "/employee/{employeeId}" + verb: "DELETE" + secured: true + scopes: [] +apiPolicies: + request: + - policyName: RequestRedirect + policyVersion: v1 + parameters: + url: "http://backend:80/anything" + statusCode: 301 diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/header-modifier-filter.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/header-modifier-filter.apk-conf similarity index 56% rename from test/cucumber-tests/src/test/resources/artifacts/apk-confs/header-modifier-filter.apk-conf rename to test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/header-modifier-filter.apk-conf index a9d08d90f..e24842b9b 100644 --- a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/header-modifier-filter.apk-conf +++ b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/header-modifier-filter.apk-conf @@ -15,41 +15,35 @@ operations: scopes: [] operationPolicies: request: - - policyName: AddHeaders + - policyName: AddHeader policyVersion: v1 parameters: - headers: - - name: "Test-Request-Header" - value: "Test-Value" - - policyName: SetHeaders + headerName: "Test-Request-Header" + headerValue: "Test-Value" + - policyName: SetHeader policyVersion: v1 parameters: - headers: - - name: "Set-Request-Header" - value: "Test-Value" - - policyName: RemoveHeaders + headerName: "Set-Request-Header" + headerValue: "Test-Value" + - policyName: RemoveHeader policyVersion: v1 parameters: - headers: - - "Authorization" + headerName: "Authorization" response: - - policyName: AddHeaders + - policyName: AddHeader policyVersion: v1 parameters: - headers: - - name: "Test-Response-Header" - value: "Test-Value" - - policyName: SetHeaders + headerName: "Test-Response-Header" + headerValue: "Test-Value" + - policyName: SetHeader policyVersion: v1 parameters: - headers: - - name: "Set-Response-Header" - value: "Test-Value" - - policyName: RemoveHeaders + headerName: "Set-Response-Header" + headerValue: "Test-Value" + - policyName: RemoveHeader policyVersion: v1 parameters: - headers: - - "content-type" + headerName: "content-type" - target: "/employee" verb: "POST" secured: true diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-mirror-filter.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/request-mirror-filter.apk-conf similarity index 100% rename from test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-mirror-filter.apk-conf rename to test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/request-mirror-filter.apk-conf diff --git a/test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-redirect-filter.apk-conf b/test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/request-redirect-filter.apk-conf similarity index 100% rename from test/cucumber-tests/src/test/resources/artifacts/apk-confs/request-redirect-filter.apk-conf rename to test/cucumber-tests/src/test/resources/artifacts/apk-confs/httproute-filters/request-redirect-filter.apk-conf diff --git a/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature index f6482ceca..75a9ee070 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature @@ -2,7 +2,7 @@ Feature: Test HTTPRoute Filter Header Modifier functionality Scenario: Test request and response header modification functionality Given The system is ready And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/header-modifier-filter.apk-conf" + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/header-modifier-filter.apk-conf" And the definition file "artifacts/definitions/employees_api.json" And make the API deployment request Then the response status code should be 200 @@ -24,5 +24,61 @@ Feature: Test HTTPRoute Filter Header Modifier functionality And I have a valid subscription When I undeploy the API whose ID is "api-with-header-modifier-filters" Then the response status code should be 202 + + Scenario: Test request and response header modification functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/api-level-header.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + And I send "POST" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + And I send "PUT" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/1" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + And I send "DELETE" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/1" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-header-modifier-filters" + Then the response status code should be 202 \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature index 0c4ed706f..3dfe4f6af 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature @@ -2,7 +2,7 @@ Feature: Test HTTPRoute Filter Request Mirror functionality Scenario: Test request mirror functionality Given The system is ready And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/request-mirror-filter.apk-conf" + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/request-mirror-filter.apk-conf" And the definition file "artifacts/definitions/employees_api.json" And make the API deployment request Then the response status code should be 200 diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature index 85b393f90..28570f428 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature @@ -2,7 +2,7 @@ Feature: Test HTTPRoute Filter Request Redirect functionality Scenario: Test request redirect functionality Given The system is ready And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/request-redirect-filter.apk-conf" + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/request-redirect-filter.apk-conf" And the definition file "artifacts/definitions/employees_api.json" And make the API deployment request Then the response status code should be 200 @@ -11,6 +11,9 @@ Feature: Test HTTPRoute Filter Request Redirect functionality And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" And I eventually receive 301 response code, not accepting | 401 | + And I send "POST" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | Scenario: Undeploy the API Given The system is ready @@ -18,4 +21,31 @@ Feature: Test HTTPRoute Filter Request Redirect functionality When I undeploy the API whose ID is "api-with-request-redirect-filter" Then the response status code should be 202 + Scenario: Test request redirect functionality with API level redirect + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "POST" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "PUT" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/1" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "DELETE" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/1" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-request-redirect-filter" + Then the response status code should be 202 \ No newline at end of file From 7edbae02c9103a45429d670df9c7a078d29875c7 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Thu, 11 Jul 2024 11:01:43 +0530 Subject: [PATCH 4/4] Add service support for mirror backend refs --- .../internal/oasparser/constants/constants.go | 2 + .../envoyconf/routes_with_clusters.go | 42 ++++--- .../oasparser/model/adapter_internal_api.go | 116 ++++++++++++++---- .../internal/oasparser/model/api_operation.go | 24 ++-- .../internal/oasparser/model/http_route.go | 17 ++- .../operator/controllers/dp/api_controller.go | 30 +++-- adapter/internal/operator/utils/utils.go | 53 ++++++++ 7 files changed, 210 insertions(+), 74 deletions(-) diff --git a/adapter/internal/oasparser/constants/constants.go b/adapter/internal/oasparser/constants/constants.go index 919b3966c..92ce52d45 100644 --- a/adapter/internal/oasparser/constants/constants.go +++ b/adapter/internal/oasparser/constants/constants.go @@ -137,6 +137,8 @@ const ( KindAPIPolicy = "APIPolicy" KindScope = "Scope" KindRateLimitPolicy = "RateLimitPolicy" + KindService = "Service" + KindBackend = "Backend" ) // API environment types diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 17ab9f110..85f673c38 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -224,29 +224,31 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte // Creating clusters for request mirroring endpoints for _, op := range resource.GetOperations() { - if op.GetMirrorEndpoints() != nil && len(op.GetMirrorEndpoints().Endpoints) > 0 { - mirrorEndpointCluster := op.GetMirrorEndpoints() - for _, mirrorEndpoint := range mirrorEndpointCluster.Endpoints { - mirrorBasepath := strings.TrimSuffix(mirrorEndpoint.Basepath, "/") - existingMirrorClusterName := getExistingClusterName(*mirrorEndpointCluster, processedEndpoints) - var mirrorClusterName string - if existingMirrorClusterName == "" { - mirrorClusterName = getClusterName(mirrorEndpointCluster.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, resource.GetID()) - mirrorCluster, mirrorAddress, err := processEndpoints(mirrorClusterName, mirrorEndpointCluster, timeout, mirrorBasepath) - if err != nil { - logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding resource level mirror filter endpoints for %s:%v-%v. %v", apiTitle, apiVersion, resourcePath, err.Error())) + if op.GetMirrorEndpointClusters() != nil && len(op.GetMirrorEndpointClusters()) > 0 { + mirrorEndpointClusters := op.GetMirrorEndpointClusters() + for _, mirrorEndpointCluster := range mirrorEndpointClusters { + for _, mirrorEndpoint := range mirrorEndpointCluster.Endpoints { + mirrorBasepath := strings.TrimSuffix(mirrorEndpoint.Basepath, "/") + existingMirrorClusterName := getExistingClusterName(*mirrorEndpointCluster, processedEndpoints) + var mirrorClusterName string + if existingMirrorClusterName == "" { + mirrorClusterName = getClusterName(mirrorEndpointCluster.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, op.GetID()) + mirrorCluster, mirrorAddress, err := processEndpoints(mirrorClusterName, mirrorEndpointCluster, timeout, mirrorBasepath) + if err != nil { + logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding resource level mirror filter endpoints for %s:%v-%v. %v", apiTitle, apiVersion, resourcePath, err.Error())) + } else { + clusters = append(clusters, mirrorCluster) + endpoints = append(endpoints, mirrorAddress...) + processedEndpoints[mirrorClusterName] = *mirrorEndpointCluster + } } else { - clusters = append(clusters, mirrorCluster) - endpoints = append(endpoints, mirrorAddress...) - processedEndpoints[mirrorClusterName] = *mirrorEndpointCluster + mirrorClusterName = existingMirrorClusterName } - } else { - mirrorClusterName = existingMirrorClusterName - } - if _, exists := mirrorClusterNames[op.GetID()]; !exists { - mirrorClusterNames[op.GetID()] = []string{} + if _, exists := mirrorClusterNames[op.GetID()]; !exists { + mirrorClusterNames[op.GetID()] = []string{} + } + mirrorClusterNames[op.GetID()] = append(mirrorClusterNames[op.GetID()], mirrorClusterName) } - mirrorClusterNames[op.GetID()] = append(mirrorClusterNames[op.GetID()], mirrorClusterName) } } } diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index d760b256f..0487432fd 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -487,8 +487,9 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap var baseIntervalInMillis uint32 hasURLRewritePolicy := false hasRequestRedirectPolicy := false - var mirrorEndpointsList []Endpoint var securityConfig []EndpointSecurity + var mirrorEndpointClusters []*EndpointCluster + backendBasePath := "" for _, backend := range rule.BackendRefs { backendName := types.NamespacedName{ @@ -702,21 +703,101 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap }) case gwapiv1.HTTPRouteFilterRequestMirror: + var mirrorTimeoutInMillis uint32 + var mirrorIdleTimeoutInSeconds uint32 + var mirrorCircuitBreaker *dpv1alpha1.CircuitBreaker + var mirrorHealthCheck *dpv1alpha1.HealthCheck + isMirrorRetryConfig := false + isMirrorRouteTimeout := false + var mirrorBackendRetryCount uint32 + var mirrorStatusCodes []uint32 + mirrorStatusCodes = append(mirrorStatusCodes, config.Envoy.Upstream.Retry.StatusCodes...) + var mirrorBaseIntervalInMillis uint32 policyParameters := make(map[string]interface{}) - backend := &filter.RequestMirror.BackendRef - backendName := types.NamespacedName{ - Name: string(backend.Name), - Namespace: utils.GetNamespace(backend.Namespace, httpRoute.Namespace), + mirrorBackend := &filter.RequestMirror.BackendRef + mirrorBackendName := types.NamespacedName{ + Name: string(mirrorBackend.Name), + Namespace: utils.GetNamespace(mirrorBackend.Namespace, httpRoute.Namespace), } - _, ok := resourceParams.BackendMapping[backendName.String()] - if !ok { - return fmt.Errorf("backend: %s has not been resolved", backendName) + resolvedMirrorBackend, ok := resourceParams.BackendMapping[mirrorBackendName.String()] + + if ok { + if resolvedMirrorBackend.CircuitBreaker != nil { + mirrorCircuitBreaker = &dpv1alpha1.CircuitBreaker{ + MaxConnections: resolvedMirrorBackend.CircuitBreaker.MaxConnections, + MaxPendingRequests: resolvedMirrorBackend.CircuitBreaker.MaxPendingRequests, + MaxRequests: resolvedMirrorBackend.CircuitBreaker.MaxRequests, + MaxRetries: resolvedMirrorBackend.CircuitBreaker.MaxRetries, + MaxConnectionPools: resolvedMirrorBackend.CircuitBreaker.MaxConnectionPools, + } + } + + if resolvedMirrorBackend.Timeout != nil { + isMirrorRouteTimeout = true + mirrorTimeoutInMillis = resolvedMirrorBackend.Timeout.UpstreamResponseTimeout * 1000 + mirrorIdleTimeoutInSeconds = resolvedMirrorBackend.Timeout.DownstreamRequestIdleTimeout + } + + if resolvedMirrorBackend.Retry != nil { + isMirrorRetryConfig = true + mirrorBackendRetryCount = resolvedMirrorBackend.Retry.Count + mirrorBaseIntervalInMillis = resolvedMirrorBackend.Retry.BaseIntervalMillis + if len(resolvedMirrorBackend.Retry.StatusCodes) > 0 { + mirrorStatusCodes = resolvedMirrorBackend.Retry.StatusCodes + } + } + + if resolvedMirrorBackend.HealthCheck != nil { + mirrorHealthCheck = &dpv1alpha1.HealthCheck{ + Interval: resolvedMirrorBackend.HealthCheck.Interval, + Timeout: resolvedMirrorBackend.HealthCheck.Timeout, + UnhealthyThreshold: resolvedMirrorBackend.HealthCheck.UnhealthyThreshold, + HealthyThreshold: resolvedMirrorBackend.HealthCheck.HealthyThreshold, + } + } + } else { + return fmt.Errorf("backend: %s has not been resolved", mirrorBackendName) } - mirrorEndpoints := GetEndpoints(backendName, resourceParams.BackendMapping) + + mirrorEndpoints := GetEndpoints(mirrorBackendName, resourceParams.BackendMapping) if len(mirrorEndpoints) > 0 { - policyParameters["endpoints"] = mirrorEndpoints + mirrorEndpointCluster := &EndpointCluster{ + Endpoints: mirrorEndpoints, + } + mirrorEndpointConfig := &EndpointConfig{} + if isMirrorRouteTimeout { + mirrorEndpointConfig.TimeoutInMillis = mirrorTimeoutInMillis + mirrorEndpointConfig.IdleTimeoutInSeconds = mirrorIdleTimeoutInSeconds + } + if mirrorCircuitBreaker != nil { + mirrorEndpointConfig.CircuitBreakers = &CircuitBreakers{ + MaxConnections: int32(mirrorCircuitBreaker.MaxConnections), + MaxRequests: int32(mirrorCircuitBreaker.MaxRequests), + MaxPendingRequests: int32(mirrorCircuitBreaker.MaxPendingRequests), + MaxRetries: int32(mirrorCircuitBreaker.MaxRetries), + MaxConnectionPools: int32(mirrorCircuitBreaker.MaxConnectionPools), + } + } + if isMirrorRetryConfig { + mirrorEndpointConfig.RetryConfig = &RetryConfig{ + Count: int32(mirrorBackendRetryCount), + StatusCodes: mirrorStatusCodes, + BaseIntervalInMillis: int32(mirrorBaseIntervalInMillis), + } + } + if mirrorHealthCheck != nil { + mirrorEndpointCluster.HealthCheck = &HealthCheck{ + Interval: mirrorHealthCheck.Interval, + Timeout: mirrorHealthCheck.Timeout, + UnhealthyThreshold: mirrorHealthCheck.UnhealthyThreshold, + HealthyThreshold: mirrorHealthCheck.HealthyThreshold, + } + } + if isMirrorRouteTimeout || mirrorCircuitBreaker != nil || mirrorHealthCheck != nil || isMirrorRetryConfig { + mirrorEndpointCluster.Config = mirrorEndpointConfig + } + mirrorEndpointClusters = append(mirrorEndpointClusters, mirrorEndpointCluster) } - mirrorEndpointsList = append(mirrorEndpointsList, mirrorEndpoints...) policies.Request = append(policies.Request, Policy{ PolicyName: string(gwapiv1.HTTPRouteFilterRequestMirror), Action: constants.ActionMirrorRequest, @@ -732,10 +813,6 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap loggers.LoggerOasparser.Debugf("Calculating auths for API ..., API_UUID = %v", adapterInternalAPI.UUID) apiAuth := getSecurity(resourceAuthScheme) - if !hasRequestRedirectPolicy && len(rule.BackendRefs) < 1 { - return fmt.Errorf("no backendref were provided") - } - for _, match := range rule.Matches { if hasURLRewritePolicy && hasRequestRedirectPolicy { return fmt.Errorf("cannot have URL Rewrite and Request Redirect under the same rule") @@ -756,14 +833,9 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap }) } resourcePath := adapterInternalAPI.xWso2Basepath + *match.Path.Value - var mirrorEndpointCluster *EndpointCluster - if len(mirrorEndpointsList) > 0 { - mirrorEndpointCluster = &EndpointCluster{ - Endpoints: mirrorEndpointsList, - } - } + operations := getAllowedOperations(match.Method, policies, apiAuth, - parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes, mirrorEndpointCluster) + parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes, mirrorEndpointClusters) resource := &Resource{path: resourcePath, methods: operations, pathMatchType: *match.Path.Type, diff --git a/adapter/internal/oasparser/model/api_operation.go b/adapter/internal/oasparser/model/api_operation.go index 0547223d1..6220dbeb6 100644 --- a/adapter/internal/oasparser/model/api_operation.go +++ b/adapter/internal/oasparser/model/api_operation.go @@ -35,15 +35,15 @@ type Operation struct { iD string method string //security map of security scheme names -> list of scopes - scopes []string - auth *Authentication - tier string - disableSecurity bool - vendorExtensions map[string]interface{} - policies OperationPolicies - mockedAPIConfig *api.MockedApiConfig - rateLimitPolicy *RateLimitPolicy - mirrorEndpoints *EndpointCluster + scopes []string + auth *Authentication + tier string + disableSecurity bool + vendorExtensions map[string]interface{} + policies OperationPolicies + mockedAPIConfig *api.MockedApiConfig + rateLimitPolicy *RateLimitPolicy + mirrorEndpointClusters []*EndpointCluster } // Authentication holds authentication related configurations @@ -126,9 +126,9 @@ func (operation *Operation) GetID() string { return operation.iD } -// GetMirrorEndpoints returns the endpoints if a mirror filter has been applied. -func (operation *Operation) GetMirrorEndpoints() *EndpointCluster { - return operation.mirrorEndpoints +// GetMirrorEndpointClusters returns the endpoints if a mirror filter has been applied. +func (operation *Operation) GetMirrorEndpointClusters() []*EndpointCluster { + return operation.mirrorEndpointClusters } // GetCallInterceptorService returns the interceptor configs for a given operation. diff --git a/adapter/internal/oasparser/model/http_route.go b/adapter/internal/oasparser/model/http_route.go index 09b3fed9b..ec77c2155 100644 --- a/adapter/internal/oasparser/model/http_route.go +++ b/adapter/internal/oasparser/model/http_route.go @@ -287,26 +287,25 @@ func getSecurity(authScheme *dpv1alpha2.Authentication) *Authentication { // getAllowedOperations retuns a list of allowed operatons, if httpMethod is not specified then all methods are allowed. func getAllowedOperations(httpMethod *gwapiv1.HTTPMethod, policies OperationPolicies, auth *Authentication, - ratelimitPolicy *RateLimitPolicy, scopes []string, mirrorEndpoints *EndpointCluster) []*Operation { + ratelimitPolicy *RateLimitPolicy, scopes []string, mirrorEndpointClusters []*EndpointCluster) []*Operation { if httpMethod != nil { return []*Operation{{iD: uuid.New().String(), method: string(*httpMethod), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}} - + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}} } return []*Operation{{iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodGet), policies: policies, auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes}, {iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodPost), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}, + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}, {iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodDelete), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}, + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}, {iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodPatch), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}, + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}, {iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodPut), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}, + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}, {iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodHead), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}, + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}, {iD: uuid.New().String(), method: string(gwapiv1.HTTPMethodOptions), policies: policies, - auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpoints: mirrorEndpoints}} + auth: auth, rateLimitPolicy: ratelimitPolicy, scopes: scopes, mirrorEndpointClusters: mirrorEndpointClusters}} } // SetInfoAPICR populates ID, ApiType, Version and XWso2BasePath of adapterInternalAPI. diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index d256c657c..292fcc637 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -851,21 +851,31 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte for _, filter := range rule.Filters { if filter.RequestMirror != nil { mirrorBackend := filter.RequestMirror.BackendRef - mirrorBackendNamespacedName := types.NamespacedName{ Name: string(mirrorBackend.Name), Namespace: utils.GetNamespace(mirrorBackend.Namespace, httpRoute.Namespace), } - if _, exists := backendMapping[mirrorBackendNamespacedName.String()]; !exists { - resolvedMirrorBackend := utils.GetResolvedBackend(ctx, apiReconciler.client, mirrorBackendNamespacedName, &api) - if resolvedMirrorBackend != nil { - backendMapping[mirrorBackendNamespacedName.String()] = resolvedMirrorBackend - } else { - return nil, fmt.Errorf("unable to find backend %s", mirrorBackendNamespacedName.String()) + if string(*mirrorBackend.Kind) == constants.KindBackend { + if _, exists := backendMapping[mirrorBackendNamespacedName.String()]; !exists { + resolvedMirrorBackend := utils.GetResolvedBackend(ctx, apiReconciler.client, mirrorBackendNamespacedName, &api) + if resolvedMirrorBackend != nil { + backendMapping[mirrorBackendNamespacedName.String()] = resolvedMirrorBackend + } else { + return nil, fmt.Errorf("unable to find backend %s", mirrorBackendNamespacedName.String()) + } + } + } else if string(*mirrorBackend.Kind) == constants.KindService { + var err error + service, err := utils.GetService(ctx, apiReconciler.client, utils.GetNamespace(mirrorBackend.Namespace, httpRoute.Namespace), string(mirrorBackend.Name)) + if err != nil { + return nil, fmt.Errorf("unable to find service %s", mirrorBackendNamespacedName.String()) + } + backendMapping[mirrorBackendNamespacedName.String()], err = utils.GetResolvedBackendFromService(service, int(*mirrorBackend.Port)) + if err != nil { + return nil, fmt.Errorf("error in getting service information %s", service) } } } - } } @@ -1839,9 +1849,7 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { authentication := rawObj.(*dpv1alpha2.Authentication) var apis []string if authentication.Spec.TargetRef.Kind == constants.KindAPI { - namespace, err := utils.ValidateAndRetrieveNamespace((*gwapiv1.Namespace)(authentication.Spec.TargetRef.Namespace), authentication.Namespace) - if err != nil { loggers.LoggerAPKOperator.Errorf("Namespace mismatch. TargetRef %s needs to be in the same namespace as the Athentication %s. Expected: %s, Actual: %s", string(authentication.Spec.TargetRef.Name), authentication.Name, authentication.Namespace, string(*authentication.Spec.TargetRef.Namespace)) @@ -1956,7 +1964,7 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } - // ratelimite policy to API indexer + // ratelimit policy to API indexer if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha1.RateLimitPolicy{}, apiRateLimitIndex, func(rawObj k8client.Object) []string { ratelimitPolicy := rawObj.(*dpv1alpha1.RateLimitPolicy) diff --git a/adapter/internal/operator/utils/utils.go b/adapter/internal/operator/utils/utils.go index 14a046aac..d07defc9c 100644 --- a/adapter/internal/operator/utils/utils.go +++ b/adapter/internal/operator/utils/utils.go @@ -33,6 +33,7 @@ import ( "github.com/wso2/apk/adapter/pkg/logging" "github.com/wso2/apk/adapter/pkg/utils/envutils" "github.com/wso2/apk/adapter/pkg/utils/stringutils" + "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" corev1 "k8s.io/api/core/v1" @@ -293,6 +294,58 @@ func getSecretValue(ctx context.Context, client k8client.Client, return string(secret.Data[key]), nil } +// GetService retrieves the Service object and returns its details. +func GetService(ctx context.Context, client k8client.Client, namespace, serviceName string) (*corev1.Service, error) { + service := &corev1.Service{} + err := client.Get(ctx, types.NamespacedName{ + Name: serviceName, + Namespace: namespace, + }, service) + if err != nil { + return nil, err + } + return service, nil +} + +// GetResolvedBackendFromService converts a Kubernetes Service to a Resolved Backend. +func GetResolvedBackendFromService(k8sService *corev1.Service, svcPort int) (*v1alpha1.ResolvedBackend, error) { + + var host string + var port uint32 + + if len(k8sService.Spec.Ports) == 0 { + port = uint32(svcPort) + } else { + servicePort := k8sService.Spec.Ports[0] + port = uint32(servicePort.Port) + } + + switch k8sService.Spec.Type { + case corev1.ServiceTypeClusterIP, corev1.ServiceTypeNodePort: + // Use the internal DNS name for clusterip and nodeport + host = fmt.Sprintf("%s.%s.svc.cluster.local", k8sService.Name, k8sService.Namespace) + case corev1.ServiceTypeLoadBalancer: + // Use the external IP or hostname for LB services + if len(k8sService.Status.LoadBalancer.Ingress) > 0 { + ingress := k8sService.Status.LoadBalancer.Ingress[0] + if ingress.IP != "" { + host = ingress.IP + } else if ingress.Hostname != "" { + host = ingress.Hostname + } else { + return nil, fmt.Errorf("no valid ingress found for LoadBalancer service %s", k8sService.Name) + } + } else { + return nil, fmt.Errorf("no load balancer ingress found for service %s", k8sService.Name) + } + default: + return nil, fmt.Errorf("unsupported service type %s", k8sService.Spec.Type) + } + + backend := &v1alpha1.ResolvedBackend{Services: []v1alpha1.Service{{Host: host, Port: port}}, Protocol: v1alpha1.HTTPProtocol} + return backend, nil +} + // ResolveAndAddBackendToMapping resolves backend from reference and adds it to the backendMapping. func ResolveAndAddBackendToMapping(ctx context.Context, client k8client.Client, backendMapping map[string]*dpv1alpha1.ResolvedBackend,