From e1c6db439f85376228cf19596a4f898b783d8e41 Mon Sep 17 00:00:00 2001 From: charleslin-appd Date: Sun, 5 Feb 2023 15:18:27 -0700 Subject: [PATCH 1/6] add appdynamicscloud section to metrics usage doc Signed-off-by: charleslin-appd --- docs/gitbook/usage/metrics.md | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/gitbook/usage/metrics.md b/docs/gitbook/usage/metrics.md index 18f899b72..9788bd3f2 100644 --- a/docs/gitbook/usage/metrics.md +++ b/docs/gitbook/usage/metrics.md @@ -623,3 +623,68 @@ Reference the template in the canary analysis: max: 1000 interval: 1m ``` + +## AppDynamicsCloud + +You can create custom metric checks using the AppDynamicsCloud provider. + +Create a secret with your AppDynamicsCloud service principal credential: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: appdynamicscloud-secret + namespace: istio-system +data: + appdcloud_client_secret_id: your-appdcloud-service-principal-id + appdcloud_client_secret_key: your-appdcloud-service-principal-key +``` + +AppDynamics metric template examples: + +```yaml +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: error-rate + namespace: istio-system +spec: + provider: + type: appdynamicscloud + address: https://tenant_name.observe.appdynamics.com + secretRef: + name: appdynamicscloud-secret + query: | + fetch epm: metrics(apm:errors_min) {timestamp, value} from entities(apm:service)[attributes(service.name) = '{{ target }}_backend'] since -10m +``` + +```yaml +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: response-time + namespace: istio-system +spec: + provider: + type: appdynamicscloud + address: https://tenant_name.observe.appdynamics.com + secretRef: + name: appdynamicscloud-secret + query: | + fetch art: metrics(apm:response_time) {timestamp, value} from entities(apm:service)[attributes(service.namespace) = '{{ namespace }}' && attributes(service.name) = '{{ target }}'] since -10m +``` + +Reference the template in the canary analysis: + +```yaml + analysis: + metrics: + - name: "error-rate" + templateRef: + name: error-rate + namespace: istio-system + thresholdRange: + max: 1 + interval: 1m +``` From c1db57c702060e21a000dec908a4fc7f0aed6b90 Mon Sep 17 00:00:00 2001 From: charleslin-appd Date: Sun, 5 Feb 2023 15:21:55 -0700 Subject: [PATCH 2/6] add appdynamicscloud provider code Signed-off-by: charleslin-appd --- pkg/metrics/providers/appdynamicscloud.go | 214 ++++++++++++++++++ .../providers/appdynamicscloud_test.go | 173 ++++++++++++++ pkg/metrics/providers/factory.go | 2 + 3 files changed, 389 insertions(+) create mode 100644 pkg/metrics/providers/appdynamicscloud.go create mode 100644 pkg/metrics/providers/appdynamicscloud_test.go diff --git a/pkg/metrics/providers/appdynamicscloud.go b/pkg/metrics/providers/appdynamicscloud.go new file mode 100644 index 000000000..f3f371f75 --- /dev/null +++ b/pkg/metrics/providers/appdynamicscloud.go @@ -0,0 +1,214 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + + "golang.org/x/oauth2/clientcredentials" +) + +// https://developer.cisco.com/docs/appdynamics/query-service/#!api-reference-appdynamics-cloud-query-service-api +const ( + clientSecretID = "appdcloud_client_secret_id" + clientSecretKey = "appdcloud_client_secret_key" + + metricsQueryPath = "/monitoring/v1/query/execute" + tenantLookupEndpoint = "https://observe-tenant-lookup-api.saas.appdynamics.com/tenants/lookup/" +) + +type AppDynamicsCloudProvider struct { + tenantID string + tenantAddress string + metricsQueryEndpoint string + clientSecretID string + clientSecretKey string + + timeout time.Duration + client *http.Client +} + +// NewAppDynamicsCloudProvider takes a provider spec and the credentials map, +// and returns a AppDynamicsCloud client ready to execute queries against the API +func NewAppDynamicsCloudProvider( + provider flaggerv1.MetricTemplateProvider, + credentials map[string][]byte) (*AppDynamicsCloudProvider, error) { + + address := provider.Address + if address == "" { + return nil, fmt.Errorf("appdynamics cloud endpoint url address is not set") + } + + tid, err := getTenantID(address) + if tid == "" || err != nil { + return nil, fmt.Errorf("failed to retrieve tenant id based on tenant URL address: %s", address) + } + + appdCloudProvider := AppDynamicsCloudProvider{ + tenantID: tid, + tenantAddress: address, + metricsQueryEndpoint: address + metricsQueryPath, + + timeout: 5 * time.Second, + } + + if b, ok := credentials[clientSecretID]; ok { + appdCloudProvider.clientSecretID = string(b) + } else { + return nil, fmt.Errorf("appdynamics cloud credentials does not contain %s", clientSecretID) + } + + if b, ok := credentials[clientSecretKey]; ok { + appdCloudProvider.clientSecretKey = string(b) + } else { + return nil, fmt.Errorf("appdynamics cloud credentials does not contain %s", clientSecretKey) + } + + return &appdCloudProvider, nil + +} + +// RunQuery executes the appdynamics cloud query against AppDynamicsCloudProvider +// metricsQueryEndpoint and returns the result as float64 +func (p *AppDynamicsCloudProvider) RunQuery(query string) (float64, error) { + if p.client == nil { + if _, err := p.IsOnline(); err != nil { + return 0, fmt.Errorf("failed to login to query endpoint: %w", err) + } + } + + jsonQuery, err := json.Marshal(map[string]string{ + "query": query, + }) + if err != nil { + return 0, fmt.Errorf("failed to marshal query: %w", err) + } + + // output to flagger container runtime log + // fmt.Print("appdynamicscloud metric query:", string(jsonQuery)) + p.client.Timeout = p.timeout + resp, err := p.client.Post(p.metricsQueryEndpoint, "application/json", bytes.NewBuffer(jsonQuery)) + if err != nil { + return 0, fmt.Errorf("failed to get query response: %w", err) + } + + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + // we want to extract just a single float value from the result, hence no need to + // un-marshalling the entire struct using Appdynamics Cloud Query API + var anyValue []map[string]any + if err := json.Unmarshal(body, &anyValue); err != nil { + return 0, fmt.Errorf("failed to un-marshaling result: %s.\n error: %w", string(body), err) + } + if len(anyValue) < 1 { + return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + } + // actual result (non-meta data) is in the data element of the last item + // of the json array + data := anyValue[len(anyValue)-1]["data"].([]any) + if len(data) < 1 { + return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + } + // nested in the data element of the first item + data_data := data[0].([]any) + if len(data_data) < 2 { + return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + } + // metrics data is the second element, the first element is the + // source, e.g. "sys:derived" + metrics_data := data_data[1].([]any) + if len(data_data) < 2 { + return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + } + // get the last metrics from the array of metrics + metric := metrics_data[len(metrics_data)-1].([]any) + if len(data_data) < 2 { + return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + } + + return metric[1].(float64), nil +} + +// IsOnline calls the Appdynamics Cloud's metrics endpoint with client ID and +// secret and fills the authToken, returns an error if the endpoint fails +func (p *AppDynamicsCloudProvider) IsOnline() (bool, error) { + // set up the struct according to clientcredentials package + ccConfig := clientcredentials.Config{ + + ClientID: p.clientSecretID, + ClientSecret: p.clientSecretKey, + TokenURL: p.tenantAddress + "/auth/" + p.tenantID + "/default/oauth2/token", + } + + // check if we can get the token + _, err := ccConfig.Token(context.Background()) + + if err != nil { + return false, fmt.Errorf("failed to authenticate : %w", err) + } + p.client = ccConfig.Client(context.Background()) + + return true, nil +} + +// getTenantID make a request to the lookup service and get the tenant id based +// on tenant url address. TenantID is used to get the auth token. +func getTenantID(address string) (string, error) { + var reqURL string + if u, err := url.Parse(address); err == nil { + host, _, _ := net.SplitHostPort(u.Host) + if host == "" { + // there is no port specified in the address + host = u.Host + } + reqURL = tenantLookupEndpoint + host + } else { + return "", fmt.Errorf("appdynamics cloud endpoint url address is misformed") + } + + httpResp, err := http.Get(reqURL) + if err != nil { + return "", fmt.Errorf("unable to get tenant id, got error: %s", err) + } + if httpResp.StatusCode > 300 { + return "", fmt.Errorf("error code returned, reqURL is %s and got error: %s", reqURL, httpResp.Status) + } + // Parse the response body to get the tenant id + body, err := io.ReadAll(httpResp.Body) + if err != nil { + return "", fmt.Errorf("unable to read api response body, got error: %s", err) + } + + var jsondata map[string]string + err = json.Unmarshal(body, &jsondata) + if err != nil { + return "", fmt.Errorf("unable to unmarshal api response body, got error: %s", body) + } + + return jsondata["tenantId"], nil +} diff --git a/pkg/metrics/providers/appdynamicscloud_test.go b/pkg/metrics/providers/appdynamicscloud_test.go new file mode 100644 index 000000000..13b3f399e --- /dev/null +++ b/pkg/metrics/providers/appdynamicscloud_test.go @@ -0,0 +1,173 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" +) + +var ( + clientID = "secretID" + clientKey = "secretKey" + + secrets = map[string][]byte{ + "appdcloud_client_secret_id": []byte(clientID), + "appdcloud_client_secret_key": []byte(clientKey), + } +) + +func TestNewAppDynamicsCloudProvider(t *testing.T) { + appdcloud, err := NewAppDynamicsCloudProvider(flaggerv1.MetricTemplateProvider{ + Address: "https://lab1.observe.appdynamics.com"}, secrets) + require.NoError(t, err) + assert.Equal(t, "https://lab1.observe.appdynamics.com/monitoring/v1/query/execute", appdcloud.metricsQueryEndpoint) + assert.Equal(t, clientID, appdcloud.clientSecretID) + assert.Equal(t, clientKey, appdcloud.clientSecretKey) +} + +func TestAppDynamicsCloudProvider_RunQuery(t *testing.T) { + goodResponse := ` +[{ + "type" : "model", + "model" : { + "name" : "m:main", + "fields" : [ { + "alias" : "cpm", + "type" : "complex", + "hints" : { + "kind" : "metric", + "type" : "apm:response_time" + }, + "form" : "reference", + "model" : { + "name" : "m:cpm", + "fields" : [ { + "alias" : "source", + "type" : "string", + "hints" : { + "kind" : "metric", + "field" : "source" + } + }, { + "alias" : "metrics", + "type" : "timeseries", + "hints" : { + "kind" : "metric", + "type" : "apm:response_time" + }, + "form" : "inline", + "model" : { + "name" : "m:metrics", + "fields" : [ { + "alias" : "timestamp", + "type" : "timestamp", + "hints" : { + "kind" : "metric", + "field" : "timestamp", + "type" : "apm:response_time" + } + }, { + "alias" : "value", + "type" : "number", + "hints" : { + "kind" : "metric", + "field" : "value", + "type" : "apm:response_time" + } + } ] + } + } ] + } + } ] + } + },{ + "type" : "data", + "model" : { + "$jsonPath" : "$..[?(@.type == 'model')]..[?(@.name == 'm:main')]", + "$model" : "m:main" + }, + "metadata" : { + "since" : "2023-02-02T16:53:36.983726752Z", + "until" : "2023-02-02T17:03:36.983726752Z" + }, + "dataset" : "d:main", + "data" : [ [ { + "$dataset" : "d:metrics-1", + "$jsonPath" : "$..[?(@.type == 'data' && @.dataset == 'd:metrics-1')]" + } ] ] + },{ + "type" : "data", + "model" : { + "$jsonPath" : "$..[?(@.type == 'model')]..[?(@.name == 'm:cpm')]", + "$model" : "m:cpm" + }, + "metadata" : { + "granularitySeconds" : 60 + }, + "dataset" : "d:metrics-1", + "data" : [ [ "sys:derived", [ [ "2023-02-02T16:55Z", 334438.4 ], [ "2023-02-02T16:57Z", 425362.28571428574 ], [ "2023-02-02T16:59Z", 364288.0 ] ] ] ] + }]` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(goodResponse)) + })) + defer ts.Close() + + provider, err := NewAppDynamicsCloudProvider(flaggerv1.MetricTemplateProvider{ + Address: "https://lab1.observe.appdynamics.com"}, secrets) + + assert.NoError(t, err) + // alter the metrics endpoint for testing + provider.tenantAddress = ts.URL + provider.metricsQueryEndpoint = provider.tenantAddress + metricsQueryPath + provider.client = http.DefaultClient + + float, err := provider.RunQuery(`fake request`) + assert.NoError(t, err) + assert.Equal(t, 364288.0, float) +} + +func TestAppDynamicsCloudProvider_IsOnline(t *testing.T) { + // test if we have client secret id and secret key defined + envID, idPresent := os.LookupEnv("APPD_CLOUD_CLIENT_ID") + envKey, keyPresent := os.LookupEnv("APPD_CLOUD_CLIENT_SECRET") + + if !idPresent || !keyPresent { + t.Log("test skipped since no credentials are set in env variables") + return + } + + secrets := map[string][]byte{ + "appdcloud_client_secret_id": []byte(envID), + "appdcloud_client_secret_key": []byte(envKey), + } + + appdcloud, err := NewAppDynamicsCloudProvider(flaggerv1.MetricTemplateProvider{ + Address: "https://lab1.observe.appdynamics.com"}, secrets) + require.NoError(t, err) + + _, err = appdcloud.IsOnline() + require.NoError(t, err) + +} diff --git a/pkg/metrics/providers/factory.go b/pkg/metrics/providers/factory.go index db5bd9f5b..1dcb6b99e 100644 --- a/pkg/metrics/providers/factory.go +++ b/pkg/metrics/providers/factory.go @@ -44,6 +44,8 @@ func (factory Factory) Provider( return NewInfluxdbProvider(provider, credentials) case "dynatrace": return NewDynatraceProvider(metricInterval, provider, credentials) + case "appdynamicscloud": + return NewAppDynamicsCloudProvider(provider, credentials) default: return NewPrometheusProvider(provider, credentials) } From 6faac47cec79bc611d193d54f55bd8f05f7d9b13 Mon Sep 17 00:00:00 2001 From: charleslin-appd Date: Sun, 5 Feb 2023 15:23:59 -0700 Subject: [PATCH 3/6] add appdynamicscloud to crds Signed-off-by: charleslin-appd --- charts/flagger/crds/crd.yaml | 1 + kustomize/base/flagger/crd.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index 02481ccb6..bc94d8021 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -1198,6 +1198,7 @@ spec: - newrelic - graphite - dynatrace + - appdynamicscloud address: description: API address of this provider type: string diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 02481ccb6..bc94d8021 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -1198,6 +1198,7 @@ spec: - newrelic - graphite - dynatrace + - appdynamicscloud address: description: API address of this provider type: string From c77d8ebefc9a7e9eaeb12d2c1abe39f7dabf930f Mon Sep 17 00:00:00 2001 From: charleslin-appd Date: Sun, 12 Feb 2023 11:00:22 -0700 Subject: [PATCH 4/6] Specify more accurate error message Signed-off-by: charleslin-appd --- pkg/metrics/providers/appdynamicscloud.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/providers/appdynamicscloud.go b/pkg/metrics/providers/appdynamicscloud.go index f3f371f75..1ec1638a2 100644 --- a/pkg/metrics/providers/appdynamicscloud.go +++ b/pkg/metrics/providers/appdynamicscloud.go @@ -126,18 +126,18 @@ func (p *AppDynamicsCloudProvider) RunQuery(query string) (float64, error) { return 0, fmt.Errorf("failed to un-marshaling result: %s.\n error: %w", string(body), err) } if len(anyValue) < 1 { - return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + return 0, fmt.Errorf("invalid response for query: %s: %w", query, ErrNoValuesFound) } // actual result (non-meta data) is in the data element of the last item // of the json array data := anyValue[len(anyValue)-1]["data"].([]any) if len(data) < 1 { - return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + return 0, fmt.Errorf("metrics not available for query: %s", query) } // nested in the data element of the first item data_data := data[0].([]any) if len(data_data) < 2 { - return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) + return 0, fmt.Errorf("invalid data item for query: %s", query) } // metrics data is the second element, the first element is the // source, e.g. "sys:derived" From 134f92e5d65565bb38405415a7b58e34dd4410d0 Mon Sep 17 00:00:00 2001 From: charleslin-appd Date: Wed, 15 Feb 2023 09:06:28 -0700 Subject: [PATCH 5/6] add appdynmiccloud to crd.yaml in artifacts folder Signed-off-by: charleslin-appd --- artifacts/flagger/crd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 02481ccb6..bc94d8021 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -1198,6 +1198,7 @@ spec: - newrelic - graphite - dynatrace + - appdynamicscloud address: description: API address of this provider type: string From 2608dc62ce5f8a4a0217712d2c4824766acbdb98 Mon Sep 17 00:00:00 2001 From: charleslin-appd Date: Sun, 19 Feb 2023 09:50:43 -0700 Subject: [PATCH 6/6] add more error check and type assertion Signed-off-by: charleslin-appd --- pkg/metrics/providers/appdynamicscloud.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/providers/appdynamicscloud.go b/pkg/metrics/providers/appdynamicscloud.go index 1ec1638a2..41dfb9fe5 100644 --- a/pkg/metrics/providers/appdynamicscloud.go +++ b/pkg/metrics/providers/appdynamicscloud.go @@ -151,7 +151,13 @@ func (p *AppDynamicsCloudProvider) RunQuery(query string) (float64, error) { return 0, fmt.Errorf("invalid response: %s: %w", string(body), ErrNoValuesFound) } - return metric[1].(float64), nil + // assert the second item of the array is numeric and return + r, ok := metric[1].(float64) + if ok { + return r, nil + } else { + return 0, fmt.Errorf("failed to retrieve numeric result: %s: %w", query, ErrNoValuesFound) + } } // IsOnline calls the Appdynamics Cloud's metrics endpoint with client ID and