From 60cabb7e98154189dcfb7597de01c5cb14658f9e Mon Sep 17 00:00:00 2001 From: Loki's Wager <32408858+LokiWager@users.noreply.github.com> Date: Thu, 22 Dec 2022 11:02:51 +0800 Subject: [PATCH] add Prometheus percentage metrics for httpserver & proxy (#879) * add percentage metrics for httpserver & proxy * fix proxy pool Objectives --- doc/reference/metrics.md | 35 +++++++++++++----------- pkg/filters/proxy/pool.go | 26 +++++++++++++++--- pkg/object/httpserver/metrics_test.go | 21 +++++++++++++++ pkg/object/httpserver/mux.go | 3 +++ pkg/object/httpserver/runtime.go | 38 ++++++++++++++++++++++----- pkg/util/prometheushelper/helper.go | 5 ++++ 6 files changed, 102 insertions(+), 26 deletions(-) diff --git a/doc/reference/metrics.md b/doc/reference/metrics.md index beaadf5a01..cd87c169c0 100644 --- a/doc/reference/metrics.md +++ b/doc/reference/metrics.md @@ -23,26 +23,31 @@ Get /apis/v2/metrics #### HTTPServer -| Metric | Type | Description | Labels | -|---------------------------------|-----------|--------------------------------------------------------------|-------------------------------------------------------------------------| -| httpserver_health | gauge | show the status for the http server: 1 for ready, 0 for down | clusterName, clusterRole, instanceName, name, kind | -| httpserver_total_requests | counter | the total count of http requests | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | -| httpserver_total_responses | counter | the total count of http resposnes | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | -| httpserver_total_error_requests | counter | the total count of http error requests | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | -| httpserver_requests_duration | histogram | request processing duration histogram | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | -| httpserver_requests_size_bytes | histogram | a histogram of the total size of the request. Includes body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | -| httpserver_responses_size_bytes | histogram | a histogram of the total size of the returned responses body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| Metric | Type | Description | Labels | +|--------------------------------------------|-----------|--------------------------------------------------------------|-------------------------------------------------------------------------| +| httpserver_health | gauge | show the status for the http server: 1 for ready, 0 for down | clusterName, clusterRole, instanceName, name, kind | +| httpserver_total_requests | counter | the total count of http requests | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_total_responses | counter | the total count of http resposnes | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_total_error_requests | counter | the total count of http error requests | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_requests_duration | histogram | request processing duration histogram | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_requests_size_bytes | histogram | a histogram of the total size of the request. Includes body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_responses_size_bytes | histogram | a histogram of the total size of the returned responses body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_requests_duration_percentage | summary | request processing duration summary | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_requests_size_bytes_percentage | summary | a summary of the total size of the request. Includes body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | +| httpserver_responses_size_bytes_percentage | summary | a summary of the total size of the returned responses body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend | ### Filters #### Proxy -| Metric | Type | Description | Labels | -|-------------------------------|-----------|-----------------------------------------------|-------------------------------------------------------------------------------------| -| proxy_total_connections | counter | the total count of proxy connections | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | -| proxy_total_error_connections | counter | the total count of proxy error connections | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | -| proxy_request_body_size | histogram | a histogram of the total size of the request | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | -| proxy_response_body_size | histogram | a histogram of the total size of the response | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | +| Metric | Type | Description | Labels | +|-------------------------------------|-----------|-----------------------------------------------|-------------------------------------------------------------------------------------| +| proxy_total_connections | counter | the total count of proxy connections | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | +| proxy_total_error_connections | counter | the total count of proxy error connections | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | +| proxy_request_body_size | histogram | a histogram of the total size of the request | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | +| proxy_response_body_size | histogram | a histogram of the total size of the response | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | +| proxy_request_body_size_percentage | summary | a summary of the total size of the request | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | +| proxy_response_body_size_percentage | summary | a summary of the total size of the response | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy | ## Create Metrics for Extended Objects and Filters diff --git a/pkg/filters/proxy/pool.go b/pkg/filters/proxy/pool.go index d44a9be844..57d90317aa 100644 --- a/pkg/filters/proxy/pool.go +++ b/pkg/filters/proxy/pool.go @@ -600,10 +600,12 @@ func (sp *ServerPool) inFailureCodes(code int) bool { type ( // ProxyMetrics is the Prometheus Metrics of ProxyMetrics Object. metrics struct { - TotalConnections *prometheus.CounterVec - TotalErrorConnections *prometheus.CounterVec - RequestBodySize prometheus.ObserverVec - ResponseBodySize prometheus.ObserverVec + TotalConnections *prometheus.CounterVec + TotalErrorConnections *prometheus.CounterVec + RequestBodySize prometheus.ObserverVec + ResponseBodySize prometheus.ObserverVec + RequestBodySizePercentage prometheus.ObserverVec + ResponseBodySizePercentage prometheus.ObserverVec } ) @@ -639,6 +641,20 @@ func (sp *ServerPool) newMetrics(name string) *metrics { Buckets: prometheushelper.DefaultBodySizeBuckets(), }, proxyLabels).MustCurryWith(commonLabels), + RequestBodySizePercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "proxy_request_body_size_percentage", + Help: "a summary of the total size of the request.", + Objectives: prometheushelper.DefaultObjectives(), + }, + proxyLabels).MustCurryWith(commonLabels), + ResponseBodySizePercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "proxy_response_body_size_percentage", + Help: "a summary of the total size of the response.", + Objectives: prometheushelper.DefaultObjectives(), + }, + proxyLabels).MustCurryWith(commonLabels), } } @@ -659,4 +675,6 @@ func (sp *ServerPool) exportPrometheusMetrics(stat *httpstat.Metric) { } sp.metrics.RequestBodySize.With(labels).Observe(float64(stat.ReqSize)) sp.metrics.ResponseBodySize.With(labels).Observe(float64(stat.RespSize)) + sp.metrics.RequestBodySizePercentage.With(labels).Observe(float64(stat.ReqSize)) + sp.metrics.ResponseBodySizePercentage.With(labels).Observe(float64(stat.RespSize)) } diff --git a/pkg/object/httpserver/metrics_test.go b/pkg/object/httpserver/metrics_test.go index 3c19ed671b..09dbd513b1 100644 --- a/pkg/object/httpserver/metrics_test.go +++ b/pkg/object/httpserver/metrics_test.go @@ -65,5 +65,26 @@ func newMockMetrics() *metrics { Buckets: prometheushelper.DefaultBodySizeBuckets(), }, mockLabels).MustCurryWith(commonLabels), + RequestsDurationPercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "mock_httpserver_requests_duration_percentage", + Help: "request processing duration summary", + Objectives: prometheushelper.DefaultObjectives(), + }, + mockLabels).MustCurryWith(commonLabels), + RequestSizeBytesPercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "mock_httpserver_requests_size_bytes_percentage", + Help: "a summary of the total size of the request. Includes body", + Objectives: prometheushelper.DefaultObjectives(), + }, + mockLabels).MustCurryWith(commonLabels), + ResponseSizeBytesPercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "mock_httpserver_responses_size_bytes_percentage", + Help: "a summary of the total size of the returned response body", + Objectives: prometheushelper.DefaultObjectives(), + }, + mockLabels).MustCurryWith(commonLabels), } } diff --git a/pkg/object/httpserver/mux.go b/pkg/object/httpserver/mux.go index d0dcea268a..49b09b16fe 100644 --- a/pkg/object/httpserver/mux.go +++ b/pkg/object/httpserver/mux.go @@ -427,4 +427,7 @@ func (mi *muxInstance) exportPrometheusMetrics(stat *httpstat.Metric, backend st mi.metrics.RequestsDuration.With(labels).Observe(float64(stat.Duration.Milliseconds())) mi.metrics.RequestSizeBytes.With(labels).Observe(float64(stat.ReqSize)) mi.metrics.ResponseSizeBytes.With(labels).Observe(float64(stat.RespSize)) + mi.metrics.RequestsDurationPercentage.With(labels).Observe(float64(stat.Duration.Milliseconds())) + mi.metrics.RequestSizeBytesPercentage.With(labels).Observe(float64(stat.ReqSize)) + mi.metrics.ResponseSizeBytesPercentage.With(labels).Observe(float64(stat.RespSize)) } diff --git a/pkg/object/httpserver/runtime.go b/pkg/object/httpserver/runtime.go index ee6af2ee61..db24b8beaa 100644 --- a/pkg/object/httpserver/runtime.go +++ b/pkg/object/httpserver/runtime.go @@ -425,13 +425,16 @@ func (s *Status) ToMetrics(service string) []*easemonitor.Metrics { type ( metrics struct { - Health *prometheus.GaugeVec - TotalRequests *prometheus.CounterVec - TotalResponses *prometheus.CounterVec - TotalErrorRequests *prometheus.CounterVec - RequestsDuration prometheus.ObserverVec - RequestSizeBytes prometheus.ObserverVec - ResponseSizeBytes prometheus.ObserverVec + Health *prometheus.GaugeVec + TotalRequests *prometheus.CounterVec + TotalResponses *prometheus.CounterVec + TotalErrorRequests *prometheus.CounterVec + RequestsDuration prometheus.ObserverVec + RequestSizeBytes prometheus.ObserverVec + ResponseSizeBytes prometheus.ObserverVec + RequestsDurationPercentage prometheus.ObserverVec + RequestSizeBytesPercentage prometheus.ObserverVec + ResponseSizeBytesPercentage prometheus.ObserverVec } ) @@ -484,6 +487,27 @@ func (r *runtime) newMetrics(name string) *metrics { Buckets: prometheushelper.DefaultBodySizeBuckets(), }, httpserverLabels).MustCurryWith(commonLabels), + RequestsDurationPercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "httpserver_requests_duration_percentage", + Help: "request processing duration summary", + Objectives: prometheushelper.DefaultObjectives(), + }, + httpserverLabels).MustCurryWith(commonLabels), + RequestSizeBytesPercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "httpserver_requests_size_bytes_percentage", + Help: "a summary of the total size of the request. Includes body", + Objectives: prometheushelper.DefaultObjectives(), + }, + httpserverLabels).MustCurryWith(commonLabels), + ResponseSizeBytesPercentage: prometheushelper.NewSummary( + prometheus.SummaryOpts{ + Name: "httpserver_responses_size_bytes_percentage", + Help: "a summary of the total size of the returned response body", + Objectives: prometheushelper.DefaultObjectives(), + }, + httpserverLabels).MustCurryWith(commonLabels), } } diff --git a/pkg/util/prometheushelper/helper.go b/pkg/util/prometheushelper/helper.go index d1525423a8..d96cc1b9cb 100644 --- a/pkg/util/prometheushelper/helper.go +++ b/pkg/util/prometheushelper/helper.go @@ -180,3 +180,8 @@ func DefaultDurationBuckets() []float64 { func DefaultBodySizeBuckets() []float64 { return prometheus.ExponentialBucketsRange(200, 400000, 10) } + +// DefaultObjectives returns default summary objectives +func DefaultObjectives() map[float64]float64 { + return map[float64]float64{0.25: 10, 0.5: 10, 0.75: 10, 0.9: 10, 0.95: 10, 0.99: 10} +}