[feat] add prometheus support for proxy & httpserver. (#877)

* [feat] add prometheus support for proxy & httpserver.

* fix pr comments.

* sorted imports

* fix deadlock

* change lock from pointer to vale

* add default buckets for histogram metrics

doc/reference/metrics.md

@@ -0,0 +1,60 @@
+# Prometheus
+
+- [Prometheus](#prometheus)
+ - [Exporter URI](#exporter-uri)
+ - [Metrics](#metrics)
+ - [Objects](#objects)
+ - [HTTPServer](#httpserver)
+ - [Filters](#filters)
+ - [Proxy](#proxy)
+ - [Create Metrics for Extended Objects and Filters](#create-metrics-for-extended-objects-and-filters)
+
+Easegress has a builtin Prometheus exporter.
+
+## Exporter URI
+
+```
+Get /apis/v2/metrics
+```
+
+## Metrics
+
+### Objects
+
+#### HTTPServer
+
+| Metric | Type | Description | Labels |
+|---------------------------------|-----------|--------------------------------------------------------------|-------------------------------------------------------------------------|
+| httpserver_health | gauge | show the status for the http server: 1 for ready, 0 for down | clusterName, clusterRole, instanceName, name, kind |
+| httpserver_total_requests | counter | the total count of http requests | clusterName, clusterRole, instanceName, name, kind, routerKind, backend |
+| httpserver_total_responses | counter | the total count of http resposnes | clusterName, clusterRole, instanceName, name, kind, routerKind, backend |
+| httpserver_total_error_requests | counter | the total count of http error requests | clusterName, clusterRole, instanceName, name, kind, routerKind, backend |
+| httpserver_requests_duration | histogram | request processing duration histogram | clusterName, clusterRole, instanceName, name, kind, routerKind, backend |
+| httpserver_requests_size_bytes | histogram | a histogram of the total size of the request. Includes body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend |
+| httpserver_responses_size_bytes | histogram | a histogram of the total size of the returned responses body | clusterName, clusterRole, instanceName, name, kind, routerKind, backend |
+
+### Filters
+
+#### Proxy
+
+| Metric | Type | Description | Labels |
+|-------------------------------|-----------|-----------------------------------------------|-------------------------------------------------------------------------------------|
+| proxy_total_connections | counter | the total count of proxy connections | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy |
+| proxy_total_error_connections | counter | the total count of proxy error connections | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy |
+| proxy_request_body_size | histogram | a histogram of the total size of the request | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy |
+| proxy_response_body_size | histogram | a histogram of the total size of the response | clusterName, clusterRole, instanceName, name, kind, loadBalancePolicy, filterPolicy |
+
+## Create Metrics for Extended Objects and Filters
+
+We provide several helper functions to help you create Prometheus metrics
+when develop your own Objects and Filters.
+
+* `prometheushelper.NewCounter`
+* `prometheushelper.NewGauge`
+* `prometheushelper.NewHistogram`
+* `prometheushelper.NewSummary`
+
+These metrics will be registered to `DefaultRegisterer` automatically.
+
+Besides, you can use native Prometheus SDK to create metrics and register them
+by yourself.

go.mod

@@ -43,6 +43,7 @@ require (
  github.com/openzipkin/zipkin-go v0.4.1
  github.com/patrickmn/go-cache v2.1.0+incompatible
  github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
+ github.com/prometheus/client_golang v1.14.0
  github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
  github.com/rs/cors v1.8.2
  github.com/spf13/cobra v1.6.1
@@ -217,7 +218,6 @@ require (
  github.com/pierrec/lz4/v4 v4.1.17 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/prometheus/client_golang v1.14.0 // indirect
  github.com/prometheus/client_model v0.3.0 // indirect
  github.com/prometheus/common v0.37.0 // indirect
  github.com/prometheus/procfs v0.8.0 // indirect

pkg/api/api.go

@@ -105,6 +105,7 @@ func (s *Server) registerAPIs() {
  group.Entries = append(group.Entries, s.aboutAPIEntries()...)
  group.Entries = append(group.Entries, s.customDataAPIEntries()...)
  group.Entries = append(group.Entries, s.profileAPIEntries()...)
+ group.Entries = append(group.Entries, s.prometheusMetricsAPIEntries()...)
 
  for _, fn := range appendAddonAPIs {
  fn(s, group)

pkg/api/prometheus.go

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2017, MegaEase
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const (
+ // PrometheusMetricsPrefix is the prefix of Prometheus metrics exporter
+ PrometheusMetricsPrefix = "/metrics"
+)
+
+func (s *Server) prometheusMetricsAPIEntries() []*Entry {
+ return []*Entry{
+ {
+ Path: PrometheusMetricsPrefix,
+ Method: "GET",
+ Handler: promhttp.Handler().ServeHTTP,
+ },
+ }
+}

pkg/filters/proxy/pool.go

@@ -35,7 +35,9 @@ import (
  "github.com/megaease/easegress/pkg/resilience"
  "github.com/megaease/easegress/pkg/tracing"
  "github.com/megaease/easegress/pkg/util/fasttime"
+ "github.com/megaease/easegress/pkg/util/prometheushelper"
  "github.com/megaease/easegress/pkg/util/readers"
+ "github.com/prometheus/client_golang/prometheus"
 )
 
 // serverPoolError is the error returned by handler function of
@@ -170,6 +172,7 @@ type ServerPool struct {
 
  httpStat *httpstat.HTTPStat
  memoryCache *MemoryCache
+ metrics *metrics
 }
 
 // ServerPoolSpec is the spec for a server pool.
@@ -214,6 +217,7 @@ func NewServerPool(proxy *Proxy, spec *ServerPoolSpec, name string) *ServerPool
  sp.failureCodes[code] = struct{}{}
  }
 
+ sp.metrics = sp.newMetrics(name)
  return sp
 }
 
@@ -262,6 +266,7 @@ func (sp *ServerPool) collectMetrics(spCtx *serverPoolContext) {
  collect := func() {
  metric.Duration = fasttime.Since(spCtx.startTime)
  sp.httpStat.Stat(metric)
+ sp.exportPrometheusMetrics(metric)
  spCtx.LazyAddTag(func() string {
  return sp.name + "#duration: " + metric.Duration.String()
  })
@@ -591,3 +596,67 @@ func (sp *ServerPool) inFailureCodes(code int) bool {
  _, exists := sp.failureCodes[code]
  return exists
 }
+
+type (
+ // ProxyMetrics is the Prometheus Metrics of ProxyMetrics Object.
+ metrics struct {
+ TotalConnections *prometheus.CounterVec
+ TotalErrorConnections *prometheus.CounterVec
+ RequestBodySize prometheus.ObserverVec
+ ResponseBodySize prometheus.ObserverVec
+ }
+)
+
+// newMetrics create the ProxyMetrics.
+func (sp *ServerPool) newMetrics(name string) *metrics {
+ commonLabels := prometheus.Labels{
+ "name": name,
+ "kind": Kind,
+ "clusterName": sp.proxy.super.Options().ClusterName,
+ "clusterRole": sp.proxy.super.Options().ClusterRole,
+ "instanceName": sp.proxy.super.Options().Name,
+ }
+ proxyLabels := []string{"clusterName", "clusterRole", "instanceName",
+ "name", "kind", "loadBalancePolicy", "filterPolicy"}
+ return &metrics{
+ TotalConnections: prometheushelper.NewCounter("proxy_total_connections",
+ "the total count of proxy connections",
+ proxyLabels).MustCurryWith(commonLabels),
+ TotalErrorConnections: prometheushelper.NewCounter("proxy_total_error_connections",
+ "the total count of proxy error connections",
+ proxyLabels).MustCurryWith(commonLabels),
+ RequestBodySize: prometheushelper.NewHistogram(
+ prometheus.HistogramOpts{
+ Name: "proxy_request_body_size",
+ Help: "a histogram of the total size of the request.",
+ Buckets: prometheushelper.DefaultBodySizeBuckets(),
+ },
+ proxyLabels).MustCurryWith(commonLabels),
+ ResponseBodySize: prometheushelper.NewHistogram(
+ prometheus.HistogramOpts{
+ Name: "proxy_response_body_size",
+ Help: "a histogram of the total size of the response.",
+ Buckets: prometheushelper.DefaultBodySizeBuckets(),
+ },
+ proxyLabels).MustCurryWith(commonLabels),
+ }
+}
+
+func (sp *ServerPool) exportPrometheusMetrics(stat *httpstat.Metric) {
+ labels := prometheus.Labels{
+ "loadBalancePolicy": "",
+ "filterPolicy": "",
+ }
+ if sp.spec.LoadBalance != nil {
+ labels["loadBalancePolicy"] = sp.spec.LoadBalance.Policy
+ }
+ if sp.spec.Filter != nil {
+ labels["filterPolicy"] = sp.spec.Filter.Policy
+ }
+ sp.metrics.TotalConnections.With(labels).Inc()
+ if stat.StatusCode >= 400 {
+ sp.metrics.TotalErrorConnections.With(labels).Inc()
+ }
+ sp.metrics.RequestBodySize.With(labels).Observe(float64(stat.ReqSize))
+ sp.metrics.ResponseBodySize.With(labels).Observe(float64(stat.RespSize))
+}

pkg/filters/proxy/pool_test.go

@@ -19,12 +19,15 @@ package proxy
 
 import (
  "net/http"
+ "sync"
  "testing"
 
  "github.com/megaease/easegress/pkg/context"
  "github.com/megaease/easegress/pkg/object/serviceregistry"
+ "github.com/megaease/easegress/pkg/option"
  "github.com/megaease/easegress/pkg/protocols/httpprot"
  "github.com/megaease/easegress/pkg/resilience"
+ "github.com/megaease/easegress/pkg/supervisor"
  "github.com/megaease/easegress/pkg/tracing"
  "github.com/megaease/easegress/pkg/util/codectool"
  "github.com/stretchr/testify/assert"
@@ -90,7 +93,10 @@ servers:
  assert.NoError(err)
  assert.NoError(spec.Validate())
 
- sp := NewServerPool(&Proxy{}, spec, "test")
+ p := &Proxy{}
+ p.super = supervisor.NewMock(option.New(), nil, sync.Map{}, sync.Map{}, nil,
+ nil, false, nil, nil)
+ sp := NewServerPool(p, spec, "test")
  policies := map[string]resilience.Policy{}
 
  assert.Panics(func() { sp.InjectResiliencePolicy(policies) })
@@ -129,7 +135,10 @@ servers:
  assert.NoError(err)
  assert.NoError(spec.Validate())
 
- sp := NewServerPool(&Proxy{}, spec, "test")
+ p := &Proxy{}
+ p.super = supervisor.NewMock(option.New(), nil, sync.Map{}, sync.Map{}, nil,
+ nil, false, nil, nil)
+ sp := NewServerPool(p, spec, "test")
  spCtx := &serverPoolContext{
  Context: context.New(tracing.NoopSpan),
  }
@@ -169,7 +178,10 @@ func TestCopyCORSHeaders(t *testing.T) {
  src.Add("X-Src", "src")
  dst.Add("X-Dst", "dst")
 
- sp := NewServerPool(&Proxy{}, &ServerPoolSpec{}, "test")
+ p := &Proxy{}
+ p.super = supervisor.NewMock(option.New(), nil, sync.Map{}, sync.Map{}, nil,
+ nil, false, nil, nil)
+ sp := NewServerPool(p, &ServerPoolSpec{}, "test")
  dst = sp.mergeResponseHeader(dst, src)
 
  assert.Equal(1, len(dst.Values("Access-Control-Allow-Origin")))
@@ -200,7 +212,10 @@ servers:
  assert.NoError(err)
  assert.NoError(spec.Validate())
 
- sp := NewServerPool(&Proxy{}, spec, "test")
+ p := &Proxy{}
+ p.super = supervisor.NewMock(option.New(), nil, sync.Map{}, sync.Map{}, nil,
+ nil, false, nil, nil)
+ sp := NewServerPool(p, spec, "test")
  svr := sp.LoadBalancer().ChooseServer(nil)
  assert.Equal("http:https://192.168.1.1", svr.URL)
 

pkg/filters/proxy/proxy_test.go

@@ -24,16 +24,19 @@ import (
  "net/http/httptest"
  "os"
  "strings"
+ "sync"
  "sync/atomic"
  "testing"
  "time"
 
  "github.com/megaease/easegress/pkg/context"
  "github.com/megaease/easegress/pkg/filters"
  "github.com/megaease/easegress/pkg/logger"
+ "github.com/megaease/easegress/pkg/option"
  "github.com/megaease/easegress/pkg/protocols/httpprot"
  "github.com/megaease/easegress/pkg/protocols/httpprot/httpstat"
  "github.com/megaease/easegress/pkg/resilience"
+ "github.com/megaease/easegress/pkg/supervisor"
  "github.com/megaease/easegress/pkg/tracing"
  "github.com/megaease/easegress/pkg/util/codectool"
  "github.com/stretchr/testify/assert"
@@ -54,6 +57,10 @@ func newTestProxy(yamlConfig string, assert *assert.Assertions) *Proxy {
  assert.NoError(err)
 
  proxy := kind.CreateInstance(spec).(*Proxy)
+
+ proxy.super = supervisor.NewMock(option.New(), nil, sync.Map{}, sync.Map{}, nil,
+ nil, false, nil, nil)
+
  proxy.Init()
 
  assert.Equal(kind, proxy.Kind())

pkg/object/httpserver/httpserver_test.go

@@ -19,11 +19,13 @@ package httpserver
 
 import (
  "os"
+ "sync"
  "testing"
  "time"
 
  "github.com/megaease/easegress/pkg/context/contexttest"
  "github.com/megaease/easegress/pkg/logger"
+ "github.com/megaease/easegress/pkg/option"
  "github.com/megaease/easegress/pkg/supervisor"
  "github.com/stretchr/testify/assert"
 )
@@ -44,7 +46,9 @@ port: 38081
 keepAlive: true
 https: false
 `
- superSpec, err := supervisor.NewSpec(yamlConfig)
+ super := supervisor.NewMock(option.New(), nil, sync.Map{}, sync.Map{}, nil,
+ nil, false, nil, nil)
+ superSpec, err := super.NewSpec(yamlConfig)
  assert.NoError(err)
 
  svr := &HTTPServer{}
@@ -57,7 +61,7 @@ port: 38082
 keepAlive: true
 https: false
 `
- superSpec, err = supervisor.NewSpec(yamlConfig)
+ superSpec, err = super.NewSpec(yamlConfig)
  assert.NoError(err)
  svr2 := &HTTPServer{}
  svr2.Inherit(superSpec, svr, &contexttest.MockedMuxMapper{})