Adds --collect-metric-time. (#104)

This commit adds the optional (default false) parameter
collect-metric-time, when passed, this will include a metric
for storing the time spent gathering each metric.

Co-Authored by: Sergei Mikhailov <newrushbolt>
Signed-off-by: Jorge Niedbalski <[email protected]>

README.md

@@ -77,6 +77,7 @@ usage: openstack-exporter [<flags>] <cloud>
 
 Flags:
  -h, --help Show context-sensitive help (also try --help-long and --help-man).
+ --log.level="info" Log level: [debug, info, warn, error, fatal]
  --web.listen-address=":9180" 
  address:port to listen on
  --web.telemetry-path="/metrics" 
@@ -85,13 +86,22 @@ Flags:
  Path to the cloud configuration file
  --prefix="openstack" Prefix for metrics
  --endpoint-type="public" openstack endpoint type to use (i.e: public, internal, admin)
+ --collect-metric-time time spent collecting each metric
  -d, --disable-metric= ... multiple --disable-metric can be specified in the format: service-metric (i.e: cinder-snapshots)
  --disable-service.network Disable the network service exporter
  --disable-service.compute Disable the compute service exporter
  --disable-service.image Disable the image service exporter
  --disable-service.volume Disable the volume service exporter
  --disable-service.identity 
  Disable the identity service exporter
+ --disable-service.object-store
+ Disable the object-store service exporter
+ --disable-service.load-balancer
+ Disable the load-balancer service exporter
+ --disable-service.container-infra
+ Disable the container-infra service exporter
+ --disable-service.dns Disable the dns service exporter
+ --version Show application version.
 
 Args:
  <cloud> name or id of the cloud to gather metrics from
@@ -185,6 +195,7 @@ openstack_identity_projects|region="RegionOne"|33.0 (float)
 openstack_identity_groups|region="RegionOne"|1.0 (float)
 openstack_identity_regions|region="RegionOne"|1.0 (float)
 openstack_object_store_objects|region="RegionOne",container_name="test2"|1.0 (float) 
+openstack_metric_collect_seconds | {openstack_metric="agent_state",openstack_service="openstack_cinder"} |1.27843913| Only if --collect-metric-time is passed
 
 ## Example metrics
 ```

exporters/cinder.go

@@ -64,15 +64,14 @@ var defaultCinderMetrics = []Metric{
  {Name: "limits_volume_used_gb", Labels: []string{"tenant", "tenant_id"}, Fn: nil},
 }
 
-func NewCinderExporter(client *gophercloud.ServiceClient, prefix string, disabledMetrics []string) (*CinderExporter, error) {
+func NewCinderExporter(config *ExporterConfig) (*CinderExporter, error) {
  exporter := CinderExporter{
  BaseOpenStackExporter{
- Name: "cinder",
- Prefix: prefix,
- Client: client,
- DisabledMetrics: disabledMetrics,
+ Name: "cinder",
+ ExporterConfig: *config,
  },
  }
+
  for _, metric := range defaultCinderMetrics {
  exporter.AddMetric(metric.Name, metric.Fn, metric.Labels, nil)
  }

exporters/containerinfra.go

@@ -1,7 +1,6 @@
 package exporters
 
 import (
- "github.com/gophercloud/gophercloud"
  "github.com/gophercloud/gophercloud/openstack/containerinfra/v1/clusters"
  "github.com/prometheus/client_golang/prometheus"
  "strconv"
@@ -46,13 +45,11 @@ var defaultContainerInfraMetrics = []Metric{
  {Name: "cluster_status", Labels: []string{"uuid", "name", "stack_id", "status", "node_count", "master_count"}, Fn: nil},
 }
 
-func NewContainerInfraExporter(client *gophercloud.ServiceClient, prefix string, disabledMetrics []string) (*ContainerInfraExporter, error) {
+func NewContainerInfraExporter(config *ExporterConfig) (*ContainerInfraExporter, error) {
  exporter := ContainerInfraExporter{
  BaseOpenStackExporter{
- Name: "container_infra",
- Prefix: prefix,
- Client: client,
- DisabledMetrics: disabledMetrics,
+ Name: "container_infra",
+ ExporterConfig: *config,
  },
  }
  for _, metric := range defaultContainerInfraMetrics {

exporters/designate.go

@@ -3,7 +3,6 @@ package exporters
 import (
  "strings"
 
- "github.com/gophercloud/gophercloud"
  "github.com/gophercloud/gophercloud/openstack/dns/v2/recordsets"
  "github.com/gophercloud/gophercloud/openstack/dns/v2/zones"
  "github.com/prometheus/client_golang/prometheus"
@@ -52,13 +51,11 @@ var defaultDesignateMetrics = []Metric{
  {Name: "recordsets_status", Labels: []string{"id", "name", "status", "zone_id", "zone_name", "type"}, Fn: nil},
 }
 
-func NewDesignateExporter(client *gophercloud.ServiceClient, prefix string, disabledMetrics []string) (*DesignateExporter, error) {
+func NewDesignateExporter(config *ExporterConfig) (*DesignateExporter, error) {
  exporter := DesignateExporter{
  BaseOpenStackExporter{
- Name: "designate",
- Prefix: prefix,
- Client: client,
- DisabledMetrics: disabledMetrics,
+ ExporterConfig: *config,
+ Name: "designate",
  },
  }
  for _, metric := range defaultDesignateMetrics {

exporters/exporter.go

@@ -4,6 +4,7 @@ import (
  "crypto/tls"
  "fmt"
  "net/http"
+ "time"
 
  "github.com/gophercloud/gophercloud"
  "github.com/gophercloud/utils/openstack/clientconfig"
@@ -36,8 +37,8 @@ type OpenStackExporter interface {
  MetricIsDisabled(name string) bool
 }
 
-func EnableExporter(service, prefix, cloud string, disabledMetrics []string, endpointType string) (*OpenStackExporter, error) {
- exporter, err := NewExporter(service, prefix, cloud, disabledMetrics, endpointType)
+func EnableExporter(service, prefix, cloud string, disabledMetrics []string, endpointType string, collectTime bool) (*OpenStackExporter, error) {
+ exporter, err := NewExporter(service, prefix, cloud, disabledMetrics, endpointType, collectTime)
  if err != nil {
  return nil, err
  }
@@ -50,12 +51,17 @@ type PrometheusMetric struct {
  Fn ListFunc
 }
 
-type BaseOpenStackExporter struct {
- Name string
- Prefix string
- Metrics map[string]*PrometheusMetric
+type ExporterConfig struct {
  Client *gophercloud.ServiceClient
+ Prefix string
  DisabledMetrics []string
+ CollectTime bool
+}
+
+type BaseOpenStackExporter struct {
+ ExporterConfig
+ Name string
+ Metrics map[string]*PrometheusMetric
 }
 
 type ListFunc func(exporter *BaseOpenStackExporter, ch chan<- prometheus.Metric) error
@@ -81,28 +87,55 @@ func (exporter *BaseOpenStackExporter) Describe(ch chan<- *prometheus.Desc) {
  }
 }
 
+func (exporter *BaseOpenStackExporter) AddMetricCollectTime(collectTimeSeconds float64, metricName string, ch chan<- prometheus.Metric) {
+ metricPromatheuslabels := prometheus.Labels{
+ "openstack_service": exporter.GetName(),
+ "openstack_metric": metricName}
+ metric := prometheus.NewDesc(
+ "openstack_metric_collect_seconds",
+ "Time needed to collect metric from OpenStack API",
+ nil,
+ metricPromatheuslabels)
+ log.Debugf("Adding metric for collecting timings: %+s", metric)
+ ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, collectTimeSeconds)
+}
+
+func (exporter *BaseOpenStackExporter) RunCollection(metric *PrometheusMetric, metricName string, ch chan<- prometheus.Metric) error {
+ log.Infof("Collecting metrics for exporter: %s, metric: %s", exporter.GetName(), metricName)
+ now := time.Now()
+ if exporter.CollectTime {
+ defer exporter.AddMetricCollectTime(time.Since(now).Seconds(), metricName, ch)
+ }
+ err := metric.Fn(exporter, ch)
+ if err != nil {
+ return fmt.Errorf("failed to collect metric: %s, error: %s", metricName, err)
+ }
+
+ log.Infof("Collected metrics for exporter: %s, metric: %s", exporter.GetName(), metricName)
+ return nil
+}
+
 func (exporter *BaseOpenStackExporter) Collect(ch chan<- prometheus.Metric) {
- serviceUp := true
+ serviceDown := 0
 
  for name, metric := range exporter.Metrics {
- log.Infof("Collecting metrics for exporter: %s, metric: %s", exporter.GetName(), name)
  if metric.Fn == nil {
  log.Debugf("No function handler set for metric: %s", name)
  continue
  }
 
- err := metric.Fn(exporter, ch)
- if err != nil {
- log.Errorln(err)
- serviceUp = false
+ if err := exporter.RunCollection(metric, name, ch); err != nil {
+ log.Errorf("Failed to collect metric for exporter: %s, error: %s", exporter.Name, err)
+ serviceDown++
  }
  }
 
- if serviceUp {
- ch <- prometheus.MustNewConstMetric(exporter.Metrics["up"].Metric, prometheus.GaugeValue, 1)
- } else {
+ if serviceDown > 0 {
  ch <- prometheus.MustNewConstMetric(exporter.Metrics["up"].Metric, prometheus.GaugeValue, 0)
+ } else {
+ ch <- prometheus.MustNewConstMetric(exporter.Metrics["up"].Metric, prometheus.GaugeValue, 1)
  }
+
 }
 
 func (exporter *BaseOpenStackExporter) AddMetric(name string, fn ListFunc, labels []string, constLabels prometheus.Labels) {
@@ -139,7 +172,7 @@ func (exporter *BaseOpenStackExporter) AddMetric(name string, fn ListFunc, label
  }
 }
 
-func NewExporter(name, prefix, cloud string, disabledMetrics []string, endpointType string) (OpenStackExporter, error) {
+func NewExporter(name, prefix, cloud string, disabledMetrics []string, endpointType string, collectTime bool) (OpenStackExporter, error) {
  var exporter OpenStackExporter
  var err error
  var transport *http.Transport
@@ -162,66 +195,73 @@ func NewExporter(name, prefix, cloud string, disabledMetrics []string, endpointT
  return nil, err
  }
 
+ exporterConfig := ExporterConfig{
+ Client: client,
+ Prefix: prefix,
+ DisabledMetrics: disabledMetrics,
+ CollectTime: collectTime,
+ }
+
  switch name {
  case "network":
  {
- exporter, err = NewNeutronExporter(client, prefix, disabledMetrics)
+ exporter, err = NewNeutronExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "compute":
  {
- exporter, err = NewNovaExporter(client, prefix, disabledMetrics)
+ exporter, err = NewNovaExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "image":
  {
- exporter, err = NewGlanceExporter(client, prefix, disabledMetrics)
+ exporter, err = NewGlanceExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "volume":
  {
- exporter, err = NewCinderExporter(client, prefix, disabledMetrics)
+ exporter, err = NewCinderExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "identity":
  {
- exporter, err = NewKeystoneExporter(client, prefix, disabledMetrics)
+ exporter, err = NewKeystoneExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "object-store":
  {
- exporter, err = NewObjectStoreExporter(client, prefix, disabledMetrics)
+ exporter, err = NewObjectStoreExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "load-balancer":
  {
- exporter, err = NewLoadbalancerExporter(client, prefix, disabledMetrics)
+ exporter, err = NewLoadbalancerExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "container-infra":
  {
- exporter, err = NewContainerInfraExporter(client, prefix, disabledMetrics)
+ exporter, err = NewContainerInfraExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }
  }
  case "dns":
  {
- exporter, err = NewDesignateExporter(client, prefix, disabledMetrics)
+ exporter, err = NewDesignateExporter(&exporterConfig)
  if err != nil {
  return nil, err
  }

exporters/exporter_test.go

@@ -106,7 +106,8 @@ func (suite *BaseOpenStackTestSuite) SetupTest() {
  suite.installFixtures()
 
  os.Setenv("OS_CLIENT_CONFIG_FILE", path.Join(baseFixturePath, "test_config.yaml"))
- exporter, err := NewExporter(suite.ServiceName, suite.Prefix, cloudName, []string{}, "public")
+
+ exporter, err := NewExporter(suite.ServiceName, suite.Prefix, cloudName, []string{}, "public", false)
  if err != nil {
  panic(err)
  }

exporters/glance.go

@@ -1,7 +1,6 @@
 package exporters
 
 import (
- "github.com/gophercloud/gophercloud"
  "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
  "github.com/prometheus/client_golang/prometheus"
 )
@@ -14,13 +13,11 @@ var defaultGlanceMetrics = []Metric{
  {Name: "images", Fn: ListImages},
 }
 
-func NewGlanceExporter(client *gophercloud.ServiceClient, prefix string, disabledMetrics []string) (*GlanceExporter, error) {
+func NewGlanceExporter(config *ExporterConfig) (*GlanceExporter, error) {
  exporter := GlanceExporter{
  BaseOpenStackExporter{
- Name: "glance",
- Prefix: prefix,
- Client: client,
- DisabledMetrics: disabledMetrics,
+ Name: "glance",
+ ExporterConfig: *config,
  },
  }
 

exporters/keystone.go

@@ -1,7 +1,6 @@
 package exporters
 
 import (
- "github.com/gophercloud/gophercloud"
  "github.com/gophercloud/gophercloud/openstack/identity/v3/domains"
  "github.com/gophercloud/gophercloud/openstack/identity/v3/groups"
  "github.com/gophercloud/gophercloud/openstack/identity/v3/projects"
@@ -22,13 +21,11 @@ var defaultKeystoneMetrics = []Metric{
  {Name: "regions", Fn: ListRegions},
 }
 
-func NewKeystoneExporter(client *gophercloud.ServiceClient, prefix string, disabledMetrics []string) (*KeystoneExporter, error) {
+func NewKeystoneExporter(config *ExporterConfig) (*KeystoneExporter, error) {
  exporter := KeystoneExporter{
  BaseOpenStackExporter{
- Name: "identity",
- Prefix: prefix,
- Client: client,
- DisabledMetrics: disabledMetrics,
+ Name: "identity",
+ ExporterConfig: *config,
  },
  }
 

exporters/loadbalancer.go

@@ -1,7 +1,6 @@
 package exporters
 
 import (
- "github.com/gophercloud/gophercloud"
  "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/amphorae"
  "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
  "github.com/prometheus/client_golang/prometheus"
@@ -58,13 +57,11 @@ var defaultLoadbalancerMetrics = []Metric{
  {Name: "amphora_status", Labels: []string{"id", "loadbalancer_id", "compute_id", "status", "role", "lb_network_ip", "ha_ip"}},
 }
 
-func NewLoadbalancerExporter(client *gophercloud.ServiceClient, prefix string, disabledMetrics []string) (*LoadbalancerExporter, error) {
+func NewLoadbalancerExporter(config *ExporterConfig) (*LoadbalancerExporter, error) {
  exporter := LoadbalancerExporter{
  BaseOpenStackExporter{
- Name: "loadbalancer",
- Prefix: prefix,
- Client: client,
- DisabledMetrics: disabledMetrics,
+ Name: "loadbalancer",
+ ExporterConfig: *config,
  },
  }
  for _, metric := range defaultLoadbalancerMetrics {