[pkg/inframetadata] Add support for metrics for host metadata

.chloggen/mx-psi_metrics-support.yaml

@@ -0,0 +1,21 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component (e.g. pkg/quantile)
+component: pkg/otlp/metrics
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for system metrics in host metadata
+
+# The PR related to this change
+issues: [184]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: | 
+ The following metrics are now added as host metadata if present:
+ - `system.cpu.physical.count`
+ - `system.cpu.logical.count`
+ - `system.cpu.frequency`
+ - `system.memory.limit`

LICENSE-3rdparty.csv

@@ -334,14 +334,20 @@ pkg/inframetadata,github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attribute
 pkg/inframetadata,github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source,UNKNOWN,"Copyright [2023-Present] Datadog, Inc."
 pkg/inframetadata,github.com/davecgh/go-spew/spew,ISC,Copyright (c) 2012-2016 Dave Collins <[email protected]>
 pkg/inframetadata,github.com/gogo/protobuf/gogoproto,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors. All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
+pkg/inframetadata,github.com/gogo/protobuf/jsonpb,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors. All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
 pkg/inframetadata,github.com/gogo/protobuf/proto,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors. All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
 pkg/inframetadata,github.com/gogo/protobuf/protoc-gen-gogo/descriptor,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors. All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
+pkg/inframetadata,github.com/gogo/protobuf/sortkeys,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors. All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
+pkg/inframetadata,github.com/gogo/protobuf/types,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors. All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
 pkg/inframetadata,github.com/golang/protobuf/jsonpb,BSD-3-Clause,Copyright 2010 The Go Authors. All rights reserved
 pkg/inframetadata,github.com/golang/protobuf/proto,BSD-3-Clause,Copyright 2010 The Go Authors. All rights reserved
 pkg/inframetadata,github.com/golang/protobuf/ptypes,BSD-3-Clause,Copyright 2010 The Go Authors. All rights reserved
 pkg/inframetadata,github.com/golang/protobuf/ptypes/any,BSD-3-Clause,Copyright 2010 The Go Authors. All rights reserved
 pkg/inframetadata,github.com/golang/protobuf/ptypes/duration,BSD-3-Clause,Copyright 2010 The Go Authors. All rights reserved
 pkg/inframetadata,github.com/golang/protobuf/ptypes/timestamp,BSD-3-Clause,Copyright 2010 The Go Authors. All rights reserved
+pkg/inframetadata,github.com/json-iterator/go,MIT,Copyright (c) 2016 json-iterator
+pkg/inframetadata,github.com/modern-go/concurrent,Apache-2.0,Copyright (c) 2018 Tao Wen
+pkg/inframetadata,github.com/modern-go/reflect2,Apache-2.0,Copyright (c) 2018 Tao Wen
 pkg/inframetadata,github.com/pmezard/go-difflib/difflib,BSD-3-Clause,"Copyright (c) 2013, Patrick Mezard"
 pkg/inframetadata,github.com/stretchr/testify/assert,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"
 pkg/inframetadata,github.com/stretchr/testify/require,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"
@@ -355,7 +361,10 @@ pkg/inframetadata,go.opentelemetry.io/collector/pdata/internal/data/protogen/log
 pkg/inframetadata,go.opentelemetry.io/collector/pdata/internal/data/protogen/metrics/v1,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/inframetadata,go.opentelemetry.io/collector/pdata/internal/data/protogen/resource/v1,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/inframetadata,go.opentelemetry.io/collector/pdata/internal/data/protogen/trace/v1,Apache-2.0,Copyright The OpenTelemetry Authors
+pkg/inframetadata,go.opentelemetry.io/collector/pdata/internal/json,Apache-2.0,Copyright The OpenTelemetry Authors
+pkg/inframetadata,go.opentelemetry.io/collector/pdata/internal/otlp,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/inframetadata,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyright The OpenTelemetry Authors
+pkg/inframetadata,go.opentelemetry.io/collector/pdata/pmetric,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/inframetadata,go.opentelemetry.io/collector/semconv/v1.18.0,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/inframetadata,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/inframetadata,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc"

pkg/inframetadata/go.mod

@@ -15,6 +15,9 @@ require (
  github.com/davecgh/go-spew v1.1.1 // indirect
  github.com/gogo/protobuf v1.3.2 // indirect
  github.com/golang/protobuf v1.5.3 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
  go.uber.org/atomic v1.7.0 // indirect
  golang.org/x/net v0.18.0 // indirect

pkg/inframetadata/go.sum

@@ -9,10 +9,18 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

pkg/inframetadata/internal/hostmap/constants.go

@@ -56,14 +56,26 @@ const (
  attributeHostCPUCacheL2Size = "host.cpu.cache.l2.size"
 )
 
+// CPU related OpenTelemetry Semantic Conventions for metrics.
+// TODO: Replace by conventions constants once available.
+const (
+ metricSystemCPUPhysicalCount = "system.cpu.physical.count"
+ metricSystemCPULogicalCount = "system.cpu.logical.count"
+ metricSystemCPUFrequency = "system.cpu.frequency"
+ metricSystemMemoryLimit = "system.memory.limit"
+)
+
 // This set of constants represent fields in the Gohai payload's CPU field.
 const (
- fieldCPUVendorID = "vendor_id"
- fieldCPUModelName = "model_name"
- fieldCPUCacheSize = "cache_size"
- fieldCPUFamily = "family"
- fieldCPUModel = "model"
- fieldCPUStepping = "stepping"
+ fieldCPUVendorID = "vendor_id"
+ fieldCPUModelName = "model_name"
+ fieldCPUCacheSize = "cache_size"
+ fieldCPUFamily = "family"
+ fieldCPUModel = "model"
+ fieldCPUStepping = "stepping"
+ fieldCPUCores = "cpu_cores"
+ fieldCPULogicalProcessors = "cpu_logical_processors"
+ fieldCPUMHz = "mhz"
 )
 
 // cpuAttributesMap defines the mapping between Gohai fieldCPU fields
@@ -77,6 +89,25 @@ var cpuAttributesMap map[string]string = map[string]string{
  fieldCPUStepping: attributeHostCPUStepping,
 }
 
+type cpuMetricsData struct {
+ FieldName string
+ ConversionFactor float64
+}
+
+var cpuMetricsMap map[string]cpuMetricsData = map[string]cpuMetricsData{
+ metricSystemCPUPhysicalCount: {FieldName: fieldCPUCores},
+ metricSystemCPULogicalCount: {FieldName: fieldCPULogicalProcessors},
+ metricSystemCPUFrequency: {FieldName: fieldCPUMHz, ConversionFactor: 1e-6},
+}
+
+// TrackedMetrics is the set of metrics that are tracked by the hostmap.
+var TrackedMetrics map[string]struct{} = map[string]struct{}{
+ metricSystemCPUPhysicalCount: {},
+ metricSystemCPULogicalCount: {},
+ metricSystemCPUFrequency: {},
+ metricSystemMemoryLimit: {},
+}
+
 // Network related OpenTelemetry Semantic Conventions for resource attributes.
 // TODO: Replace by conventions constants once available.
 const (

pkg/inframetadata/internal/hostmap/hostmap.go

@@ -11,6 +11,7 @@ import (
  "sync"
 
  "go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/pmetric"
  conventions "go.opentelemetry.io/collector/semconv/v1.18.0"
  "go.uber.org/multierr"
 
@@ -158,6 +159,21 @@ func (m *HostMap) Set(md payload.HostMetadata) error {
  return nil
 }
 
+// newOrFetchHostMetadata returns the host metadata payload for a given host or creates a new one.
+// This method is NOT thread-safe and should be called while holding the m.mu mutex.
+func (m *HostMap) newOrFetchHostMetadata(host string) (payload.HostMetadata, bool) {
+ md, ok := m.hosts[host]
+ if !ok {
+ md = payload.HostMetadata{
+ Flavor: "otelcol-contrib",
+ Meta: &payload.Meta{},
+ Tags: &payload.HostTags{},
+ Payload: gohai.NewEmpty(),
+ }
+ }
+ return md, ok
+}
+
 // Update the information about a given host by providing a resource.
 // The function reports:
 // - Whether the information about the `host` has changed
@@ -172,21 +188,9 @@ func (m *HostMap) Set(md payload.HostMetadata) error {
 // The order in which resource attributes are read does not affect the final
 // host metadata payload, even if non-fatal errors are raised during execution.
 func (m *HostMap) Update(host string, res pcommon.Resource) (changed bool, md payload.HostMetadata, err error) {
- md = payload.HostMetadata{
- Flavor: "otelcol-contrib",
- Meta: &payload.Meta{},
- Tags: &payload.HostTags{},
- Payload: gohai.NewEmpty(),
- }
  m.mu.Lock()
  defer m.mu.Unlock()
-
- var found bool
- if old, ok := m.hosts[host]; ok {
- found = true
- md = old
- }
-
+ md, found := m.newOrFetchHostMetadata(host)
  md.InternalHostname = host
  md.Meta.Hostname = host
 
@@ -276,6 +280,50 @@ func (m *HostMap) Update(host string, res pcommon.Resource) (changed bool, md pa
  return
 }
 
+func (m *HostMap) UpdateFromMetric(host string, metric pmetric.Metric) {
+ // Take last available point
+ var datapoints pmetric.NumberDataPointSlice
+ switch metric.Type() {
+ case pmetric.MetricTypeGauge:
+ datapoints = metric.Gauge().DataPoints()
+ case pmetric.MetricTypeSum:
+ datapoints = metric.Sum().DataPoints()
+ default:
+ return // unsupported type
+ }
+ nPoints := datapoints.Len()
+ if nPoints == 0 {
+ return // no points
+ }
+ lastIndex := nPoints - 1
+ point := datapoints.At(lastIndex)
+
+ // Take value from point
+ var value float64
+ switch point.ValueType() {
+ case pmetric.NumberDataPointValueTypeInt:
+ value = float64(point.IntValue())
+ case pmetric.NumberDataPointValueTypeDouble:
+ value = point.DoubleValue()
+ default:
+ // unsupported type
+ return
+ }
+
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ md, _ := m.newOrFetchHostMetadata(host)
+
+ // Gohai - CPU
+ data, ok := cpuMetricsMap[metric.Name()]
+ if ok {
+ if data.ConversionFactor != 0 {
+ value = value * data.ConversionFactor
+ }
+ md.CPU()[data.FieldName] = fmt.Sprintf("%g", value)
+ }
+}
+
 // Flush all the host metadata payloads and clear them from the HostMap.
 func (m *HostMap) Flush() map[string]payload.HostMetadata {
  m.mu.Lock()