Allow customers to set custom resource attributes as container Tags (#…

…32224) Currently users of DDconnector can only set fixed set of resource attributes as container Tags , list is defined [here](https://github.com/DataDog/opentelemetry-mapping-go/blob/ba4072f84888749a00f5748cca9f2a7fc2e42fc9/pkg/otlp/attributes/attributes.go#L41) . This PR enables the customers to choose any resource attribute to be set as container Tag on the APM Trace metrics payload. --------- Co-authored-by: Pablo Baeyens <[email protected]>
open-telemetry · Apr 10, 2024 · 867d670 · 867d670
1 parent 2a8dfd3
commit 867d670
Show file tree

Hide file tree

Showing 15 changed files with 72 additions and 42 deletions.
diff --git a/.chloggen/dinesh.gurumurthy_expand-resource-attributes.yaml b/.chloggen/dinesh.gurumurthy_expand-resource-attributes.yaml
@@ -0,0 +1,23 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadog/connector
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Enable connector to use any attribute from the resource as Container Tag
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [32224]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+ This change allows the connector to use any attribute from the resource as a container tag. This is useful when you want to use a custom attribute from the resource as a container tag. For example, you can use the `namespace` attribute from the resource as a container tag.
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []
diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod
@@ -25,7 +25,7 @@ require (
  github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 // indirect
  github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect
  github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
- github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.13.4 // indirect
+ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.13.4 // indirect
  github.com/gocql/gocql v1.6.0 // indirect
  github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect

diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum
diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod
@@ -284,7 +284,7 @@ require (
  github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
  github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.13.4 // indirect
- github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.13.4 // indirect
+ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.13.4 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.13.4 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.13.4 // indirect

diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum
diff --git a/connector/datadogconnector/connector.go b/connector/datadogconnector/connector.go
@@ -40,7 +40,7 @@ type traceToMetricConnector struct {
  // from the agent to OTLP Metrics.
  translator *metrics.Translator
 
- resourceAttrs map[string]string
+ enrichedTags  map[string]string
  containerTagCache *cache.Cache
 
  // in specifies the channel through which the agent will output Stats Payloads
@@ -79,18 +79,20 @@ func newTraceToMetricConnector(set component.TelemetrySettings, cfg component.Co
 
  ctags := make(map[string]string, len(cfg.(*Config).Traces.ResourceAttributesAsContainerTags))
  for _, val := range cfg.(*Config).Traces.ResourceAttributesAsContainerTags {
- ctags[val] = ""
+ if v, ok := attributes.ContainerMappings[val]; ok {
+ ctags[v] = ""
+ } else {
+ ctags[val] = ""
+ }
  }
- ddtags := attributes.ContainerTagFromAttributes(ctags)
-
  ctx := context.Background()
  return &traceToMetricConnector{
  logger: set.Logger,
  agent: datadog.NewAgentWithConfig(ctx, getTraceAgentCfg(cfg.(*Config).Traces, attributesTranslator), in, metricsClient, timingReporter),
  translator: trans,
  in: in,
  metricsConsumer: metricsConsumer,
- resourceAttrs: ddtags,
+ enrichedTags:  ctags,
  containerTagCache: cache.New(cacheExpiration, cacheCleanupInterval),
  exit: make(chan struct{}),
  }, nil
@@ -146,31 +148,34 @@ func (c *traceToMetricConnector) Capabilities() consumer.Capabilities {
  return consumer.Capabilities{MutatesData: false}
 }
 
+func (c *traceToMetricConnector) addToCache(containerID string, key string) {
+ if tags, ok := c.containerTagCache.Get(containerID); ok {
+ tagList := tags.(*sync.Map)
+ tagList.Store(key, struct{}{})
+ } else {
+ tagList := &sync.Map{}
+ tagList.Store(key, struct{}{})
+ c.containerTagCache.Set(containerID, tagList, cache.DefaultExpiration)
+ }
+}
+
 func (c *traceToMetricConnector) populateContainerTagsCache(traces ptrace.Traces) {
  for i := 0; i < traces.ResourceSpans().Len(); i++ {
  rs := traces.ResourceSpans().At(i)
  attrs := rs.Resource().Attributes()
+
  containerID, ok := attrs.Get(semconv.AttributeContainerID)
  if !ok {
  continue
  }
  ddContainerTags := attributes.ContainerTagsFromResourceAttributes(attrs)
- for attr := range c.resourceAttrs {
+ for attr := range c.enrichedTags {
  if val, ok := ddContainerTags[attr]; ok {
  key := fmt.Sprintf("%s:%s", attr, val)
- if v, ok := c.containerTagCache.Get(containerID.AsString()); ok {
- cacheVal := v.(*sync.Map)
- // check if the key already exists in the cache
- if _, ok := cacheVal.Load(key); ok {
- continue
- }
- cacheVal.Store(key, struct{}{})
- } else {
- cacheVal := &sync.Map{}
- cacheVal.Store(key, struct{}{})
- c.containerTagCache.Set(containerID.AsString(), cacheVal, cache.DefaultExpiration)
- }
-
+ c.addToCache(containerID.AsString(), key)
+ } else if incomingVal, ok := attrs.Get(attr); ok {
+ key := fmt.Sprintf("%s:%s", attr, incomingVal.Str())
+ c.addToCache(containerID.AsString(), key)
  }
  }
  }
@@ -214,7 +219,7 @@ func (c *traceToMetricConnector) run() {
  var mx pmetric.Metrics
  var err error
  // Enrich the stats with container tags
- if len(c.resourceAttrs) > 0 {
+ if len(c.enrichedTags) > 0 {
  c.enrichStatsPayload(stats)
  }
 

diff --git a/connector/datadogconnector/connector_test.go b/connector/datadogconnector/connector_test.go
@@ -68,6 +68,8 @@ func generateTrace() ptrace.Traces {
  res.Attributes().PutStr("container.id", "my-container-id")
  res.Attributes().PutStr("cloud.availability_zone", "my-zone")
  res.Attributes().PutStr("cloud.region", "my-region")
+ // add a custom Resource attribute
+ res.Attributes().PutStr("az", "my-az")
 
  ss := td.ResourceSpans().At(0).ScopeSpans().AppendEmpty().Spans()
  ss.EnsureCapacity(1)
@@ -103,7 +105,7 @@ func creteConnector(t *testing.T) (*traceToMetricConnector, *consumertest.Metric
 
  creationParams := connectortest.NewNopCreateSettings()
  cfg := factory.CreateDefaultConfig().(*Config)
- cfg.Traces.ResourceAttributesAsContainerTags = []string{semconv.AttributeCloudAvailabilityZone, semconv.AttributeCloudRegion}
+ cfg.Traces.ResourceAttributesAsContainerTags = []string{semconv.AttributeCloudAvailabilityZone, semconv.AttributeCloudRegion, "az"}
 
  metricsSink := &consumertest.MetricsSink{}
 
@@ -142,7 +144,7 @@ func TestContainerTags(t *testing.T) {
  count++
  return true
  })
- assert.Equal(t, 2, count)
+ assert.Equal(t, 3, count)
 
  for {
  if len(metricsSink.AllMetrics()) > 0 {
@@ -166,8 +168,8 @@ func TestContainerTags(t *testing.T) {
  require.NoError(t, err)
 
  tags := sp.Stats[0].Tags
- assert.Equal(t, 2, len(tags))
- assert.ElementsMatch(t, []string{"region:my-region", "zone:my-zone"}, tags)
+ assert.Equal(t, 3, len(tags))
+ assert.ElementsMatch(t, []string{"region:my-region", "zone:my-zone", "az:my-az"}, tags)
 }
 
 func newTranslatorWithStatsChannel(t *testing.T, logger *zap.Logger, ch chan []byte) *otlpmetrics.Translator {

diff --git a/connector/datadogconnector/go.mod b/connector/datadogconnector/go.mod
@@ -6,7 +6,7 @@ require (
  github.com/DataDog/datadog-agent/pkg/proto v0.52.1-0.20240321095122-a3c5dbb936ae
  github.com/DataDog/datadog-agent/pkg/trace v0.52.1-0.20240321095122-a3c5dbb936ae
  github.com/DataDog/datadog-go/v5 v5.5.0
- github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.13.4
+ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.13.4
  github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter v0.97.0
  github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.97.0

diff --git a/connector/datadogconnector/go.sum b/connector/datadogconnector/go.sum
diff --git a/exporter/datadogexporter/go.mod b/exporter/datadogexporter/go.mod
@@ -10,7 +10,7 @@ require (
  github.com/DataDog/datadog-go/v5 v5.5.0
  github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee
  github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.13.4
- github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.13.4
+ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.13.4
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.13.4
  github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.13.4

diff --git a/exporter/datadogexporter/go.sum b/exporter/datadogexporter/go.sum
diff --git a/exporter/datadogexporter/integrationtest/go.mod b/exporter/datadogexporter/integrationtest/go.mod
@@ -44,7 +44,7 @@ require (
  github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
  github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.13.4 // indirect
- github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.13.4 // indirect
+ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.13.4 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.13.4 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.13.4 // indirect

diff --git a/exporter/datadogexporter/integrationtest/go.sum b/exporter/datadogexporter/integrationtest/go.sum
diff --git a/go.mod b/go.mod
@@ -248,7 +248,7 @@ require (
  github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
  github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.13.4 // indirect
- github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.13.4 // indirect
+ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.13.4 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.13.4 // indirect
  github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.13.4 // indirect

diff --git a/go.sum b/go.sum