From 6df57dca30b4f3101bb0019b8dc4afd45bfbb55a Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 25 Oct 2023 12:03:20 -0400 Subject: [PATCH] [exporter/datadog] Add config trace_buffer (#28582) **Description:** Add a new traces config `trace_buffer` that specifies the number of Datadog Agent TracerPayloads to buffer before dropping. **Link to tracking Issue:** Fixes #28577 **Testing:** **Documentation:** --- .chloggen/datadog-trace-buffer.yaml | 27 +++++++++++++++++++ exporter/datadogexporter/config.go | 4 +++ exporter/datadogexporter/config_test.go | 7 +++++ .../datadogexporter/examples/collector.yaml | 6 +++++ exporter/datadogexporter/factory_test.go | 1 + exporter/datadogexporter/testdata/config.yaml | 1 + exporter/datadogexporter/traces_exporter.go | 3 +++ 7 files changed, 49 insertions(+) create mode 100644 .chloggen/datadog-trace-buffer.yaml diff --git a/.chloggen/datadog-trace-buffer.yaml b/.chloggen/datadog-trace-buffer.yaml new file mode 100644 index 0000000000000..fed7baa9476d5 --- /dev/null +++ b/.chloggen/datadog-trace-buffer.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: datadogexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add a new traces config `trace_buffer` that specifies the number of outgoing trace payloads to buffer before dropping." + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [28577] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: "If you start seeing log messages like `Payload in channel full. Dropped 1 payload.` in the datadog exporter, consider setting a higher `trace_buffer` to avoid traces being dropped." + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/datadogexporter/config.go b/exporter/datadogexporter/config.go index 9787aa86f08c8..3d1c67c793078 100644 --- a/exporter/datadogexporter/config.go +++ b/exporter/datadogexporter/config.go @@ -279,6 +279,10 @@ type TracesConfig struct { // If the overhead remains high, it will be due to a high cardinality of `peer.service` values from the traces. You may need to check your instrumentation. PeerServiceAggregation bool `mapstructure:"peer_service_aggregation"` + // TraceBuffer specifies the number of Datadog Agent TracerPayloads to buffer before dropping. + // The default value is 0, meaning the Datadog Agent TracerPayloads are unbuffered. + TraceBuffer int `mapstructure:"trace_buffer"` + // flushInterval defines the interval in seconds at which the writer flushes traces // to the intake; used in tests. flushInterval float64 diff --git a/exporter/datadogexporter/config_test.go b/exporter/datadogexporter/config_test.go index e0f8575c90e0f..9bdabdfd2ba7a 100644 --- a/exporter/datadogexporter/config_test.go +++ b/exporter/datadogexporter/config_test.go @@ -101,6 +101,13 @@ func TestValidate(t *testing.T) { }, }, }, + { + name: "With trace_buffer", + cfg: &Config{ + API: APIConfig{Key: "notnull"}, + Traces: TracesConfig{TraceBuffer: 10}, + }, + }, } for _, testInstance := range tests { t.Run(testInstance.name, func(t *testing.T) { diff --git a/exporter/datadogexporter/examples/collector.yaml b/exporter/datadogexporter/examples/collector.yaml index f2bfb30d2626f..8a5c692ede111 100644 --- a/exporter/datadogexporter/examples/collector.yaml +++ b/exporter/datadogexporter/examples/collector.yaml @@ -340,6 +340,12 @@ exporters: # # peer_service_aggregation: true + ## @param trace_buffer - specifies the number of Datadog Agent TracerPayloads to buffer before dropping. + ## If unset, the default value is 0, meaning the Datadog Agent TracerPayloads are unbuffered. + ## If you start seeing log messages like `Payload in channel full. Dropped 1 payload.` in the datadog exporter, consider + ## setting a higher `trace_buffer` to avoid traces being dropped. + # trace_buffer: 10 + ## @param host_metadata - custom object - optional ## Host metadata specific configuration. ## Host metadata is the information used for populating the infrastructure list, the host map and providing host tags functionality within the Datadog app. diff --git a/exporter/datadogexporter/factory_test.go b/exporter/datadogexporter/factory_test.go index 2bfa0a6d96b7a..4f440810414a0 100644 --- a/exporter/datadogexporter/factory_test.go +++ b/exporter/datadogexporter/factory_test.go @@ -217,6 +217,7 @@ func TestLoadConfig(t *testing.T) { }, SpanNameAsResourceName: true, IgnoreResources: []string{}, + TraceBuffer: 10, }, Logs: LogsConfig{ TCPAddr: confignet.TCPAddr{ diff --git a/exporter/datadogexporter/testdata/config.yaml b/exporter/datadogexporter/testdata/config.yaml index cc9f2167329a5..10fea4ec91c06 100644 --- a/exporter/datadogexporter/testdata/config.yaml +++ b/exporter/datadogexporter/testdata/config.yaml @@ -10,6 +10,7 @@ datadog/api: "old_name1": "new_name1" "old_name2": "new_name2" span_name_as_resource_name: true + trace_buffer: 10 datadog/api2: hostname: customhostname diff --git a/exporter/datadogexporter/traces_exporter.go b/exporter/datadogexporter/traces_exporter.go index 7eb800052a3b7..5ce3084be205f 100644 --- a/exporter/datadogexporter/traces_exporter.go +++ b/exporter/datadogexporter/traces_exporter.go @@ -194,6 +194,9 @@ func newTraceAgent(ctx context.Context, params exporter.CreateSettings, cfg *Con if v := cfg.Traces.flushInterval; v > 0 { acfg.TraceWriter.FlushPeriodSeconds = v } + if v := cfg.Traces.TraceBuffer; v > 0 { + acfg.TraceBuffer = v + } if addr := cfg.Traces.Endpoint; addr != "" { acfg.Endpoints[0].Host = addr }