[exporter/datadog] Use exact sum and count on distributions when avai…

…lable (open-telemetry#7830)

* [exporter/datadog] Modify sketches tests so that they only use MapMetrics

* [exporter/datadog] Use exact sum, count and average on sketches

* Add Changelog entry

* Empty commit to re-trigger CI

Co-authored-by: Alex Boten <[email protected]>

CHANGELOG.md

@@ -14,6 +14,7 @@
 - `awscloudwatchlogsexporter`: Remove name from aws cloudwatch logs exporter (#7554)
 - `hostreceiver/memoryscraper`: Add memory.utilization (#6221)
 - `elasticsearchexporter`: Remove usage of deprecated LogRecord.Name field (#7829).
+- `datadogexporter`: Use exact sum, count and average on Datadog distributions (#7830)
 
 ### 🛑 Breaking changes 🛑
 

exporter/datadogexporter/internal/model/translator/metrics_translator.go

@@ -158,11 +158,21 @@ func getBounds(p pdata.HistogramDataPoint, idx int) (lowerBound float64, upperBo
  return
 }
 
+type histogramInfo struct {
+ // sum of histogram (exact)
+ sum float64
+ // count of histogram (exact)
+ count uint64
+ // ok to use
+ ok bool
+}
+
 func (t *Translator) getSketchBuckets(
  ctx context.Context,
  consumer SketchConsumer,
  pointDims metricsDimensions,
  p pdata.HistogramDataPoint,
+ histInfo histogramInfo,
  delta bool,
 ) {
  startTs := uint64(p.StartTimestamp())
@@ -199,6 +209,12 @@ func (t *Translator) getSketchBuckets(
 
  sketch := as.Finish()
  if sketch != nil {
+ if histInfo.ok {
+ // override approximate sum, count and average in sketch with exact values if available.
+ sketch.Basic.Cnt = int64(histInfo.count)
+ sketch.Basic.Sum = histInfo.sum
+ sketch.Basic.Avg = sketch.Basic.Sum / float64(sketch.Basic.Cnt)
+ }
  consumer.ConsumeSketch(ctx, pointDims.name, ts, sketch, pointDims.tags, pointDims.host)
  }
 }
@@ -257,33 +273,41 @@ func (t *Translator) mapHistogramMetrics(
  ts := uint64(p.Timestamp())
  pointDims := dims.WithAttributeMap(p.Attributes())
 
- if t.cfg.SendCountSum {
- count := float64(p.Count())
- countDims := pointDims.WithSuffix("count")
+ histInfo := histogramInfo{ok: true}
+
+ countDims := pointDims.WithSuffix("count")
+ if delta {
+ histInfo.count = p.Count()
+ } else if dx, ok := t.prevPts.Diff(countDims, startTs, ts, float64(p.Count())); ok {
+ histInfo.count = uint64(dx)
+ } else { // not ok
+ histInfo.ok = false
+ }
+
+ sumDims := pointDims.WithSuffix("sum")
+ if !t.isSkippable(sumDims.name, p.Sum()) {
  if delta {
- consumer.ConsumeTimeSeries(ctx, countDims.name, Count, ts, count, countDims.tags, countDims.host)
- } else if dx, ok := t.prevPts.Diff(countDims, startTs, ts, count); ok {
- consumer.ConsumeTimeSeries(ctx, countDims.name, Count, ts, dx, countDims.tags, countDims.host)
+ histInfo.sum = p.Sum()
+ } else if dx, ok := t.prevPts.Diff(sumDims, startTs, ts, p.Sum()); ok {
+ histInfo.sum = dx
+ } else { // not ok
+ histInfo.ok = false
  }
+ } else { // skippable
+ histInfo.ok = false
  }
 
- if t.cfg.SendCountSum {
- sum := p.Sum()
- sumDims := pointDims.WithSuffix("sum")
- if !t.isSkippable(sumDims.name, p.Sum()) {
- if delta {
- consumer.ConsumeTimeSeries(ctx, sumDims.name, Count, ts, sum, sumDims.tags, sumDims.host)
- } else if dx, ok := t.prevPts.Diff(sumDims, startTs, ts, sum); ok {
- consumer.ConsumeTimeSeries(ctx, sumDims.name, Count, ts, dx, sumDims.tags, sumDims.host)
- }
- }
+ if t.cfg.SendCountSum && histInfo.ok {
+ // We only send the sum and count if both values were ok.
+ consumer.ConsumeTimeSeries(ctx, countDims.name, Count, ts, float64(histInfo.count), countDims.tags, countDims.host)
+ consumer.ConsumeTimeSeries(ctx, sumDims.name, Count, ts, histInfo.sum, sumDims.tags, sumDims.host)
  }
 
  switch t.cfg.HistMode {
  case HistogramModeCounters:
  t.getLegacyBuckets(ctx, consumer, pointDims, p, delta)
  case HistogramModeDistributions:
- t.getSketchBuckets(ctx, consumer, pointDims, p, delta)
+ t.getSketchBuckets(ctx, consumer, pointDims, p, histInfo, delta)
  }
  }
 }

exporter/datadogexporter/internal/model/translator/metrics_translator_test.go

@@ -563,18 +563,18 @@ func TestMapDeltaHistogramMetrics(t *testing.T) {
  newSketch(dims, uint64(ts), summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
- Cnt: 20,
+ Sum: point.Sum(),
+ Avg: point.Sum() / float64(point.Count()),
+ Cnt: int64(point.Count()),
  }),
  }
 
  sketchesAttributeTags := []sketch{
  newSketch(dimsTags, uint64(ts), summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
+ Sum: point.Sum(),
+ Avg: point.Sum() / float64(point.Count()),
  Cnt: 20,
  }),
  }
@@ -716,8 +716,8 @@ func TestMapCumulativeHistogramMetrics(t *testing.T) {
  newSketch(dims, uint64(seconds(2)), summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
+ Sum: 20,
+ Avg: 20.0 / 30.0,
  Cnt: 30,
  }),
  }
@@ -1183,8 +1183,8 @@ func TestMapMetrics(t *testing.T) {
  newSketchWithHostname("double.histogram", summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
+ Sum: math.Phi,
+ Avg: math.Phi / 20,
  Cnt: 20,
  }, attrTags),
  },
@@ -1213,8 +1213,8 @@ func TestMapMetrics(t *testing.T) {
  newSketchWithHostname("double.histogram", summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
+ Sum: math.Phi,
+ Avg: math.Phi / 20.0,
  Cnt: 20,
  }, attrTags),
  },
@@ -1243,8 +1243,8 @@ func TestMapMetrics(t *testing.T) {
  newSketchWithHostname("double.histogram", summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
+ Sum: math.Phi,
+ Avg: math.Phi / 20,
  Cnt: 20,
  }, append(attrTags, ilTags...)),
  },
@@ -1273,8 +1273,8 @@ func TestMapMetrics(t *testing.T) {
  newSketchWithHostname("double.histogram", summary.Summary{
  Min: 0,
  Max: 0,
- Sum: 0,
- Avg: 0,
+ Sum: math.Phi,
+ Avg: math.Phi / 20,
  Cnt: 20,
  }, append(attrTags, ilTags...)),
  },
@@ -1428,5 +1428,5 @@ func TestNaNMetrics(t *testing.T) {
  })
 
  // One metric type was unknown or unsupported
- assert.Equal(t, observed.FilterMessage("Unsupported metric value").Len(), 6)
+ assert.Equal(t, observed.FilterMessage("Unsupported metric value").Len(), 7)
 }