[exporter/elasticsearch] add missing scope info in span/log attributes (

open-telemetry#27288)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->

**Link to tracking Issue:**
open-telemetry#27282

**Testing:** <Describe what testing was performed and which tests were
added.>

**Documentation:** <Describe the documentation added.>

---------

Signed-off-by: Jared Tan <[email protected]>
Signed-off-by: Dominik Rosiek <[email protected]>
Signed-off-by: Raphael Silva <[email protected]>
Signed-off-by: Alex Boten <[email protected]>
Signed-off-by: Pavol Loffay <[email protected]>
Co-authored-by: Antoine Toulme <[email protected]>
Co-authored-by: Raj Nishtala <[email protected]>
Co-authored-by: Dominik Rosiek <[email protected]>
Co-authored-by: Povilas Versockas <[email protected]>
Co-authored-by: Priyanshu Raj <[email protected]>
Co-authored-by: sakulali <[email protected]>
Co-authored-by: Raphael Philipe Mendes da Silva <[email protected]>
Co-authored-by: Anthony Mirabella <[email protected]>
Co-authored-by: Yotam loewenbach <[email protected]>
Co-authored-by: Alex Boten <[email protected]>
Co-authored-by: bryan-aguilar <[email protected]>
Co-authored-by: Daniel Jaglowski <[email protected]>
Co-authored-by: Jina Jain <[email protected]>
Co-authored-by: Yang Song <[email protected]>
Co-authored-by: Dmitrii Anoshin <[email protected]>
Co-authored-by: Curtis Robert <[email protected]>
Co-authored-by: Abhishek Saharn <[email protected]>
Co-authored-by: Ramachandran A G <[email protected]>
Co-authored-by: Ziqi Zhao <[email protected]>
Co-authored-by: Ramachandran A G <[email protected]>
Co-authored-by: Faith Chikwekwe <[email protected]>
Co-authored-by: Tyler Helmuth <[email protected]>
Co-authored-by: Daniel Kuiper <[email protected]>
Co-authored-by: Carlos Castro <[email protected]>
Co-authored-by: Christian <[email protected]>
Co-authored-by: ArchangelSDY <[email protected]>
Co-authored-by: Pavol Loffay <[email protected]>
Co-authored-by: Paulo Janotti <[email protected]>
Co-authored-by: Nathan Burke <[email protected]>
Co-authored-by: VihasMakwana <[email protected]>
Co-authored-by: shalper2 <[email protected]>
Co-authored-by: OpenTelemetry Bot <[email protected]>
Co-authored-by: Martin Majlis <[email protected]>
Co-authored-by: hovavza <[email protected]>

.chloggen/elasticsearchexporter_add_missing_scope_info_in_span.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "add missing scope info in span attributes"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [27282]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

cmd/otelcontribcol/exporters_test.go

@@ -83,6 +83,7 @@ func TestDefaultExporters(t *testing.T) {
  cfg := expFactories["awscloudwatchlogs"].CreateDefaultConfig().(*awscloudwatchlogsexporter.Config)
  cfg.Endpoint = "https://" + endpoint
  cfg.Region = "local"
+
  // disable queue/retry to validate passing the test data synchronously
  cfg.QueueSettings.Enabled = false
  cfg.RetrySettings.Enabled = false

exporter/elasticsearchexporter/logs_exporter.go

@@ -83,9 +83,10 @@ func (e *elasticsearchLogsExporter) pushLogsData(ctx context.Context, ld plog.Lo
  resource := rl.Resource()
  ills := rl.ScopeLogs()
  for j := 0; j < ills.Len(); j++ {
+ scope := ills.At(j).Scope()
  logs := ills.At(j).LogRecords()
  for k := 0; k < logs.Len(); k++ {
- if err := e.pushLogRecord(ctx, resource, logs.At(k)); err != nil {
+ if err := e.pushLogRecord(ctx, resource, logs.At(k), scope); err != nil {
  if cerr := ctx.Err(); cerr != nil {
  return cerr
  }
@@ -99,7 +100,7 @@ func (e *elasticsearchLogsExporter) pushLogsData(ctx context.Context, ld plog.Lo
  return errors.Join(errs...)
 }
 
-func (e *elasticsearchLogsExporter) pushLogRecord(ctx context.Context, resource pcommon.Resource, record plog.LogRecord) error {
+func (e *elasticsearchLogsExporter) pushLogRecord(ctx context.Context, resource pcommon.Resource, record plog.LogRecord, scope pcommon.InstrumentationScope) error {
  fIndex := e.index
  if e.dynamicIndex {
  prefix := getFromBothResourceAndAttribute(indexPrefix, resource, record)
@@ -108,7 +109,7 @@ func (e *elasticsearchLogsExporter) pushLogRecord(ctx context.Context, resource
  fIndex = fmt.Sprintf("%s%s%s", prefix, fIndex, suffix)
  }
 
- document, err := e.model.encodeLog(resource, record)
+ document, err := e.model.encodeLog(resource, record, scope)
  if err != nil {
  return fmt.Errorf("Failed to encode log event: %w", err)
  }

exporter/elasticsearchexporter/logs_exporter_test.go

@@ -403,9 +403,12 @@ func mustSend(t *testing.T, exporter *elasticsearchLogsExporter, contents string
 // send trace with span & resource attributes
 func mustSendLogsWithAttributes(t *testing.T, exporter *elasticsearchLogsExporter, attrMp map[string]string, resMp map[string]string) {
  logs := newLogsWithAttributeAndResourceMap(attrMp, resMp)
- resSpans := logs.ResourceLogs().At(0)
- logRecords := resSpans.ScopeLogs().At(0).LogRecords().At(0)
+ resLogs := logs.ResourceLogs().At(0)
+ logRecords := resLogs.ScopeLogs().At(0).LogRecords().At(0)
 
- err := exporter.pushLogRecord(context.TODO(), resSpans.Resource(), logRecords)
+ scopeLogs := resLogs.ScopeLogs().AppendEmpty()
+ scope := scopeLogs.Scope()
+
+ err := exporter.pushLogRecord(context.TODO(), resLogs.Resource(), logRecords, scope)
  require.NoError(t, err)
 }

exporter/elasticsearchexporter/model.go

@@ -17,8 +17,8 @@ import (
 )
 
 type mappingModel interface {
- encodeLog(pcommon.Resource, plog.LogRecord) ([]byte, error)
- encodeSpan(pcommon.Resource, ptrace.Span) ([]byte, error)
+ encodeLog(pcommon.Resource, plog.LogRecord, pcommon.InstrumentationScope) ([]byte, error)
+ encodeSpan(pcommon.Resource, ptrace.Span, pcommon.InstrumentationScope) ([]byte, error)
 }
 
 // encodeModel tries to keep the event as close to the original open telemetry semantics as is.
@@ -38,7 +38,7 @@ const (
  attributeField = "attribute"
 )
 
-func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord) ([]byte, error) {
+func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord, scope pcommon.InstrumentationScope) ([]byte, error) {
  var document objmodel.Document
  document.AddTimestamp("@timestamp", record.Timestamp()) // We use @timestamp in order to ensure that we can index if the default data stream logs template is used.
  document.AddTraceID("TraceId", record.TraceID())
@@ -49,6 +49,7 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord
  document.AddAttribute("Body", record.Body())
  document.AddAttributes("Attributes", record.Attributes())
  document.AddAttributes("Resource", resource.Attributes())
+ document.AddAttributes("Scope", scopeToAttributes(scope))
 
  if m.dedup {
  document.Dedup()
@@ -61,7 +62,7 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord
  return buf.Bytes(), err
 }
 
-func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span) ([]byte, error) {
+func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, scope pcommon.InstrumentationScope) ([]byte, error) {
  var document objmodel.Document
  document.AddTimestamp("@timestamp", span.StartTimestamp()) // We use @timestamp in order to ensure that we can index if the default data stream logs template is used.
  document.AddTimestamp("EndTimestamp", span.EndTimestamp())
@@ -76,6 +77,7 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span) ([
  document.AddAttributes("Resource", resource.Attributes())
  document.AddEvents("Events", span.Events())
  document.AddInt("Duration", durationAsMicroseconds(span.StartTimestamp().AsTime(), span.EndTimestamp().AsTime())) // unit is microseconds
+ document.AddAttributes("Scope", scopeToAttributes(scope))
 
  if m.dedup {
  document.Dedup()
@@ -107,3 +109,13 @@ func spanLinksToString(spanLinkSlice ptrace.SpanLinkSlice) string {
 func durationAsMicroseconds(start, end time.Time) int64 {
  return (end.UnixNano() - start.UnixNano()) / 1000
 }
+
+func scopeToAttributes(scope pcommon.InstrumentationScope) pcommon.Map {
+ attrs := pcommon.NewMap()
+ attrs.PutStr("name", scope.Name())
+ attrs.PutStr("version", scope.Version())
+ for k, v := range scope.Attributes().AsRaw() {
+ attrs.PutStr(k, v.(string))
+ }
+ return attrs
+}

exporter/elasticsearchexporter/model_test.go

@@ -13,12 +13,12 @@ import (
  semconv "go.opentelemetry.io/collector/semconv/v1.18.0"
 )
 
-var expectedSpanBody = `{"@timestamp":"2023-04-19T03:04:05.000000006Z","Attributes.service.instance.id":"23","Duration":1000000,"EndTimestamp":"2023-04-19T03:04:06.000000006Z","Events.fooEvent.evnetMockBar":"bar","Events.fooEvent.evnetMockFoo":"foo","Events.fooEvent.time":"2023-04-19T03:04:05.000000006Z","Kind":"SPAN_KIND_CLIENT","Link":"[{\"attribute\":{},\"spanID\":\"\",\"traceID\":\"01020304050607080807060504030200\"}]","Name":"client span","Resource.cloud.platform":"aws_elastic_beanstalk","Resource.cloud.provider":"aws","Resource.deployment.environment":"BETA","Resource.service.instance.id":"23","Resource.service.name":"some-service","Resource.service.version":"env-version-1234","SpanId":"1920212223242526","TraceId":"01020304050607080807060504030201","TraceStatus":0}`
+var expectedSpanBody = `{"@timestamp":"2023-04-19T03:04:05.000000006Z","Attributes.service.instance.id":"23","Duration":1000000,"EndTimestamp":"2023-04-19T03:04:06.000000006Z","Events.fooEvent.evnetMockBar":"bar","Events.fooEvent.evnetMockFoo":"foo","Events.fooEvent.time":"2023-04-19T03:04:05.000000006Z","Kind":"SPAN_KIND_CLIENT","Link":"[{\"attribute\":{},\"spanID\":\"\",\"traceID\":\"01020304050607080807060504030200\"}]","Name":"client span","Resource.cloud.platform":"aws_elastic_beanstalk","Resource.cloud.provider":"aws","Resource.deployment.environment":"BETA","Resource.service.instance.id":"23","Resource.service.name":"some-service","Resource.service.version":"env-version-1234","Scope.lib-foo":"lib-bar","Scope.name":"io.opentelemetry.rabbitmq-2.7","Scope.version":"1.30.0-alpha","SpanId":"1920212223242526","TraceId":"01020304050607080807060504030201","TraceStatus":0}`
 
 func TestEncodeSpan(t *testing.T) {
  model := &encodeModel{dedup: true, dedot: false}
  td := mockResourceSpans()
- spanByte, err := model.encodeSpan(td.ResourceSpans().At(0).Resource(), td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0))
+ spanByte, err := model.encodeSpan(td.ResourceSpans().At(0).Resource(), td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0), td.ResourceSpans().At(0).ScopeSpans().At(0).Scope())
  assert.NoError(t, err)
  assert.Equal(t, expectedSpanBody, string(spanByte))
 }
@@ -40,6 +40,10 @@ func mockResourceSpans() ptrace.Traces {
  tEnd := time.Date(2023, 4, 19, 3, 4, 6, 6, time.UTC)
 
  scopeSpans := resourceSpans.ScopeSpans().AppendEmpty()
+ scopeSpans.Scope().SetName("io.opentelemetry.rabbitmq-2.7")
+ scopeSpans.Scope().SetVersion("1.30.0-alpha")
+ scopeSpans.Scope().Attributes().PutStr("lib-foo", "lib-bar")
+
  span := scopeSpans.Spans().AppendEmpty()
  span.SetName("client span")
  span.SetSpanID([8]byte{0x19, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26})

exporter/elasticsearchexporter/trace_exporter.go

@@ -76,9 +76,10 @@ func (e *elasticsearchTracesExporter) pushTraceData(
  resource := il.Resource()
  scopeSpans := il.ScopeSpans()
  for j := 0; j < scopeSpans.Len(); j++ {
+ scope := scopeSpans.At(j).Scope()
  spans := scopeSpans.At(j).Spans()
  for k := 0; k < spans.Len(); k++ {
- if err := e.pushTraceRecord(ctx, resource, spans.At(k)); err != nil {
+ if err := e.pushTraceRecord(ctx, resource, spans.At(k), scope); err != nil {
  if cerr := ctx.Err(); cerr != nil {
  return cerr
  }
@@ -91,7 +92,7 @@ func (e *elasticsearchTracesExporter) pushTraceData(
  return errors.Join(errs...)
 }
 
-func (e *elasticsearchTracesExporter) pushTraceRecord(ctx context.Context, resource pcommon.Resource, span ptrace.Span) error {
+func (e *elasticsearchTracesExporter) pushTraceRecord(ctx context.Context, resource pcommon.Resource, span ptrace.Span, scope pcommon.InstrumentationScope) error {
  fIndex := e.index
  if e.dynamicIndex {
  prefix := getFromBothResourceAndAttribute(indexPrefix, resource, span)
@@ -100,7 +101,7 @@ func (e *elasticsearchTracesExporter) pushTraceRecord(ctx context.Context, resou
  fIndex = fmt.Sprintf("%s%s%s", prefix, fIndex, suffix)
  }
 
- document, err := e.model.encodeSpan(resource, span)
+ document, err := e.model.encodeSpan(resource, span, scope)
  if err != nil {
  return fmt.Errorf("Failed to encode trace record: %w", err)
  }

exporter/elasticsearchexporter/traces_exporter_test.go

@@ -404,7 +404,8 @@ func mustSendTracesWithAttributes(t *testing.T, exporter *elasticsearchTracesExp
  traces := newTracesWithAttributeAndResourceMap(attrMp, resMp)
  resSpans := traces.ResourceSpans().At(0)
  span := resSpans.ScopeSpans().At(0).Spans().At(0)
+ scope := resSpans.ScopeSpans().At(0).Scope()
 
- err := exporter.pushTraceRecord(context.TODO(), resSpans.Resource(), span)
+ err := exporter.pushTraceRecord(context.TODO(), resSpans.Resource(), span, scope)
  require.NoError(t, err)
 }