Fix nano timestamp precision: Issue #16546 (#16568)

1. The current timestamp used in the collector is RFC3339, this causes truncation and makes it hard when much more granular precision of time is needed. This is changed to RFC3339Nano 2. Updates underlying SDK to interact with the database Co-authored-by: Abhishek Saharn <[email protected]>
codeboten · Dec 1, 2022 · 089e5e7 · 089e5e7
1 parent 31064e1
commit 089e5e7
Show file tree

Hide file tree

Showing 14 changed files with 43 additions and 27 deletions.
diff --git a/.chloggen/change-tsprecision-adx-receiver.yaml b/.chloggen/change-tsprecision-adx-receiver.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/azuredataexplorerexporter
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Makes timestamp precision to nanos, updated azure-kusto-go to 0.9.2
+
+# One or more tracking issues related to the change
+issues: [16546]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod
@@ -37,7 +37,7 @@ require (
  github.com/AthenZ/athenz v1.10.39 // indirect
  github.com/Azure/azure-amqp-common-go/v3 v3.2.3 // indirect
  github.com/Azure/azure-event-hubs-go/v3 v3.3.19 // indirect
- github.com/Azure/azure-kusto-go v0.8.1 // indirect
+ github.com/Azure/azure-kusto-go v0.9.2 // indirect
  github.com/Azure/azure-pipeline-go v0.2.3 // indirect
  github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect
  github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect

diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum
diff --git a/exporter/azuredataexplorerexporter/README.md b/exporter/azuredataexplorerexporter/README.md
@@ -122,11 +122,11 @@ This exporter maps OpenTelemetry [trace](https://opentelemetry.io/docs/referenc
 The following tables need to be created in the database specified in the configuration.
 
 ```kql
-.create-merge table <Logs-Table-Name> (Timestamp:datetime, ObservedTimestamp:datetime, TraceId:string, SpanId:string, SeverityText:string, SeverityNumber:int, Body:string, ResourceAttributes:dynamic, LogsAttributes:dynamic) 
+.create-merge table <Logs-Table-Name> (Timestamp:datetime, ObservedTimestamp:datetime, TraceID:string, SpanID:string, SeverityText:string, SeverityNumber:int, Body:string, ResourceAttributes:dynamic, LogsAttributes:dynamic) 
 
 .create-merge table <Metrics-Table-Name> (Timestamp:datetime, MetricName:string, MetricType:string, MetricUnit:string, MetricDescription:string, MetricValue:real, Host:string, ResourceAttributes:dynamic,MetricAttributes:dynamic) 
 
-.create-merge table <Traces-Table-Name> (TraceId:string, SpanId:string, ParentId:string, SpanName:string, SpanStatus:string, SpanKind:string, StartTime:datetime, EndTime:datetime, ResourceAttributes:dynamic, TraceAttributes:dynamic, Events:dynamic, Links:dynamic) 
+.create-merge table <Traces-Table-Name> (TraceID:string, SpanID:string, ParentID:string, SpanName:string, SpanStatus:string, SpanKind:string, StartTime:datetime, EndTime:datetime, ResourceAttributes:dynamic, TraceAttributes:dynamic, Events:dynamic, Links:dynamic) 
 
 //Enable streaming ingestion( for managed streaming) for the created tables using
 .alter table <Table-Name> policy streamingingestion enable

diff --git a/exporter/azuredataexplorerexporter/go.mod b/exporter/azuredataexplorerexporter/go.mod
@@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azured
 go 1.18
 
 require (
- github.com/Azure/azure-kusto-go v0.8.1
+ github.com/Azure/azure-kusto-go v0.9.2
  github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
  github.com/json-iterator/go v1.1.12
  github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.66.0

diff --git a/exporter/azuredataexplorerexporter/go.sum b/exporter/azuredataexplorerexporter/go.sum
diff --git a/exporter/azuredataexplorerexporter/logsdata_to_adx.go b/exporter/azuredataexplorerexporter/logsdata_to_adx.go
@@ -25,8 +25,8 @@ import (
 )
 
 type AdxLog struct {
- Timestamp string // The timestamp of the occurrence. Formatted into string as RFC3339
- ObservedTimestamp string // The timestamp of logs observed in opentelemetry collector. Formatted into string as RFC3339
+ Timestamp string // The timestamp of the occurrence. Formatted into string as RFC3339Nano
+ ObservedTimestamp string // The timestamp of logs observed in opentelemetry collector. Formatted into string as RFC3339Nano
  TraceID string // TraceId associated to the log
  SpanID string // SpanId associated to the log
  SeverityText string // The severity level of the log
@@ -43,8 +43,8 @@ func mapToAdxLog(resource pcommon.Resource, scope pcommon.InstrumentationScope,
  clonedLogAttrib := cloneMap(logAttrib)
  copyMap(clonedLogAttrib, getScopeMap(scope))
  adxLog := &AdxLog{
- Timestamp: logData.Timestamp().AsTime().Format(time.RFC3339),
- ObservedTimestamp: logData.ObservedTimestamp().AsTime().Format(time.RFC3339),
+ Timestamp: logData.Timestamp().AsTime().Format(time.RFC3339Nano),
+ ObservedTimestamp: logData.ObservedTimestamp().AsTime().Format(time.RFC3339Nano),
  TraceID: traceutil.TraceIDToHexOrEmptyString(logData.TraceID()),
  SpanID: traceutil.SpanIDToHexOrEmptyString(logData.SpanID()),
  SeverityText: logData.SeverityText(),

diff --git a/exporter/azuredataexplorerexporter/logsdata_to_adx_test.go b/exporter/azuredataexplorerexporter/logsdata_to_adx_test.go
@@ -28,8 +28,8 @@ import (
 
 func Test_mapToAdxLog(t *testing.T) {
  logger := zap.NewNop()
- epoch, _ := time.Parse("2006-01-02T15:04:05Z07:00", "1970-01-01T00:00:00Z")
- defaultTime := pcommon.NewTimestampFromTime(epoch).AsTime().Format(time.RFC3339)
+ epoch, _ := time.Parse("2006-01-02T15:04:05.999999999Z07:00", "1970-01-01T00:00:00.000000000Z")
+ defaultTime := pcommon.NewTimestampFromTime(epoch).AsTime().Format(time.RFC3339Nano)
  tmap := make(map[string]interface{})
  tmap["key"] = "value"
  tmap[hostkey] = testhost

diff --git a/exporter/azuredataexplorerexporter/metricsdata_to_adx.go b/exporter/azuredataexplorerexporter/metricsdata_to_adx.go
@@ -44,7 +44,7 @@ const (
 
 // This is derived from the specification https://opentelemetry.io/docs/reference/specification/metrics/datamodel/
 type AdxMetric struct {
- Timestamp string // The timestamp of the occurrence. A metric is measured at a point of time. Formatted into string as RFC3339
+ Timestamp string // The timestamp of the occurrence. A metric is measured at a point of time. Formatted into string as RFC3339Nano
  // Including name, the Metric object is defined by the following properties:
  MetricName string // Name of the metric field
  MetricType string // The data point type (e.g. Sum, Gauge, Histogram ExponentialHistogram, Summary)
@@ -81,7 +81,7 @@ func mapToAdxMetric(res pcommon.Resource, md pmetric.Metric, scopeattrs map[stri
  desc = md.Description()
  }
  return &AdxMetric{
- Timestamp: times.Format(time.RFC3339),
+ Timestamp: times.Format(time.RFC3339Nano),
  MetricName: name,
  MetricType: mt.String(),
  MetricUnit: md.Unit(),

diff --git a/exporter/azuredataexplorerexporter/metricsdata_to_adx_test.go b/exporter/azuredataexplorerexporter/metricsdata_to_adx_test.go
@@ -36,7 +36,7 @@ const (
 // The timestamps used for the tests
 var tsUnix = time.Unix(time.Now().Unix(), time.Now().UnixNano())
 var ts = pcommon.NewTimestampFromTime(tsUnix)
-var tstr = ts.AsTime().Format(time.RFC3339)
+var tstr = ts.AsTime().Format(time.RFC3339Nano)
 
 // the histogram values and distribution for the tests
 var distributionBounds = []float64{1, 2, 4}

diff --git a/exporter/azuredataexplorerexporter/tracesdata_to_adx.go b/exporter/azuredataexplorerexporter/tracesdata_to_adx.go
@@ -30,8 +30,8 @@ type AdxTrace struct {
  SpanName string // The SpanName of the Trace
  SpanStatus string // The SpanStatus associated to the Trace
  SpanKind string // The SpanKind of the Trace
- StartTime string // The start time of the occurrence. Formatted into string as RFC3339
- EndTime string // The end time of the occurrence. Formatted into string as RFC3339
+ StartTime string // The start time of the occurrence. Formatted into string as RFC3339Nano
+ EndTime string // The end time of the occurrence. Formatted into string as RFC3339Nano
  ResourceAttributes map[string]interface{} // JSON Resource attributes that can then be parsed.
  TraceAttributes map[string]interface{} // JSON attributes that can then be parsed.
  Events []*Event // Array containing the events in a span
@@ -64,8 +64,8 @@ func mapToAdxTrace(resource pcommon.Resource, scope pcommon.InstrumentationScope
  SpanName: spanData.Name(),
  SpanStatus: traceutil.StatusCodeStr(spanData.Status().Code()),
  SpanKind: traceutil.SpanKindStr(spanData.Kind()),
- StartTime: spanData.StartTimestamp().AsTime().Format(time.RFC3339),
- EndTime: spanData.EndTimestamp().AsTime().Format(time.RFC3339),
+ StartTime: spanData.StartTimestamp().AsTime().Format(time.RFC3339Nano),
+ EndTime: spanData.EndTimestamp().AsTime().Format(time.RFC3339Nano),
  ResourceAttributes: resource.Attributes().AsRaw(),
  TraceAttributes: clonedTraceAttrib,
  Events: getEventsData(spanData),
@@ -78,7 +78,7 @@ func getEventsData(sd ptrace.Span) []*Event {
 
  for i := 0; i < sd.Events().Len(); i++ {
  event := &Event{
- Timestamp: sd.Events().At(i).Timestamp().AsTime().Format(time.RFC3339),
+ Timestamp: sd.Events().At(i).Timestamp().AsTime().Format(time.RFC3339Nano),
  EventName: sd.Events().At(i).Name(),
  EventAttributes: sd.Events().At(i).Attributes().AsRaw(),
  }

diff --git a/exporter/azuredataexplorerexporter/tracesdata_to_adx_test.go b/exporter/azuredataexplorerexporter/tracesdata_to_adx_test.go
@@ -25,8 +25,8 @@ import (
 )
 
 func Test_mapToAdxTrace(t *testing.T) {
- epoch, _ := time.Parse("2006-01-02T15:04:05Z07:00", "1970-01-01T00:00:00Z")
- defaultTime := pcommon.NewTimestampFromTime(epoch).AsTime().Format(time.RFC3339)
+ epoch, _ := time.Parse("2006-01-02T15:04:05.999999999Z07:00", "1970-01-01T00:00:00.000000000Z")
+ defaultTime := pcommon.NewTimestampFromTime(epoch).AsTime().Format(time.RFC3339Nano)
  tmap := make(map[string]interface{})
  tmap["key"] = "value"
  tmap[hostkey] = testhost

diff --git a/go.mod b/go.mod
@@ -201,7 +201,7 @@ require (
  github.com/AthenZ/athenz v1.10.39 // indirect
  github.com/Azure/azure-amqp-common-go/v3 v3.2.3 // indirect
  github.com/Azure/azure-event-hubs-go/v3 v3.3.19 // indirect
- github.com/Azure/azure-kusto-go v0.8.1 // indirect
+ github.com/Azure/azure-kusto-go v0.9.2 // indirect
  github.com/Azure/azure-pipeline-go v0.2.3 // indirect
  github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect
  github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect

diff --git a/go.sum b/go.sum