From 6aac96df083146495e2dca3171b252a20097b0d6 Mon Sep 17 00:00:00 2001 From: taloric Date: Mon, 27 Jun 2022 14:33:07 +0000 Subject: [PATCH 1/7] Fix skywalking traceid and spanid convertion. --- .../skywalkingproto_to_traces.go | 101 ++++++++++++------ .../skywalkingproto_to_traces_test.go | 83 ++++++++++++++ 2 files changed, 150 insertions(+), 34 deletions(-) diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go index b12e5b0768824..27b5db16c214f 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go @@ -16,9 +16,9 @@ package skywalkingreceiver // import "github.com/open-telemetry/opentelemetry-co import ( "encoding/binary" - "reflect" + "encoding/hex" + "strings" "time" - "unsafe" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" @@ -60,7 +60,7 @@ func SkywalkingToTraces(segment *agentV3.SegmentObject) ptrace.Traces { } il := resourceSpan.ScopeSpans().AppendEmpty() - swSpansToSpanSlice(segment.GetTraceId(), swSpans, il.Spans()) + swSpansToSpanSlice(segment.GetTraceId(), segment.GetTraceSegmentId(), swSpans, il.Spans()) return traceData } @@ -86,7 +86,7 @@ func swTagsToInternalResource(span *agentV3.SpanObject, dest pcommon.Resource) { } } -func swSpansToSpanSlice(traceID string, spans []*agentV3.SpanObject, dest ptrace.SpanSlice) { +func swSpansToSpanSlice(traceID string, segmentID string, spans []*agentV3.SpanObject, dest ptrace.SpanSlice) { if len(spans) == 0 { return } @@ -96,17 +96,19 @@ func swSpansToSpanSlice(traceID string, spans []*agentV3.SpanObject, dest ptrace if span == nil { continue } - swSpanToSpan(traceID, span, dest.AppendEmpty()) + swSpanToSpan(traceID, segmentID, span, dest.AppendEmpty()) } } -func swSpanToSpan(traceID string, span *agentV3.SpanObject, dest ptrace.Span) { +func swSpanToSpan(traceID string, segmentID string, span *agentV3.SpanObject, dest ptrace.Span) { dest.SetTraceID(stringToTraceID(traceID)) - dest.SetSpanID(uInt32ToSpanID(uint32(span.GetSpanId()))) + // skywalking defines segmentId + spanId as unique identifier + // so use segmentId to convert to an unique otel-span + dest.SetSpanID(segmentIDToSpanID(segmentID, uint32(span.GetSpanId()))) // parent spanid = -1, means(root span) no parent span in skywalking,so just make otlp's parent span id empty. if span.ParentSpanId != -1 { - dest.SetParentSpanID(uInt32ToSpanID(uint32(span.GetParentSpanId()))) + dest.SetParentSpanID(segmentIDToSpanID(segmentID, uint32(span.GetParentSpanId()))) } dest.SetName(span.OperationName) @@ -155,7 +157,7 @@ func swReferencesToSpanLinks(refs []*agentV3.SegmentReference, dest ptrace.SpanL for _, ref := range refs { link := dest.AppendEmpty() link.SetTraceID(stringToTraceID(ref.TraceId)) - link.SetSpanID(stringToParentSpanID(ref.ParentTraceSegmentId)) + link.SetSpanID(segmentIDToSpanID(ref.ParentTraceSegmentId, uint32(ref.ParentSpanId))) link.SetTraceState("") kvParis := []*common.KeyStringValuePair{ { @@ -236,37 +238,68 @@ func microsecondsToTimestamp(ms int64) pcommon.Timestamp { } func stringToTraceID(traceID string) pcommon.TraceID { - return pcommon.NewTraceID(unsafeStringToBytes(traceID)) + dst, err := stringTo16Bytes(&traceID) + if err != nil { + return pcommon.InvalidTraceID() + } + return pcommon.NewTraceID(dst) } -func stringToParentSpanID(traceID string) pcommon.SpanID { - return pcommon.NewSpanID(unsafeStringTo8Bytes(traceID)) +func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { + if i := strings.LastIndexByte(segmentID, '.'); i >= 0 && i+1 < len(segmentID) { + segmentID = segmentID[i+1:] + } + segments, err := stringTo8Bytes(&segmentID) + if err != nil { + return pcommon.InvalidSpanID() + } + binary.PutUvarint(segments[:], uint64(spanID)) + return pcommon.NewSpanID(segments) } -// uInt32ToSpanID converts the uint64 representation of a SpanID to pcommon.SpanID. -func uInt32ToSpanID(id uint32) pcommon.SpanID { - spanID := [8]byte{} - binary.BigEndian.PutUint32(spanID[:], id) - return pcommon.NewSpanID(spanID) +func stringTo16Bytes(s *string) ([16]byte, error) { + var dst [16]byte + var mid [32]byte + h := stringToHexBytes(s) + copy(mid[:], h) + _, err := hex.Decode(dst[:], mid[:]) + if err != nil { + return dst, err + } + return dst, nil } -func unsafeStringToBytes(s string) [16]byte { - p := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&s)).Data) - - var b [16]byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - hdr.Data = uintptr(p) - hdr.Cap = len(s) - hdr.Len = len(s) - return b +func stringTo8Bytes(s *string) ([8]byte, error) { + var dst [8]byte + var mid [16]byte + copy(mid[:], ([]byte(*s))) + _, err := hex.Decode(dst[:], mid[:]) + if err != nil { + return dst, err + } + return dst, nil } -func unsafeStringTo8Bytes(s string) [8]byte { - p := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&s)).Data) - var b [8]byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - hdr.Data = uintptr(p) - hdr.Cap = len(s) - hdr.Len = len(s) - return b +// hex table: +// 48-57: 0-9 +// 65-70: a-f +// 97-102: A-F +func stringToHexBytes(s *string) []byte { + src := make([]byte, 0, len(*s)) + for i := 0; i < len(*s); i++ { + r := (*s)[i] + switch { + case r < 48: + fallthrough + case r > 57 && r < 65: + fallthrough + case r > 70 && r < 97: + fallthrough + case r > 102: + // if 'r' is not a hex digit, drop it + continue + } + src = append(src, r) + } + return src } diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go index f96b5d0f5cfaa..154907430f1a9 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go @@ -160,6 +160,89 @@ func TestSwLogsToSpanEvents(t *testing.T) { }) } } + +func Test_stringToTraceID(t *testing.T) { + type args struct { + traceID string + } + tests := []struct { + name string + segmentObject args + want [16]byte + }{ + { + name: "mock-sw-normal-trace-id", + segmentObject: args{traceID: "de5980b8-fce3-4a37-aab9-b4ac3af7eedd"}, + want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, + }, + { + name: "mock-sw-normal-trace-id", + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd"}, + want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, + }, + { + name: "mock-sw-trace-id-length-shorter", + segmentObject: args{traceID: "de59"}, + want: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + { + name: "mock-sw-trace-id-length-overflow", + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eeddddddde5980"}, + want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, + }, + { + name: "mock-sw-trace-id-illegal", + segmentObject: args{traceID: ".,<>?/-=+MNop"}, + want: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stringToTraceID(tt.segmentObject.traceID) + assert.Equal(t, tt.want, got.Bytes()) + }) + } +} + +func Test_segmentIdToSpanId(t *testing.T) { + type args struct { + segmentID string + spanID uint32 + } + tests := []struct { + name string + args args + want [8]byte + }{ + { + name: "mock-sw-span-id-normal", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 123}, + want: [8]byte{123, 86, 6, 7, 54, 153, 80, 6}, + }, + { + name: "mock-sw-span-id-short", + args: args{segmentID: "16560607369950066", spanID: 12}, + want: [8]byte{12, 86, 6, 7, 54, 153, 80, 6}, + }, + { + name: "mock-sw-span-id-illegal-1", + args: args{segmentID: "1", spanID: 2}, + want: [8]byte{0, 0, 0, 0, 0, 0, 0, 0}, + }, + { + name: "mock-sw-span-id-illegal-char", + args: args{segmentID: ".,<>?/-=+MNop", spanID: 2}, + want: [8]byte{0, 0, 0, 0, 0, 0, 0, 0}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := segmentIDToSpanID(tt.args.segmentID, tt.args.spanID) + assert.Equal(t, tt.want, got.Bytes()) + }) + } +} + func generateTracesOneEmptyResourceSpans() ptrace.Span { td := ptrace.NewTraces() resourceSpan := td.ResourceSpans().AppendEmpty() From 2d1a940978295262e49766716c1b66b93b6975ef Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 27 Jun 2022 13:28:26 -0700 Subject: [PATCH 2/7] Update CHANGELOG.md --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d47dccabcbef..bdcbcd843ddd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,16 @@ - `transformprocessor`: replace_pattern` and `replace_all_patterns` use regex for pattern matching and replacing text in attributes/metrics (#11125) ### 🧰 Bug fixes 🧰 +<<<<<<< HEAD +======= + +- `redactionprocessor`: respect allow_all_keys configuration (#11542) +- `skywalkingreceiver`: Fix skywalking traceid and spanid convertion. (#11562) + +### Unmaintained components + +- `simpleprometheusreceiver`(#11133) +>>>>>>> Update CHANGELOG.md - `aerospikereceiver`: Fix issue where namespaces would not be collected (#11465) - `aerospikereceiver`: Fix typo in metric name. Ensure namespace transactions are collected (#12085) (#12083) - `coralogixexporter`: Fix metrics bearer token (#11831) From cbfb141d08b637b56382b1e413c11b2f6cc2ae53 Mon Sep 17 00:00:00 2001 From: taloric Date: Tue, 28 Jun 2022 11:33:40 +0000 Subject: [PATCH 3/7] use uuid.Parse instead of hextable to convert id --- receiver/skywalkingreceiver/go.mod | 1 + receiver/skywalkingreceiver/go.sum | 2 + .../skywalkingproto_to_traces.go | 44 +++---------------- .../skywalkingproto_to_traces_test.go | 2 +- 4 files changed, 10 insertions(+), 39 deletions(-) diff --git a/receiver/skywalkingreceiver/go.mod b/receiver/skywalkingreceiver/go.mod index c44445a9d7e6b..ee729d9b19f61 100644 --- a/receiver/skywalkingreceiver/go.mod +++ b/receiver/skywalkingreceiver/go.mod @@ -24,6 +24,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/knadh/koanf v1.4.2 // indirect diff --git a/receiver/skywalkingreceiver/go.sum b/receiver/skywalkingreceiver/go.sum index 385ae55823127..35547bf2b7281 100644 --- a/receiver/skywalkingreceiver/go.sum +++ b/receiver/skywalkingreceiver/go.sum @@ -191,6 +191,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go index 27b5db16c214f..c19e8c26c73e2 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/google/uuid" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" conventions "go.opentelemetry.io/collector/semconv/v1.8.0" @@ -238,11 +239,14 @@ func microsecondsToTimestamp(ms int64) pcommon.Timestamp { } func stringToTraceID(traceID string) pcommon.TraceID { - dst, err := stringTo16Bytes(&traceID) + if i := strings.IndexByte(traceID, '.'); i >= 0 { + traceID = traceID[:i] + } + uid, err := uuid.Parse(traceID) if err != nil { return pcommon.InvalidTraceID() } - return pcommon.NewTraceID(dst) + return pcommon.NewTraceID(uid) } func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { @@ -257,18 +261,6 @@ func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { return pcommon.NewSpanID(segments) } -func stringTo16Bytes(s *string) ([16]byte, error) { - var dst [16]byte - var mid [32]byte - h := stringToHexBytes(s) - copy(mid[:], h) - _, err := hex.Decode(dst[:], mid[:]) - if err != nil { - return dst, err - } - return dst, nil -} - func stringTo8Bytes(s *string) ([8]byte, error) { var dst [8]byte var mid [16]byte @@ -279,27 +271,3 @@ func stringTo8Bytes(s *string) ([8]byte, error) { } return dst, nil } - -// hex table: -// 48-57: 0-9 -// 65-70: a-f -// 97-102: A-F -func stringToHexBytes(s *string) []byte { - src := make([]byte, 0, len(*s)) - for i := 0; i < len(*s); i++ { - r := (*s)[i] - switch { - case r < 48: - fallthrough - case r > 57 && r < 65: - fallthrough - case r > 70 && r < 97: - fallthrough - case r > 102: - // if 'r' is not a hex digit, drop it - continue - } - src = append(src, r) - } - return src -} diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go index 154907430f1a9..a7bae86985b1c 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go @@ -187,7 +187,7 @@ func Test_stringToTraceID(t *testing.T) { }, { name: "mock-sw-trace-id-length-overflow", - segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eeddddddde5980"}, + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.1.16563474296430001"}, want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, }, { From 393b4c2ae85817f97f4464df18bbbcd28bace798 Mon Sep 17 00:00:00 2001 From: taloric Date: Fri, 1 Jul 2022 10:01:54 +0000 Subject: [PATCH 4/7] [receiver/skywalking] fix skywalking-rcv tid convertion --- CHANGELOG.md | 14 +-- receiver/skywalkingreceiver/go.mod | 2 +- .../skywalkingproto_to_traces.go | 118 ++++++++++++++---- .../skywalkingproto_to_traces_test.go | 79 ++++++++++-- 4 files changed, 170 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcbcd843ddd0..ed17e4f5e94c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,16 +64,6 @@ - `transformprocessor`: replace_pattern` and `replace_all_patterns` use regex for pattern matching and replacing text in attributes/metrics (#11125) ### 🧰 Bug fixes 🧰 -<<<<<<< HEAD -======= - -- `redactionprocessor`: respect allow_all_keys configuration (#11542) -- `skywalkingreceiver`: Fix skywalking traceid and spanid convertion. (#11562) - -### Unmaintained components - -- `simpleprometheusreceiver`(#11133) ->>>>>>> Update CHANGELOG.md - `aerospikereceiver`: Fix issue where namespaces would not be collected (#11465) - `aerospikereceiver`: Fix typo in metric name. Ensure namespace transactions are collected (#12085) (#12083) - `coralogixexporter`: Fix metrics bearer token (#11831) @@ -91,7 +81,11 @@ - `sapmreceiver`: Fix issue where component instance use in multiple pipelines leads to start failures (#11518) - `signalfxreceiver`: Fix issue where component instance use in multiple pipelines leads to start failures (#11513) - `splunkhecreceiver`: Fix issue where component instance use in multiple pipelines leads to start failures (#11517) +<<<<<<< HEAD +======= +- `skywalkingreceiver`: Fix skywalking traceid and spanid convertion (#11562) +>>>>>>> [receiver/skywalking] fix skywalking-rcv tid convertion ## v0.54.0 diff --git a/receiver/skywalkingreceiver/go.mod b/receiver/skywalkingreceiver/go.mod index ee729d9b19f61..c477f254070e0 100644 --- a/receiver/skywalkingreceiver/go.mod +++ b/receiver/skywalkingreceiver/go.mod @@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/skywal go 1.17 require ( + github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/stretchr/testify v1.8.0 go.opentelemetry.io/collector v0.55.0 @@ -24,7 +25,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/knadh/koanf v1.4.2 // indirect diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go index c19e8c26c73e2..7368a49852d68 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go @@ -15,10 +15,12 @@ package skywalkingreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/skywalkingreceiver" import ( - "encoding/binary" + "bytes" "encoding/hex" - "strings" + "reflect" + "strconv" "time" + "unsafe" "github.com/google/uuid" "go.opentelemetry.io/collector/pdata/pcommon" @@ -102,7 +104,7 @@ func swSpansToSpanSlice(traceID string, segmentID string, spans []*agentV3.SpanO } func swSpanToSpan(traceID string, segmentID string, span *agentV3.SpanObject, dest ptrace.Span) { - dest.SetTraceID(stringToTraceID(traceID)) + dest.SetTraceID(swTraceIDToTraceID(traceID)) // skywalking defines segmentId + spanId as unique identifier // so use segmentId to convert to an unique otel-span dest.SetSpanID(segmentIDToSpanID(segmentID, uint32(span.GetSpanId()))) @@ -157,7 +159,7 @@ func swReferencesToSpanLinks(refs []*agentV3.SegmentReference, dest ptrace.SpanL for _, ref := range refs { link := dest.AppendEmpty() - link.SetTraceID(stringToTraceID(ref.TraceId)) + link.SetTraceID(swTraceIDToTraceID(ref.TraceId)) link.SetSpanID(segmentIDToSpanID(ref.ParentTraceSegmentId, uint32(ref.ParentSpanId))) link.SetTraceState("") kvParis := []*common.KeyStringValuePair{ @@ -238,36 +240,102 @@ func microsecondsToTimestamp(ms int64) pcommon.Timestamp { return pcommon.NewTimestampFromTime(time.UnixMilli(ms)) } -func stringToTraceID(traceID string) pcommon.TraceID { - if i := strings.IndexByte(traceID, '.'); i >= 0 { - traceID = traceID[:i] - } - uid, err := uuid.Parse(traceID) - if err != nil { - return pcommon.InvalidTraceID() +func swTraceIDToTraceID(traceID string) pcommon.TraceID { + // skywalking traceid format: + // de5980b8-fce3-4a37-aab9-b4ac3af7eedd: from browser/js-sdk/envoy/nginx-lua sdk/py-agent + // 56a5e1c519ae4c76a2b8b11d92cead7f.12.16563474296430001: from java-agent + + if len(traceID) <= 36 { // 36: uuid length (rfc4122) + uid, err := uuid.Parse(traceID) + if err != nil { + return pcommon.InvalidTraceID() + } + return pcommon.NewTraceID(uid) } - return pcommon.NewTraceID(uid) + return pcommon.NewTraceID(swStringToUUID(traceID, 0)) } func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { - if i := strings.LastIndexByte(segmentID, '.'); i >= 0 && i+1 < len(segmentID) { - segmentID = segmentID[i+1:] + // skywalking segmentid format: + // 56a5e1c519ae4c76a2b8b11d92cead7f.12.16563474296430001: from TraceSegmentId + // 56a5e1c519ae4c76a2b8b11d92cead7f: from ParentTraceSegmentId + + if len(segmentID) < 32 { + return pcommon.InvalidSpanID() } - segments, err := stringTo8Bytes(&segmentID) + return pcommon.NewSpanID(uuidTo8Bytes(swStringToUUID(segmentID, spanID))) +} + +func swStringToUUID(s string, extra uint32) (dst [16]byte) { + // there are 2 possible formats for 's': + // s format = 56a5e1c519ae4c76a2b8b11d92cead7f.0000000000.000000000000000000 + // ^ start(length=32) ^ mid(u32) ^ last(u64) + // uid = UUID(start) XOR ([4]byte(extra) . [4]byte(uint32(mid)) . [8]byte(uint64(last))) + + // s format = 56a5e1c519ae4c76a2b8b11d92cead7f + // ^ start(length=32) + // uid = UUID(start) XOR [4]byte(extra) + + if len(s) < 32 { + return + } + + t := unsafeGetBytes(s) + var uid [16]byte + _, err := hex.Decode(uid[:], t[:32]) if err != nil { - return pcommon.InvalidSpanID() + return uid } - binary.PutUvarint(segments[:], uint64(spanID)) - return pcommon.NewSpanID(segments) + + for i := 0; i < 4; i++ { + uid[i] ^= byte(extra) + extra >>= 8 + } + + if len(s) == 32 { + return uid + } + + index1 := bytes.IndexByte(t, '.') + index2 := bytes.LastIndexByte(t, '.') + if index1 != 32 || index2 < 0 { + return + } + + mid, err := strconv.Atoi(s[index1+1 : index2]) + if err != nil { + return + } + + last, err := strconv.Atoi(s[index2+1:]) + if err != nil { + return + } + + for i := 4; i < 8; i++ { + uid[i] ^= byte(mid) + mid >>= 8 + } + + for i := 8; i < 16; i++ { + uid[i] ^= byte(last) + last >>= 8 + } + + return uid } -func stringTo8Bytes(s *string) ([8]byte, error) { +func uuidTo8Bytes(uuid [16]byte) [8]byte { + // high bit XOR low bit var dst [8]byte - var mid [16]byte - copy(mid[:], ([]byte(*s))) - _, err := hex.Decode(dst[:], mid[:]) - if err != nil { - return dst, err + for i := 0; i < 8; i++ { + dst[i] = uuid[i] ^ uuid[i+8] } - return dst, nil + return dst +} + +func unsafeGetBytes(s string) []byte { + return (*[0x7fff0000]byte)(unsafe.Pointer( + (*reflect.StringHeader)(unsafe.Pointer(&s)).Data), + )[:len(s):len(s)] } diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go index a7bae86985b1c..b0375b6d2f4a7 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go @@ -171,12 +171,12 @@ func Test_stringToTraceID(t *testing.T) { want [16]byte }{ { - name: "mock-sw-normal-trace-id", + name: "mock-sw-normal-trace-id-rfc4122v4", segmentObject: args{traceID: "de5980b8-fce3-4a37-aab9-b4ac3af7eedd"}, want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, }, { - name: "mock-sw-normal-trace-id", + name: "mock-sw-normal-trace-id-rfc4122", segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd"}, want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, }, @@ -186,9 +186,9 @@ func Test_stringToTraceID(t *testing.T) { want: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, }, { - name: "mock-sw-trace-id-length-overflow", + name: "mock-sw-trace-id-length-java-agent", segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.1.16563474296430001"}, - want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, + want: [16]byte{222, 89, 128, 184, 253, 227, 74, 55, 27, 228, 27, 205, 94, 47, 212, 221}, }, { name: "mock-sw-trace-id-illegal", @@ -198,12 +198,42 @@ func Test_stringToTraceID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := stringToTraceID(tt.segmentObject.traceID) + got := swTraceIDToTraceID(tt.segmentObject.traceID) assert.Equal(t, tt.want, got.Bytes()) }) } } +func Test_stringToTraceID_Unique(t *testing.T) { + type args struct { + traceID string + } + tests := []struct { + name string + segmentObject args + }{ + { + name: "mock-sw-trace-id-unique-1", + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.133.16563474296430001"}, + }, + { + name: "mock-sw-trace-id-unique-2", + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.133.16534574123430001"}, + }, + } + + var results [2][16]byte + for i := 0; i < 2; i++ { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + got := swTraceIDToTraceID(tt.segmentObject.traceID) + results[i] = got.Bytes() + }) + } + assert.NotEqual(t, tests[0].segmentObject.traceID, t, tests[1].segmentObject.traceID) + assert.NotEqual(t, results[0], results[1]) +} + func Test_segmentIdToSpanId(t *testing.T) { type args struct { segmentID string @@ -217,12 +247,17 @@ func Test_segmentIdToSpanId(t *testing.T) { { name: "mock-sw-span-id-normal", args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 123}, - want: [8]byte{123, 86, 6, 7, 54, 153, 80, 6}, + want: [8]byte{233, 196, 85, 168, 37, 66, 48, 106}, + }, + { + name: "mock-sw-span-id-python-agent", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86", spanID: 123}, + want: [8]byte{155, 55, 217, 119, 204, 151, 10, 106}, }, { name: "mock-sw-span-id-short", args: args{segmentID: "16560607369950066", spanID: 12}, - want: [8]byte{12, 86, 6, 7, 54, 153, 80, 6}, + want: [8]byte{0, 0, 0, 0, 0, 0, 0, 0}, }, { name: "mock-sw-span-id-illegal-1", @@ -243,6 +278,36 @@ func Test_segmentIdToSpanId(t *testing.T) { } } +func Test_segmentIdToSpanId_Unique(t *testing.T) { + type args struct { + segmentID string + spanID uint32 + } + tests := []struct { + name string + args args + }{ + { + name: "mock-sw-span-id-unique-1", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 123}, + }, + { + name: "mock-sw-span-id-unique-2", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 1}, + }, + } + var results [2][8]byte + for i := 0; i < 2; i++ { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + got := segmentIDToSpanID(tt.args.segmentID, tt.args.spanID) + results[i] = got.Bytes() + }) + } + + assert.NotEqual(t, results[0], results[1]) +} + func generateTracesOneEmptyResourceSpans() ptrace.Span { td := ptrace.NewTraces() resourceSpan := td.ResourceSpans().AppendEmpty() From 2c3ecf761a871004047f2b085725c746d4842f28 Mon Sep 17 00:00:00 2001 From: taloric Date: Tue, 28 Jun 2022 11:33:40 +0000 Subject: [PATCH 5/7] use uuid.Parse instead of hextable to convert id --- receiver/skywalkingreceiver/go.mod | 1 + receiver/skywalkingreceiver/go.sum | 2 + .../skywalkingproto_to_traces.go | 44 +++---------------- .../skywalkingproto_to_traces_test.go | 2 +- 4 files changed, 10 insertions(+), 39 deletions(-) diff --git a/receiver/skywalkingreceiver/go.mod b/receiver/skywalkingreceiver/go.mod index c44445a9d7e6b..ee729d9b19f61 100644 --- a/receiver/skywalkingreceiver/go.mod +++ b/receiver/skywalkingreceiver/go.mod @@ -24,6 +24,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/knadh/koanf v1.4.2 // indirect diff --git a/receiver/skywalkingreceiver/go.sum b/receiver/skywalkingreceiver/go.sum index 385ae55823127..35547bf2b7281 100644 --- a/receiver/skywalkingreceiver/go.sum +++ b/receiver/skywalkingreceiver/go.sum @@ -191,6 +191,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go index 27b5db16c214f..c19e8c26c73e2 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/google/uuid" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" conventions "go.opentelemetry.io/collector/semconv/v1.8.0" @@ -238,11 +239,14 @@ func microsecondsToTimestamp(ms int64) pcommon.Timestamp { } func stringToTraceID(traceID string) pcommon.TraceID { - dst, err := stringTo16Bytes(&traceID) + if i := strings.IndexByte(traceID, '.'); i >= 0 { + traceID = traceID[:i] + } + uid, err := uuid.Parse(traceID) if err != nil { return pcommon.InvalidTraceID() } - return pcommon.NewTraceID(dst) + return pcommon.NewTraceID(uid) } func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { @@ -257,18 +261,6 @@ func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { return pcommon.NewSpanID(segments) } -func stringTo16Bytes(s *string) ([16]byte, error) { - var dst [16]byte - var mid [32]byte - h := stringToHexBytes(s) - copy(mid[:], h) - _, err := hex.Decode(dst[:], mid[:]) - if err != nil { - return dst, err - } - return dst, nil -} - func stringTo8Bytes(s *string) ([8]byte, error) { var dst [8]byte var mid [16]byte @@ -279,27 +271,3 @@ func stringTo8Bytes(s *string) ([8]byte, error) { } return dst, nil } - -// hex table: -// 48-57: 0-9 -// 65-70: a-f -// 97-102: A-F -func stringToHexBytes(s *string) []byte { - src := make([]byte, 0, len(*s)) - for i := 0; i < len(*s); i++ { - r := (*s)[i] - switch { - case r < 48: - fallthrough - case r > 57 && r < 65: - fallthrough - case r > 70 && r < 97: - fallthrough - case r > 102: - // if 'r' is not a hex digit, drop it - continue - } - src = append(src, r) - } - return src -} diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go index 154907430f1a9..a7bae86985b1c 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go @@ -187,7 +187,7 @@ func Test_stringToTraceID(t *testing.T) { }, { name: "mock-sw-trace-id-length-overflow", - segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eeddddddde5980"}, + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.1.16563474296430001"}, want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, }, { From 395196ee1d74319d68e4356da52edfbc71e196b7 Mon Sep 17 00:00:00 2001 From: taloric Date: Fri, 1 Jul 2022 10:01:54 +0000 Subject: [PATCH 6/7] [receiver/skywalking] fix skywalking-rcv tid convertion --- receiver/skywalkingreceiver/go.mod | 2 +- .../skywalkingproto_to_traces.go | 118 ++++++++++++++---- .../skywalkingproto_to_traces_test.go | 79 ++++++++++-- 3 files changed, 166 insertions(+), 33 deletions(-) diff --git a/receiver/skywalkingreceiver/go.mod b/receiver/skywalkingreceiver/go.mod index ee729d9b19f61..c477f254070e0 100644 --- a/receiver/skywalkingreceiver/go.mod +++ b/receiver/skywalkingreceiver/go.mod @@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/skywal go 1.17 require ( + github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/stretchr/testify v1.8.0 go.opentelemetry.io/collector v0.55.0 @@ -24,7 +25,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/knadh/koanf v1.4.2 // indirect diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go index c19e8c26c73e2..7368a49852d68 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go @@ -15,10 +15,12 @@ package skywalkingreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/skywalkingreceiver" import ( - "encoding/binary" + "bytes" "encoding/hex" - "strings" + "reflect" + "strconv" "time" + "unsafe" "github.com/google/uuid" "go.opentelemetry.io/collector/pdata/pcommon" @@ -102,7 +104,7 @@ func swSpansToSpanSlice(traceID string, segmentID string, spans []*agentV3.SpanO } func swSpanToSpan(traceID string, segmentID string, span *agentV3.SpanObject, dest ptrace.Span) { - dest.SetTraceID(stringToTraceID(traceID)) + dest.SetTraceID(swTraceIDToTraceID(traceID)) // skywalking defines segmentId + spanId as unique identifier // so use segmentId to convert to an unique otel-span dest.SetSpanID(segmentIDToSpanID(segmentID, uint32(span.GetSpanId()))) @@ -157,7 +159,7 @@ func swReferencesToSpanLinks(refs []*agentV3.SegmentReference, dest ptrace.SpanL for _, ref := range refs { link := dest.AppendEmpty() - link.SetTraceID(stringToTraceID(ref.TraceId)) + link.SetTraceID(swTraceIDToTraceID(ref.TraceId)) link.SetSpanID(segmentIDToSpanID(ref.ParentTraceSegmentId, uint32(ref.ParentSpanId))) link.SetTraceState("") kvParis := []*common.KeyStringValuePair{ @@ -238,36 +240,102 @@ func microsecondsToTimestamp(ms int64) pcommon.Timestamp { return pcommon.NewTimestampFromTime(time.UnixMilli(ms)) } -func stringToTraceID(traceID string) pcommon.TraceID { - if i := strings.IndexByte(traceID, '.'); i >= 0 { - traceID = traceID[:i] - } - uid, err := uuid.Parse(traceID) - if err != nil { - return pcommon.InvalidTraceID() +func swTraceIDToTraceID(traceID string) pcommon.TraceID { + // skywalking traceid format: + // de5980b8-fce3-4a37-aab9-b4ac3af7eedd: from browser/js-sdk/envoy/nginx-lua sdk/py-agent + // 56a5e1c519ae4c76a2b8b11d92cead7f.12.16563474296430001: from java-agent + + if len(traceID) <= 36 { // 36: uuid length (rfc4122) + uid, err := uuid.Parse(traceID) + if err != nil { + return pcommon.InvalidTraceID() + } + return pcommon.NewTraceID(uid) } - return pcommon.NewTraceID(uid) + return pcommon.NewTraceID(swStringToUUID(traceID, 0)) } func segmentIDToSpanID(segmentID string, spanID uint32) pcommon.SpanID { - if i := strings.LastIndexByte(segmentID, '.'); i >= 0 && i+1 < len(segmentID) { - segmentID = segmentID[i+1:] + // skywalking segmentid format: + // 56a5e1c519ae4c76a2b8b11d92cead7f.12.16563474296430001: from TraceSegmentId + // 56a5e1c519ae4c76a2b8b11d92cead7f: from ParentTraceSegmentId + + if len(segmentID) < 32 { + return pcommon.InvalidSpanID() } - segments, err := stringTo8Bytes(&segmentID) + return pcommon.NewSpanID(uuidTo8Bytes(swStringToUUID(segmentID, spanID))) +} + +func swStringToUUID(s string, extra uint32) (dst [16]byte) { + // there are 2 possible formats for 's': + // s format = 56a5e1c519ae4c76a2b8b11d92cead7f.0000000000.000000000000000000 + // ^ start(length=32) ^ mid(u32) ^ last(u64) + // uid = UUID(start) XOR ([4]byte(extra) . [4]byte(uint32(mid)) . [8]byte(uint64(last))) + + // s format = 56a5e1c519ae4c76a2b8b11d92cead7f + // ^ start(length=32) + // uid = UUID(start) XOR [4]byte(extra) + + if len(s) < 32 { + return + } + + t := unsafeGetBytes(s) + var uid [16]byte + _, err := hex.Decode(uid[:], t[:32]) if err != nil { - return pcommon.InvalidSpanID() + return uid } - binary.PutUvarint(segments[:], uint64(spanID)) - return pcommon.NewSpanID(segments) + + for i := 0; i < 4; i++ { + uid[i] ^= byte(extra) + extra >>= 8 + } + + if len(s) == 32 { + return uid + } + + index1 := bytes.IndexByte(t, '.') + index2 := bytes.LastIndexByte(t, '.') + if index1 != 32 || index2 < 0 { + return + } + + mid, err := strconv.Atoi(s[index1+1 : index2]) + if err != nil { + return + } + + last, err := strconv.Atoi(s[index2+1:]) + if err != nil { + return + } + + for i := 4; i < 8; i++ { + uid[i] ^= byte(mid) + mid >>= 8 + } + + for i := 8; i < 16; i++ { + uid[i] ^= byte(last) + last >>= 8 + } + + return uid } -func stringTo8Bytes(s *string) ([8]byte, error) { +func uuidTo8Bytes(uuid [16]byte) [8]byte { + // high bit XOR low bit var dst [8]byte - var mid [16]byte - copy(mid[:], ([]byte(*s))) - _, err := hex.Decode(dst[:], mid[:]) - if err != nil { - return dst, err + for i := 0; i < 8; i++ { + dst[i] = uuid[i] ^ uuid[i+8] } - return dst, nil + return dst +} + +func unsafeGetBytes(s string) []byte { + return (*[0x7fff0000]byte)(unsafe.Pointer( + (*reflect.StringHeader)(unsafe.Pointer(&s)).Data), + )[:len(s):len(s)] } diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go index a7bae86985b1c..b0375b6d2f4a7 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces_test.go @@ -171,12 +171,12 @@ func Test_stringToTraceID(t *testing.T) { want [16]byte }{ { - name: "mock-sw-normal-trace-id", + name: "mock-sw-normal-trace-id-rfc4122v4", segmentObject: args{traceID: "de5980b8-fce3-4a37-aab9-b4ac3af7eedd"}, want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, }, { - name: "mock-sw-normal-trace-id", + name: "mock-sw-normal-trace-id-rfc4122", segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd"}, want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, }, @@ -186,9 +186,9 @@ func Test_stringToTraceID(t *testing.T) { want: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, }, { - name: "mock-sw-trace-id-length-overflow", + name: "mock-sw-trace-id-length-java-agent", segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.1.16563474296430001"}, - want: [16]byte{222, 89, 128, 184, 252, 227, 74, 55, 170, 185, 180, 172, 58, 247, 238, 221}, + want: [16]byte{222, 89, 128, 184, 253, 227, 74, 55, 27, 228, 27, 205, 94, 47, 212, 221}, }, { name: "mock-sw-trace-id-illegal", @@ -198,12 +198,42 @@ func Test_stringToTraceID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := stringToTraceID(tt.segmentObject.traceID) + got := swTraceIDToTraceID(tt.segmentObject.traceID) assert.Equal(t, tt.want, got.Bytes()) }) } } +func Test_stringToTraceID_Unique(t *testing.T) { + type args struct { + traceID string + } + tests := []struct { + name string + segmentObject args + }{ + { + name: "mock-sw-trace-id-unique-1", + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.133.16563474296430001"}, + }, + { + name: "mock-sw-trace-id-unique-2", + segmentObject: args{traceID: "de5980b8fce34a37aab9b4ac3af7eedd.133.16534574123430001"}, + }, + } + + var results [2][16]byte + for i := 0; i < 2; i++ { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + got := swTraceIDToTraceID(tt.segmentObject.traceID) + results[i] = got.Bytes() + }) + } + assert.NotEqual(t, tests[0].segmentObject.traceID, t, tests[1].segmentObject.traceID) + assert.NotEqual(t, results[0], results[1]) +} + func Test_segmentIdToSpanId(t *testing.T) { type args struct { segmentID string @@ -217,12 +247,17 @@ func Test_segmentIdToSpanId(t *testing.T) { { name: "mock-sw-span-id-normal", args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 123}, - want: [8]byte{123, 86, 6, 7, 54, 153, 80, 6}, + want: [8]byte{233, 196, 85, 168, 37, 66, 48, 106}, + }, + { + name: "mock-sw-span-id-python-agent", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86", spanID: 123}, + want: [8]byte{155, 55, 217, 119, 204, 151, 10, 106}, }, { name: "mock-sw-span-id-short", args: args{segmentID: "16560607369950066", spanID: 12}, - want: [8]byte{12, 86, 6, 7, 54, 153, 80, 6}, + want: [8]byte{0, 0, 0, 0, 0, 0, 0, 0}, }, { name: "mock-sw-span-id-illegal-1", @@ -243,6 +278,36 @@ func Test_segmentIdToSpanId(t *testing.T) { } } +func Test_segmentIdToSpanId_Unique(t *testing.T) { + type args struct { + segmentID string + spanID uint32 + } + tests := []struct { + name string + args args + }{ + { + name: "mock-sw-span-id-unique-1", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 123}, + }, + { + name: "mock-sw-span-id-unique-2", + args: args{segmentID: "4f2f27748b8e44ecaf18fe0347194e86.33.16560607369950066", spanID: 1}, + }, + } + var results [2][8]byte + for i := 0; i < 2; i++ { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + got := segmentIDToSpanID(tt.args.segmentID, tt.args.spanID) + results[i] = got.Bytes() + }) + } + + assert.NotEqual(t, results[0], results[1]) +} + func generateTracesOneEmptyResourceSpans() ptrace.Span { td := ptrace.NewTraces() resourceSpan := td.ResourceSpans().AppendEmpty() From d9c014ede88b880539d0a3b9d93702cc8109b3e2 Mon Sep 17 00:00:00 2001 From: taloric Date: Mon, 11 Jul 2022 09:52:55 +0000 Subject: [PATCH 7/7] [receiver/skywalking] Add skywalking attributes into tracedata --- .../skywalkingreceiver/skywalkingproto_to_traces.go | 13 ++++++++++--- receiver/skywalkingreceiver/trace_receiver.go | 2 +- .../skywalkingreceiver/tracing_report_service.go | 2 +- unreleased/skywalkingreceiver-fix.yaml | 4 ++++ 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 unreleased/skywalkingreceiver-fix.yaml diff --git a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go index 7368a49852d68..ad423a5bc0c5d 100644 --- a/receiver/skywalkingreceiver/skywalkingproto_to_traces.go +++ b/receiver/skywalkingreceiver/skywalkingproto_to_traces.go @@ -35,6 +35,9 @@ const ( AttributeParentService = "parent.service" AttributeParentInstance = "parent.service.instance" AttributeParentEndpoint = "parent.endpoint" + AttributeSkywalkingSpanID = "sw8.span_id" + AttributeSkywalkingTraceID = "sw8.trace_id" + AttributeSkywalkingSegmentID = "sw8.segment_id" AttributeNetworkAddressUsedAtPeer = "network.AddressUsedAtPeer" ) @@ -56,12 +59,15 @@ func SkywalkingToTraces(segment *agentV3.SegmentObject) ptrace.Traces { resourceSpan := traceData.ResourceSpans().AppendEmpty() rs := resourceSpan.Resource() + for _, span := range swSpans { swTagsToInternalResource(span, rs) - rs.Attributes().Insert(conventions.AttributeServiceName, pcommon.NewValueString(segment.GetService())) - rs.Attributes().Insert(conventions.AttributeServiceInstanceID, pcommon.NewValueString(segment.GetServiceInstance())) } + rs.Attributes().Insert(conventions.AttributeServiceName, pcommon.NewValueString(segment.GetService())) + rs.Attributes().Insert(conventions.AttributeServiceInstanceID, pcommon.NewValueString(segment.GetServiceInstance())) + rs.Attributes().Insert(AttributeSkywalkingTraceID, pcommon.NewValueString(segment.GetTraceId())) + il := resourceSpan.ScopeSpans().AppendEmpty() swSpansToSpanSlice(segment.GetTraceId(), segment.GetTraceSegmentId(), swSpans, il.Spans()) @@ -125,7 +131,8 @@ func swSpanToSpan(traceID string, segmentID string, span *agentV3.SpanObject, de if attrs.Len() == 0 { attrs.Clear() } - + attrs.InsertInt(AttributeSkywalkingSpanID, int64(span.GetSpanId())) + attrs.InsertString(AttributeSkywalkingSegmentID, segmentID) setInternalSpanStatus(span, dest.Status()) switch { diff --git a/receiver/skywalkingreceiver/trace_receiver.go b/receiver/skywalkingreceiver/trace_receiver.go index 32e22caab1c7b..02e02c7c66f2e 100644 --- a/receiver/skywalkingreceiver/trace_receiver.go +++ b/receiver/skywalkingreceiver/trace_receiver.go @@ -222,7 +222,7 @@ func (sr *swReceiver) httpHandler(rsp http.ResponseWriter, r *http.Request) { } for _, segment := range data { - err = consumeTraces(context.Background(), segment, sr.nextConsumer) + err = consumeTraces(r.Context(), segment, sr.nextConsumer) if err != nil { fmt.Printf("cannot consume traces, %v", err) } diff --git a/receiver/skywalkingreceiver/tracing_report_service.go b/receiver/skywalkingreceiver/tracing_report_service.go index f3ad4fcdb1f5c..05ee826bfda56 100644 --- a/receiver/skywalkingreceiver/tracing_report_service.go +++ b/receiver/skywalkingreceiver/tracing_report_service.go @@ -41,7 +41,7 @@ func (s *traceSegmentReportService) Collect(stream agent.TraceSegmentReportServi return err } - err = consumeTraces(context.Background(), segmentObject, s.sr.nextConsumer) + err = consumeTraces(stream.Context(), segmentObject, s.sr.nextConsumer) if err != nil { return stream.SendAndClose(&common.Commands{}) } diff --git a/unreleased/skywalkingreceiver-fix.yaml b/unreleased/skywalkingreceiver-fix.yaml new file mode 100644 index 0000000000000..c2e14c33e138e --- /dev/null +++ b/unreleased/skywalkingreceiver-fix.yaml @@ -0,0 +1,4 @@ +change_type: bug_fix +component: skywalkingreceiver +note: Fix skywalking traceid and spanid convertion +issues: [11562] \ No newline at end of file