stats/opentelemetry: Add CSM Plugin Option (#7205)

.github/workflows/testing.yml

@@ -105,8 +105,15 @@ jobs:
  cd "${GITHUB_WORKSPACE}"
  for MOD_FILE in $(find . -name 'go.mod' | grep -Ev '^\./go\.mod'); do
  pushd "$(dirname ${MOD_FILE})"
- go test ${{ matrix.testflags }} -cpu 1,4 -timeout 2m ./...
+ # Skip OpenTelemetry module if 3 releases ago, Go OpenTelemetry only supports
+ # previous two releases.
+ if [[ "${{ matrix.goversion }}" == "1.20" && "${PWD}" =~ /stats/opentelemetry$ ]]; then
+ echo "Skipping ${MOD_FILE}"
+ else
+ go test ${{ matrix.testflags }} -cpu 1,4 -timeout 2m ./...
+ fi
  popd
+
  done
 
  # Non-core gRPC tests (examples, interop, etc)

stats/opentelemetry/csm/pluginoption.go

@@ -0,0 +1,309 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package csm contains utilities for Google Cloud Service Mesh observability.
+package csm
+
+import (
+ "context"
+ "encoding/base64"
+ "net/url"
+ "os"
+ "strings"
+
+ "google.golang.org/grpc/grpclog"
+ "google.golang.org/grpc/internal/xds/bootstrap"
+ "google.golang.org/grpc/metadata"
+ "google.golang.org/grpc/stats/opentelemetry/internal"
+
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/structpb"
+
+ "go.opentelemetry.io/contrib/detectors/gcp"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/sdk/resource"
+)
+
+var logger = grpclog.Component("csm-observability-plugin")
+
+// pluginOption emits CSM Labels from the environment and metadata exchange
+// for csm channels and all servers.
+//
+// Do not use this directly; use newPluginOption instead.
+type pluginOption struct {
+ // localLabels are the labels that identify the local environment a binary
+ // is run in, and will be emitted from the CSM Plugin Option.
+ localLabels map[string]string
+ // metadataExchangeLabelsEncoded are the metadata exchange labels to be sent
+ // as the value of metadata key "x-envoy-peer-metadata" in proto wire format
+ // and base 64 encoded. This gets sent out from all the servers running in
+ // this process and for csm channels.
+ metadataExchangeLabelsEncoded string
+}
+
+// newPluginOption returns a new pluginOption with local labels and metadata
+// exchange labels derived from the environment.
+func newPluginOption(ctx context.Context) internal.PluginOption {
+ localLabels, metadataExchangeLabelsEncoded := constructMetadataFromEnv(ctx)
+
+ return &pluginOption{
+ localLabels: localLabels,
+ metadataExchangeLabelsEncoded: metadataExchangeLabelsEncoded,
+ }
+}
+
+// NewLabelsMD returns a metadata.MD with the CSM labels as an encoded protobuf
+// Struct as the value of "x-envoy-peer-metadata".
+func (cpo *pluginOption) GetMetadata() metadata.MD {
+ return metadata.New(map[string]string{
+ metadataExchangeKey: cpo.metadataExchangeLabelsEncoded,
+ })
+}
+
+// GetLabels gets the CSM peer labels from the metadata provided. It returns
+// "unknown" for labels not found. Labels returned depend on the remote type.
+// Additionally, local labels determined at initialization time are appended to
+// labels returned, in addition to the optionalLabels provided.
+func (cpo *pluginOption) GetLabels(md metadata.MD) map[string]string {
+ labels := map[string]string{ // Remote labels if type is unknown (i.e. unset or error processing x-envoy-peer-metadata)
+ "csm.remote_workload_type": "unknown",
+ "csm.remote_workload_canonical_service": "unknown",
+ }
+ // Append the local labels.
+ for k, v := range cpo.localLabels {
+ labels[k] = v
+ }
+
+ val := md.Get("x-envoy-peer-metadata")
+ // This can't happen if corresponding csm client because of proto wire
+ // format encoding, but since it is arbitrary off the wire be safe.
+ if len(val) != 1 {
+ logger.Warningf("length of md values of \"x-envoy-peer-metadata\" is not 1, is %v", len(val))
+ return labels
+ }
+
+ protoWireFormat, err := base64.RawStdEncoding.DecodeString(val[0])
+ if err != nil {
+ logger.Warningf("error base 64 decoding value of \"x-envoy-peer-metadata\": %v", err)
+ return labels
+ }
+
+ spb := &structpb.Struct{}
+ if err := proto.Unmarshal(protoWireFormat, spb); err != nil {
+ logger.Warningf("error unmarshalling value of \"x-envoy-peer-metadata\" into proto: %v", err)
+ return labels
+ }
+
+ fields := spb.GetFields()
+
+ labels["csm.remote_workload_type"] = getFromMetadata("type", fields)
+ // The value of “csm.remote_workload_canonical_service” comes from
+ // MetadataExchange with the key “canonical_service”. (Note that this should
+ // be read even if the remote type is unknown.)
+ labels["csm.remote_workload_canonical_service"] = getFromMetadata("canonical_service", fields)
+
+ // Unset/unknown types, and types that aren't GKE or GCP return early with
+ // just local labels, remote_workload_type and
+ // remote_workload_canonical_service labels.
+ workloadType := labels["csm.remote_workload_type"]
+ if workloadType != "gcp_kubernetes_engine" && workloadType != "gcp_compute_engine" {
+ return labels
+ }
+ // GKE and GCE labels.
+ labels["csm.remote_workload_project_id"] = getFromMetadata("project_id", fields)
+ labels["csm.remote_workload_location"] = getFromMetadata("location", fields)
+ labels["csm.remote_workload_name"] = getFromMetadata("workload_name", fields)
+ if workloadType == "gcp_compute_engine" {
+ return labels
+ }
+
+ // GKE only labels.
+ labels["csm.remote_workload_cluster_name"] = getFromMetadata("cluster_name", fields)
+ labels["csm.remote_workload_namespace_name"] = getFromMetadata("namespace_name", fields)
+ return labels
+}
+
+// getFromMetadata gets the value for the metadata key from the protobuf
+// metadata. Returns "unknown" if the metadata is not found in the protobuf
+// metadata, or if the value is not a string value. Returns the string value
+// otherwise.
+func getFromMetadata(metadataKey string, metadata map[string]*structpb.Value) string {
+ if metadata != nil {
+ if metadataVal, ok := metadata[metadataKey]; ok {
+ if _, ok := metadataVal.GetKind().(*structpb.Value_StringValue); ok {
+ return metadataVal.GetStringValue()
+ }
+ }
+ }
+ return "unknown"
+}
+
+// getFromResource gets the value for the resource key from the attribute set.
+// Returns "unknown" if the resourceKey is not found in the attribute set or is
+// not a string value, the string value otherwise.
+func getFromResource(resourceKey attribute.Key, set *attribute.Set) string {
+ if set != nil {
+ if resourceVal, ok := set.Value(resourceKey); ok && resourceVal.Type() == attribute.STRING {
+ return resourceVal.AsString()
+ }
+ }
+ return "unknown"
+}
+
+// getEnv returns "unknown" if environment variable is unset, the environment
+// variable otherwise.
+func getEnv(name string) string {
+ if val, ok := os.LookupEnv(name); ok {
+ return val
+ }
+ return "unknown"
+}
+
+var (
+ // This function will be overridden in unit tests.
+ getAttrSetFromResourceDetector = func(ctx context.Context) *attribute.Set {
+ r, err := resource.New(ctx, resource.WithDetectors(gcp.NewDetector()))
+ if err != nil {
+ logger.Errorf("error reading OpenTelemetry resource: %v", err)
+ return nil
+ }
+ return r.Set()
+ }
+)
+
+// constructMetadataFromEnv creates local labels and labels to send to the peer
+// using metadata exchange based off resource detection and environment
+// variables.
+//
+// Returns local labels, and base 64 encoded protobuf.Struct containing metadata
+// exchange labels.
+func constructMetadataFromEnv(ctx context.Context) (map[string]string, string) {
+ set := getAttrSetFromResourceDetector(ctx)
+
+ labels := make(map[string]string)
+ labels["type"] = getFromResource("cloud.platform", set)
+ labels["canonical_service"] = getEnv("CSM_CANONICAL_SERVICE_NAME")
+
+ // If type is not GCE or GKE only metadata exchange labels are "type" and
+ // "canonical_service".
+ cloudPlatformVal := labels["type"]
+ if cloudPlatformVal != "gcp_kubernetes_engine" && cloudPlatformVal != "gcp_compute_engine" {
+ return initializeLocalAndMetadataLabels(labels)
+ }
+
+ // GCE and GKE labels:
+ labels["workload_name"] = getEnv("CSM_WORKLOAD_NAME")
+
+ locationVal := "unknown"
+ if resourceVal, ok := set.Value("cloud.availability_zone"); ok && resourceVal.Type() == attribute.STRING {
+ locationVal = resourceVal.AsString()
+ } else if resourceVal, ok = set.Value("cloud.region"); ok && resourceVal.Type() == attribute.STRING {
+ locationVal = resourceVal.AsString()
+ }
+ labels["location"] = locationVal
+
+ labels["project_id"] = getFromResource("cloud.account.id", set)
+ if cloudPlatformVal == "gcp_compute_engine" {
+ return initializeLocalAndMetadataLabels(labels)
+ }
+
+ // GKE specific labels:
+ labels["namespace_name"] = getFromResource("k8s.namespace.name", set)
+ labels["cluster_name"] = getFromResource("k8s.cluster.name", set)
+ return initializeLocalAndMetadataLabels(labels)
+}
+
+// parseMeshIDString parses the mesh id from the node id according to the format
+// "projects/[GCP Project number]/networks/mesh:[Mesh ID]/nodes/[UUID]". Returns
+// "unknown" if there is a syntax error in the node ID.
+func parseMeshIDFromNodeID(nodeID string) string {
+ meshSplit := strings.Split(nodeID, "/")
+ if len(meshSplit) != 6 {
+ return "unknown"
+ }
+ if meshSplit[0] != "projects" || meshSplit[2] != "networks" || meshSplit[4] != "nodes" {
+ return "unknown"
+ }
+ meshID, ok := strings.CutPrefix(meshSplit[3], "mesh:")
+ if !ok { // errors become "unknown"
+ return "unknown"
+ }
+ return meshID
+}
+
+// initializeLocalAndMetadataLabels csm local labels for a CSM Plugin Option to
+// record. It also builds out a base 64 encoded protobuf.Struct containing the
+// metadata exchange labels to be sent as part of metadata exchange from a CSM
+// Plugin Option.
+func initializeLocalAndMetadataLabels(labels map[string]string) (map[string]string, string) {
+ // The value of “csm.workload_canonical_service” comes from
+ // “CSM_CANONICAL_SERVICE_NAME” env var, “unknown” if unset.
+ val := labels["canonical_service"]
+ localLabels := make(map[string]string)
+ localLabels["csm.workload_canonical_service"] = val
+ // Get the CSM Mesh ID from the bootstrap file.
+ nodeID := getNodeID()
+ localLabels["csm.mesh_id"] = parseMeshIDFromNodeID(nodeID)
+
+ // Metadata exchange labels - can go ahead and encode into proto, and then
+ // base64.
+ pbLabels := &structpb.Struct{
+ Fields: map[string]*structpb.Value{},
+ }
+ for k, v := range labels {
+ pbLabels.Fields[k] = structpb.NewStringValue(v)
+ }
+ protoWireFormat, err := proto.Marshal(pbLabels)
+ metadataExchangeLabelsEncoded := ""
+ if err == nil {
+ metadataExchangeLabelsEncoded = base64.RawStdEncoding.EncodeToString(protoWireFormat)
+ }
+ // else - This behavior triggers server side to reply (if sent from a gRPC
+ // Client within this binary) with the metadata exchange labels. Even if
+ // client side has a problem marshaling proto into wire format, it can
+ // still use server labels so send an empty string as the value of
+ // x-envoy-peer-metadata. The presence of this metadata exchange header
+ // will cause server side to respond with metadata exchange labels.
+
+ return localLabels, metadataExchangeLabelsEncoded
+}
+
+// getNodeID gets the Node ID from the bootstrap data.
+func getNodeID() string {
+ cfg, err := bootstrap.NewConfig()
+ if err != nil {
+ return "" // will become "unknown"
+ }
+ if cfg.NodeProto == nil {
+ return ""
+ }
+ return cfg.NodeProto.GetId()
+}
+
+// metadataExchangeKey is the key for HTTP metadata exchange.
+const metadataExchangeKey = "x-envoy-peer-metadata"
+
+func determineTargetCSM(parsedTarget *url.URL) bool {
+ // On the client-side, the channel target is used to determine if a channel is a
+ // CSM channel or not. CSM channels need to have an “xds” scheme and a
+ // "traffic-director-global.xds.googleapis.com" authority. In the cases where no
+ // authority is mentioned, the authority is assumed to be CSM. MetadataExchange
+ // is performed only for CSM channels. Non-metadata exchange labels are detected
+ // as described below.
+ return parsedTarget.Scheme == "xds" && (parsedTarget.Host == "" || parsedTarget.Host == "traffic-director-global.xds.googleapis.com")
+}