[processor/transform] Add enum capabilities to grammar and contexts (o…

…pen-telemetry#11787)

* Add enum capabilities to grammar and contexts

* Updated changelog and readme

* Update changelog entry

processor/transformprocessor/README.md

@@ -20,6 +20,7 @@ in the OTLP protobuf definition. e.g., `status.code`, `attributes["http.method"]
  - Until the grammar can handle booleans, `is_monotic` is handled via strings the strings `"true"` and `"false"`.
  - Hex String of traceid and spanid are handled using `trace_id.string`,`span_id.string` accessor.
 - Literals: Strings, ints, floats, bools, and nil can be referenced as literal values. Byte slices can be references as a literal value via a hex string prefaced with `0x`, such as `0x0001`. 
+- Enums: Any enum in the OTLP protobuf can be used directly. For example, you can set the span kind like `set(kind, SPAN_KIND_UNSPECIFIED) where kind != SPAN_KIND_UNSPECIFIED`. You can also use the literal int value if you desire. In addition, the grammar recognises `METRIC_DATA_TYPE_NONE`, `METRIC_DATA_TYPE_GAUGE`, `METRIC_DATA_TYPE_SUM`, `METRIC_DATA_TYPE_HISTOGRAM`, `METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM`, and `METRIC_DATA_TYPE_SUMMARY` for `metric.type`
 - Function invocations: Functions can be invoked with arguments matching the function's expected arguments. The literal nil cannot be used as a replacement for maps or slices in function calls.
 - Where clause: Telemetry to modify can be filtered by appending `where a <op> b`, with `a` and `b` being any of the above.
 

processor/transformprocessor/config.go

@@ -44,15 +44,15 @@ var _ config.Processor = (*Config)(nil)
 
 func (c *Config) Validate() error {
  var errors error
- _, err := common.ParseQueries(c.Traces.Queries, c.Traces.functions, traces.ParsePath)
+ _, err := common.ParseQueries(c.Traces.Queries, c.Traces.functions, traces.ParsePath, traces.ParseEnum)
  if err != nil {
  errors = multierr.Append(errors, err)
  }
- _, err = common.ParseQueries(c.Metrics.Queries, c.Metrics.functions, metrics.ParsePath)
+ _, err = common.ParseQueries(c.Metrics.Queries, c.Metrics.functions, metrics.ParsePath, metrics.ParseEnum)
  if err != nil {
  errors = multierr.Append(errors, err)
  }
- _, err = common.ParseQueries(c.Logs.Queries, c.Logs.functions, logs.ParsePath)
+ _, err = common.ParseQueries(c.Logs.Queries, c.Logs.functions, logs.ParsePath, logs.ParseEnum)
  if err != nil {
  errors = multierr.Append(errors, err)
  }

processor/transformprocessor/internal/common/condition.go

@@ -24,15 +24,15 @@ var alwaysTrue = func(ctx TransformContext) bool {
  return true
 }
 
-func newConditionEvaluator(cond *Condition, functions map[string]interface{}, pathParser PathExpressionParser) (condFunc, error) {
+func newConditionEvaluator(cond *Condition, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (condFunc, error) {
  if cond == nil {
  return alwaysTrue, nil
  }
- left, err := NewGetter(cond.Left, functions, pathParser)
+ left, err := NewGetter(cond.Left, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
- right, err := NewGetter(cond.Right, functions, pathParser)
+ right, err := NewGetter(cond.Right, functions, pathParser, enumParser)
  // TODO(anuraaga): Check if both left and right are literals and const-evaluate
  if err != nil {
  return nil, err

processor/transformprocessor/internal/common/condition_test.go

@@ -94,10 +94,28 @@ func Test_newConditionEvaluator(t *testing.T) {
  name: "no condition",
  cond: nil,
  },
+ {
+ name: "compare Enum to int",
+ cond: &Condition{
+ Left: Value{
+ Path: &Path{
+ Fields: []Field{
+ {
+ Name: "TEST_ENUM",
+ },
+ },
+ },
+ },
+ Right: Value{
+ Int: testhelper.Intp(0),
+ },
+ Op: "==",
+ },
+ },
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- evaluate, err := newConditionEvaluator(tt.cond, DefaultFunctions(), testParsePath)
+ evaluate, err := newConditionEvaluator(tt.cond, DefaultFunctions(), testParsePath, testParseEnum)
  assert.NoError(t, err)
  assert.True(t, evaluate(testhelper.TestTransformContext{
  Item: tt.item,
@@ -114,7 +132,7 @@ func Test_newConditionEvaluator(t *testing.T) {
  Right: Value{
  String: testhelper.Strp("cat"),
  },
- }, DefaultFunctions(), testParsePath)
+ }, DefaultFunctions(), testParsePath, testParseEnum)
  assert.Error(t, err)
  })
 }

processor/transformprocessor/internal/common/expression.go

@@ -28,6 +28,8 @@ type TransformContext interface {
 
 type ExprFunc func(ctx TransformContext) interface{}
 
+type Enum int64
+
 type Getter interface {
  Get(ctx TransformContext) interface{}
 }
@@ -57,7 +59,7 @@ func (g exprGetter) Get(ctx TransformContext) interface{} {
  return g.expr(ctx)
 }
 
-func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpressionParser) (Getter, error) {
+func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (Getter, error) {
  if val.IsNil != nil && *val.IsNil {
  return &literal{value: nil}, nil
  }
@@ -79,14 +81,17 @@ func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpre
  }
 
  if val.Path != nil {
+ if enum, ok := enumParser(val.Path); ok {
+ return &literal{value: int64(*enum)}, nil
+ }
  return pathParser(val.Path)
  }
 
  if val.Invocation == nil {
  // In practice, can't happen since the DSL grammar guarantees one is set
  return nil, fmt.Errorf("no value field set. This is a bug in the transformprocessor")
  }
- call, err := NewFunctionCall(*val.Invocation, functions, pathParser)
+ call, err := NewFunctionCall(*val.Invocation, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }

processor/transformprocessor/internal/common/expression_test.go

@@ -98,13 +98,26 @@ func Test_newGetter(t *testing.T) {
  },
  want: "world",
  },
+ {
+ name: "enum",
+ val: Value{
+ Path: &Path{
+ Fields: []Field{
+ {
+ Name: "TEST_ENUM",
+ },
+ },
+ },
+ },
+ want: int64(0),
+ },
  }
 
  functions := map[string]interface{}{"hello": hello}
 
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- reader, err := NewGetter(tt.val, functions, testParsePath)
+ reader, err := NewGetter(tt.val, functions, testParsePath, testParseEnum)
  assert.NoError(t, err)
  val := reader.Get(testhelper.TestTransformContext{
  Item: tt.want,
@@ -114,7 +127,7 @@ func Test_newGetter(t *testing.T) {
  }
 
  t.Run("empty value", func(t *testing.T) {
- _, err := NewGetter(Value{}, functions, testParsePath)
+ _, err := NewGetter(Value{}, functions, testParsePath, testParseEnum)
  assert.Error(t, err)
  })
 }

processor/transformprocessor/internal/common/functions.go

@@ -37,14 +37,16 @@ var registry = map[string]interface{}{
 
 type PathExpressionParser func(*Path) (GetSetter, error)
 
+type EnumParser func(*Path) (*Enum, bool)
+
 func DefaultFunctions() map[string]interface{} {
  return registry
 }
 
 // NewFunctionCall Visible for testing
-func NewFunctionCall(inv Invocation, functions map[string]interface{}, pathParser PathExpressionParser) (ExprFunc, error) {
+func NewFunctionCall(inv Invocation, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (ExprFunc, error) {
  if f, ok := functions[inv.Function]; ok {
- args, err := buildArgs(inv, reflect.TypeOf(f), functions, pathParser)
+ args, err := buildArgs(inv, reflect.TypeOf(f), functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
@@ -62,7 +64,7 @@ func NewFunctionCall(inv Invocation, functions map[string]interface{}, pathParse
  return nil, fmt.Errorf("undefined function %v", inv.Function)
 }
 
-func buildArgs(inv Invocation, fType reflect.Type, functions map[string]interface{}, pathParser PathExpressionParser) ([]reflect.Value, error) {
+func buildArgs(inv Invocation, fType reflect.Type, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) ([]reflect.Value, error) {
  args := make([]reflect.Value, 0)
  for i := 0; i < fType.NumIn(); i++ {
  argType := fType.In(i)
@@ -78,7 +80,7 @@ func buildArgs(inv Invocation, fType reflect.Type, functions map[string]interfac
  }
 
  argDef := inv.Arguments[i]
- err := buildArg(argDef, argType, i, &args, functions, pathParser)
+ err := buildArg(argDef, argType, i, &args, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
@@ -128,7 +130,7 @@ func buildSliceArg(inv Invocation, argType reflect.Type, startingIndex int, args
 }
 
 func buildArg(argDef Value, argType reflect.Type, index int, args *[]reflect.Value,
- functions map[string]interface{}, pathParser PathExpressionParser) error {
+ functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) error {
  switch argType.Name() {
  case "Setter":
  fallthrough
@@ -139,11 +141,17 @@ func buildArg(argDef Value, argType reflect.Type, index int, args *[]reflect.Val
  }
  *args = append(*args, reflect.ValueOf(arg))
  case "Getter":
- arg, err := NewGetter(argDef, functions, pathParser)
+ arg, err := NewGetter(argDef, functions, pathParser, enumParser)
  if err != nil {
  return fmt.Errorf("invalid argument at position %v %w", index, err)
  }
  *args = append(*args, reflect.ValueOf(arg))
+ case "Enum":
+ arg, ok := enumParser(argDef.Path)
+ if !ok {
+ return fmt.Errorf("invalid argument at position %v must be an Enum", index)
+ }
+ *args = append(*args, reflect.ValueOf(*arg))
  case "string":
  if argDef.String == nil {
  return fmt.Errorf("invalid argument at position %v, must be an string", index)

processor/transformprocessor/internal/common/functions_test.go

@@ -124,7 +124,7 @@ func Test_NewFunctionCall_invalid(t *testing.T) {
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- _, err := NewFunctionCall(tt.inv, functions, testParsePath)
+ _, err := NewFunctionCall(tt.inv, functions, testParsePath, testParseEnum)
  assert.Error(t, err)
  })
  }
@@ -144,6 +144,7 @@ func Test_NewFunctionCall(t *testing.T) {
  functions["testing_int"] = functionWithInt
  functions["testing_bool"] = functionWithBool
  functions["testing_multiple_args"] = functionWithMultipleArgs
+ functions["testing_enum"] = functionWithEnum
 
  tests := []struct {
  name string
@@ -349,10 +350,27 @@ func Test_NewFunctionCall(t *testing.T) {
  },
  },
  },
+ {
+ name: "Enum arg",
+ inv: Invocation{
+ Function: "testing_enum",
+ Arguments: []Value{
+ {
+ Path: &Path{
+ Fields: []Field{
+ {
+ Name: "TEST_ENUM",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- _, err := NewFunctionCall(tt.inv, functions, testParsePath)
+ _, err := NewFunctionCall(tt.inv, functions, testParsePath, testParseEnum)
  assert.NoError(t, err)
  })
  }
@@ -436,3 +454,9 @@ func functionThatHasAnError() (ExprFunc, error) {
  return "anything"
  }, err
 }
+
+func functionWithEnum(_ Enum) (ExprFunc, error) {
+ return func(ctx TransformContext) interface{} {
+ return "anything"
+ }, nil
+}

processor/transformprocessor/internal/common/parser.go

@@ -109,7 +109,7 @@ func (n *IsNil) Capture(_ []string) error {
  return nil
 }
 
-func ParseQueries(statements []string, functions map[string]interface{}, pathParser PathExpressionParser) ([]Query, error) {
+func ParseQueries(statements []string, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) ([]Query, error) {
  queries := make([]Query, 0)
  var errors error
 
@@ -119,12 +119,12 @@ func ParseQueries(statements []string, functions map[string]interface{}, pathPar
  errors = multierr.Append(errors, err)
  continue
  }
- function, err := NewFunctionCall(parsed.Invocation, functions, pathParser)
+ function, err := NewFunctionCall(parsed.Invocation, functions, pathParser, enumParser)
  if err != nil {
  errors = multierr.Append(errors, err)
  continue
  }
- condition, err := newConditionEvaluator(parsed.Condition, functions, pathParser)
+ condition, err := newConditionEvaluator(parsed.Condition, functions, pathParser, enumParser)
  if err != nil {
  errors = multierr.Append(errors, err)
  continue

processor/transformprocessor/internal/common/parser_test.go

@@ -407,3 +407,17 @@ func testParsePath(val *Path) (GetSetter, error) {
  }
  return nil, fmt.Errorf("bad path %v", val)
 }
+
+var testSymbolTable = map[string]Enum{
+ "TEST_ENUM": 0,
+}
+
+func testParseEnum(val *Path) (*Enum, bool) {
+ if val != nil && len(val.Fields) > 0 {
+ if enum, ok := testSymbolTable[val.Fields[0].Name]; ok {
+ return &enum, true
+ }
+ return nil, false
+ }
+ return nil, false
+}

processor/transformprocessor/internal/logs/logs.go

@@ -57,6 +57,43 @@ func (path pathGetSetter) Set(ctx common.TransformContext, val interface{}) {
  path.setter(ctx, val)
 }
 
+var symbolTable = map[string]common.Enum{
+ "SEVERITY_NUMBER_UNSPECIFIED": 0,
+ "SEVERITY_NUMBER_TRACE": 1,
+ "SEVERITY_NUMBER_TRACE2": 2,
+ "SEVERITY_NUMBER_TRACE3": 3,
+ "SEVERITY_NUMBER_TRACE4": 4,
+ "SEVERITY_NUMBER_DEBUG": 5,
+ "SEVERITY_NUMBER_DEBUG2": 6,
+ "SEVERITY_NUMBER_DEBUG3": 7,
+ "SEVERITY_NUMBER_DEBUG4": 8,
+ "SEVERITY_NUMBER_INFO": 9,
+ "SEVERITY_NUMBER_INFO2": 10,
+ "SEVERITY_NUMBER_INFO3": 11,
+ "SEVERITY_NUMBER_INFO4": 12,
+ "SEVERITY_NUMBER_WARN": 13,
+ "SEVERITY_NUMBER_WARN2": 14,
+ "SEVERITY_NUMBER_WARN3": 15,
+ "SEVERITY_NUMBER_WARN4": 16,
+ "SEVERITY_NUMBER_ERROR": 17,
+ "SEVERITY_NUMBER_ERROR2": 18,
+ "SEVERITY_NUMBER_ERROR3": 19,
+ "SEVERITY_NUMBER_ERROR4": 20,
+ "SEVERITY_NUMBER_FATAL": 21,
+ "SEVERITY_NUMBER_FATAL2": 22,
+ "SEVERITY_NUMBER_FATAL3": 23,
+ "SEVERITY_NUMBER_FATAL4": 24,
+}
+
+func ParseEnum(val *common.Path) (*common.Enum, bool) {
+ if val != nil && len(val.Fields) > 0 {
+ if enum, ok := symbolTable[val.Fields[0].Name]; ok {
+ return &enum, true
+ }
+ }
+ return nil, false
+}
+
 func ParsePath(val *common.Path) (common.GetSetter, error) {
  if val != nil && len(val.Fields) > 0 {
  return newPathGetSetter(val.Fields)
@@ -236,7 +273,7 @@ func accessObservedTimeUnixNano() pathGetSetter {
 func accessSeverityNumber() pathGetSetter {
  return pathGetSetter{
  getter: func(ctx common.TransformContext) interface{} {
- return ctx.GetItem().(plog.LogRecord).SeverityNumber()
+ return int64(ctx.GetItem().(plog.LogRecord).SeverityNumber())
  },
  setter: func(ctx common.TransformContext, val interface{}) {
  if i, ok := val.(int64); ok {