[pkg/telemetryquerylanguage] Add enums to TQL (open-telemetry#12337)

* Add enums to TQL

* Add changelog entry

* run make gotidy

* Added more tests

* Add another test

* Make Enum separate from Path

* Updated negative tests

* Updated README

* Updated README

* ran make gotidy

pkg/telemetryquerylanguage/tql/README.md

@@ -25,20 +25,21 @@ Example Invocations
 
 ### Values
 
-Values are the things that get passed to an Invocation or used in an Expression. Values can be either a Path, a Literal, or an Invocation.
+Values are the things that get passed to an Invocation or used in an Expression. Values can be either a Path, a Literal, an Enum, or an Invocation.
 
 Invocations as Values allows calling functions as parameters to other functions. See [Invocations](#invocations) for details on Invocation syntax.
 
 #### Paths
 
-A Path Value is a reference to a telemetry field. Paths are made up of string identifiers, dots (`.`), and square brackets combined with a string key (`["key"]`). **The interpretation of a Path is NOT implemented by the TQL.** Instead, the user must provide a `PathExpressionParser` that the TQL can use to interpret paths. As a result, how the Path parts are used is up to the user. However, it is recommended, that the parts be used like so:
+A Path Value is a reference to a telemetry field. Paths are made up of lowercase identifiers, dots (`.`), and square brackets combined with a string key (`["key"]`). **The interpretation of a Path is NOT implemented by the TQL.** Instead, the user must provide a `PathExpressionParser` that the TQL can use to interpret paths. As a result, how the Path parts are used is up to the user. However, it is recommended, that the parts be used like so:
 
 - Identifiers are used to map to a telemetry field.
 - Dots (`.`) are used to separate nested fields.
 - Square brackets and keys (`["key"]`) are used to access maps or slices.
 
 Example Paths
 - `name`
+- `value_double`
 - `resource.name`
 - `resource.attributes["key"]`
 
@@ -61,6 +62,15 @@ Example Literals
 - `nil`,
 - `0x0001`
 
+#### Enums
+
+Enums are uppercase identifiers that get interpreted during parsing and converted to an `int64`. **The interpretation of an Enum is NOT implemented by the TQL.** Instead, the user must provide a `EnumParser` that the TQL can use to interpret the Enum. The `EnumParser` returns an `int64` instead of a function, which means that the Enum's numeric value is retrieved during parsing instead of during execution.
+
+Within the grammar Enums are always used as `int64`. As a result, the Enum's symbol can be used as if it is an Int value.
+
+When defining a function that will be used as an Invocation by the TQL, if the function needs to take an Enum then the function must use the `Enum` type for that argument, not an `int64`.
+
+
 ### Expressions
 
 Expressions allow a decision to be made about whether an Invocation should be called. Expressions are optional. When used, the parsed query will include a `Condition`, which can be used to evaluate the result of the query's Expression. Expressions always evaluate to a boolean value (true or false).

pkg/telemetryquerylanguage/tql/boolean_value.go

@@ -55,15 +55,15 @@ func orFuncs(funcs []BoolExpressionEvaluator) BoolExpressionEvaluator {
  }
 }
 
-func newComparisonEvaluator(comparison *Comparison, functions map[string]interface{}, pathParser PathExpressionParser) (BoolExpressionEvaluator, error) {
+func newComparisonEvaluator(comparison *Comparison, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (BoolExpressionEvaluator, error) {
  if comparison == nil {
  return alwaysTrue, nil
  }
- left, err := NewGetter(comparison.Left, functions, pathParser)
+ left, err := NewGetter(comparison.Left, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
- right, err := NewGetter(comparison.Right, functions, pathParser)
+ right, err := NewGetter(comparison.Right, functions, pathParser, enumParser)
  // TODO(anuraaga): Check if both left and right are literals and const-evaluate
  if err != nil {
  return nil, err
@@ -87,17 +87,17 @@ func newComparisonEvaluator(comparison *Comparison, functions map[string]interfa
  return nil, fmt.Errorf("unrecognized boolean operation %v", comparison.Op)
 }
 
-func newBooleanExpressionEvaluator(expr *BooleanExpression, functions map[string]interface{}, pathParser PathExpressionParser) (BoolExpressionEvaluator, error) {
+func newBooleanExpressionEvaluator(expr *BooleanExpression, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (BoolExpressionEvaluator, error) {
  if expr == nil {
  return alwaysTrue, nil
  }
- f, err := newBooleanTermEvaluator(expr.Left, functions, pathParser)
+ f, err := newBooleanTermEvaluator(expr.Left, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
  funcs := []BoolExpressionEvaluator{f}
  for _, rhs := range expr.Right {
- f, err := newBooleanTermEvaluator(rhs.Term, functions, pathParser)
+ f, err := newBooleanTermEvaluator(rhs.Term, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
@@ -107,17 +107,17 @@ func newBooleanExpressionEvaluator(expr *BooleanExpression, functions map[string
  return orFuncs(funcs), nil
 }
 
-func newBooleanTermEvaluator(term *Term, functions map[string]interface{}, pathParser PathExpressionParser) (BoolExpressionEvaluator, error) {
+func newBooleanTermEvaluator(term *Term, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (BoolExpressionEvaluator, error) {
  if term == nil {
  return alwaysTrue, nil
  }
- f, err := newBooleanValueEvaluator(term.Left, functions, pathParser)
+ f, err := newBooleanValueEvaluator(term.Left, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
  funcs := []BoolExpressionEvaluator{f}
  for _, rhs := range term.Right {
- f, err := newBooleanValueEvaluator(rhs.Value, functions, pathParser)
+ f, err := newBooleanValueEvaluator(rhs.Value, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
@@ -127,13 +127,13 @@ func newBooleanTermEvaluator(term *Term, functions map[string]interface{}, pathP
  return andFuncs(funcs), nil
 }
 
-func newBooleanValueEvaluator(value *BooleanValue, functions map[string]interface{}, pathParser PathExpressionParser) (BoolExpressionEvaluator, error) {
+func newBooleanValueEvaluator(value *BooleanValue, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (BoolExpressionEvaluator, error) {
  if value == nil {
  return alwaysTrue, nil
  }
  switch {
  case value.Comparison != nil:
- comparison, err := newComparisonEvaluator(value.Comparison, functions, pathParser)
+ comparison, err := newComparisonEvaluator(value.Comparison, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }
@@ -144,7 +144,7 @@ func newBooleanValueEvaluator(value *BooleanValue, functions map[string]interfac
  }
  return alwaysFalse, nil
  case value.SubExpr != nil:
- return newBooleanExpressionEvaluator(value.SubExpr, functions, pathParser)
+ return newBooleanExpressionEvaluator(value.SubExpr, functions, pathParser, enumParser)
  }
 
  return nil, fmt.Errorf("unhandled boolean operation %v", value)

pkg/telemetryquerylanguage/tql/boolean_value_test.go

@@ -24,13 +24,13 @@ import (
 
 func Test_newComparisonEvaluator(t *testing.T) {
  tests := []struct {
- name string
- cond *Comparison
- item interface{}
+ name  string
+ comparison *Comparison
+ item  interface{}
  }{
  {
  name: "literals match",
- cond: &Comparison{
+ comparison: &Comparison{
  Left: Value{
  String: tqltest.Strp("hello"),
  },
@@ -42,7 +42,7 @@ func Test_newComparisonEvaluator(t *testing.T) {
  },
  {
  name: "literals don't match",
- cond: &Comparison{
+ comparison: &Comparison{
  Left: Value{
  String: tqltest.Strp("hello"),
  },
@@ -54,7 +54,7 @@ func Test_newComparisonEvaluator(t *testing.T) {
  },
  {
  name: "path expression matches",
- cond: &Comparison{
+ comparison: &Comparison{
  Left: Value{
  Path: &Path{
  Fields: []Field{
@@ -73,7 +73,7 @@ func Test_newComparisonEvaluator(t *testing.T) {
  },
  {
  name: "path expression not matches",
- cond: &Comparison{
+ comparison: &Comparison{
  Left: Value{
  Path: &Path{
  Fields: []Field{
@@ -91,32 +91,82 @@ func Test_newComparisonEvaluator(t *testing.T) {
  item: "bear",
  },
  {
- name: "no condition",
- cond: nil,
+ name: "no condition",
+ comparison: nil,
+ },
+
+ {
+ name: "compare Enum to int",
+ comparison: &Comparison{
+ Left: Value{
+ Enum: (*EnumSymbol)(tqltest.Strp("TEST_ENUM")),
+ },
+ Right: Value{
+ Int: tqltest.Intp(0),
+ },
+ Op: "==",
+ },
+ },
+ {
+ name: "compare int to Enum",
+ comparison: &Comparison{
+ Left: Value{
+ Int: tqltest.Intp(2),
+ },
+ Op: "==",
+ Right: Value{
+ Enum: (*EnumSymbol)(tqltest.Strp("TEST_ENUM_TWO")),
+ },
+ },
  },
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- evaluate, err := newComparisonEvaluator(tt.cond, DefaultFunctionsForTests(), testParsePath)
+ evaluate, err := newComparisonEvaluator(tt.comparison, DefaultFunctionsForTests(), testParsePath, testParseEnum)
  assert.NoError(t, err)
  assert.True(t, evaluate(tqltest.TestTransformContext{
  Item: tt.item,
  }))
  })
  }
+}
 
- t.Run("invalid", func(t *testing.T) {
- _, err := newComparisonEvaluator(&Comparison{
- Left: Value{
- String: tqltest.Strp("bear"),
+func Test_newConditionEvaluator_invalid(t *testing.T) {
+ tests := []struct {
+ name string
+ comparison *Comparison
+ }{
+ {
+ name: "unknown operation",
+ comparison: &Comparison{
+ Left: Value{
+ String: tqltest.Strp("bear"),
+ },
+ Op: "<>",
+ Right: Value{
+ String: tqltest.Strp("cat"),
+ },
  },
- Op: "<>",
- Right: Value{
- String: tqltest.Strp("cat"),
+ },
+ {
+ name: "unknown Path",
+ comparison: &Comparison{
+ Left: Value{
+ Enum: (*EnumSymbol)(tqltest.Strp("SYMBOL_NOT_FOUND")),
+ },
+ Op: "==",
+ Right: Value{
+ String: tqltest.Strp("trash"),
+ },
  },
- }, DefaultFunctionsForTests(), testParsePath)
- assert.Error(t, err)
- })
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := newComparisonEvaluator(tt.comparison, DefaultFunctionsForTests(), testParsePath, testParseEnum)
+ assert.Error(t, err)
+ })
+ }
 }
 
 func Test_newBooleanExpressionEvaluator(t *testing.T) {
@@ -302,7 +352,7 @@ func Test_newBooleanExpressionEvaluator(t *testing.T) {
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- evaluate, err := newBooleanExpressionEvaluator(tt.expr, DefaultFunctionsForTests(), testParsePath)
+ evaluate, err := newBooleanExpressionEvaluator(tt.expr, DefaultFunctionsForTests(), testParsePath, testParseEnum)
  assert.NoError(t, err)
  assert.Equal(t, tt.want, evaluate(tqltest.TestTransformContext{
  Item: nil,

pkg/telemetryquerylanguage/tql/expression.go

@@ -28,6 +28,8 @@ type TransformContext interface {
 
 type ExprFunc func(ctx TransformContext) interface{}
 
+type Enum int64
+
 type Getter interface {
  Get(ctx TransformContext) interface{}
 }
@@ -57,7 +59,7 @@ func (g exprGetter) Get(ctx TransformContext) interface{} {
  return g.expr(ctx)
 }
 
-func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpressionParser) (Getter, error) {
+func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpressionParser, enumParser EnumParser) (Getter, error) {
  if val.IsNil != nil && *val.IsNil {
  return &literal{value: nil}, nil
  }
@@ -78,6 +80,14 @@ func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpre
  return &literal{value: ([]byte)(*b)}, nil
  }
 
+ if val.Enum != nil {
+ enum, err := enumParser(val.Enum)
+ if err != nil {
+ return nil, err
+ }
+ return &literal{value: int64(*enum)}, nil
+ }
+
  if val.Path != nil {
  return pathParser(val.Path)
  }
@@ -86,7 +96,7 @@ func NewGetter(val Value, functions map[string]interface{}, pathParser PathExpre
  // In practice, can't happen since the DSL grammar guarantees one is set
  return nil, fmt.Errorf("no value field set. This is a bug in the transformprocessor")
  }
- call, err := NewFunctionCall(*val.Invocation, functions, pathParser)
+ call, err := NewFunctionCall(*val.Invocation, functions, pathParser, enumParser)
  if err != nil {
  return nil, err
  }

pkg/telemetryquerylanguage/tql/expression_test.go

@@ -98,13 +98,20 @@ func Test_newGetter(t *testing.T) {
  },
  want: "world",
  },
+ {
+ name: "enum",
+ val: Value{
+ Enum: (*EnumSymbol)(tqltest.Strp("TEST_ENUM_ONE")),
+ },
+ want: int64(1),
+ },
  }
 
  functions := map[string]interface{}{"hello": hello}
 
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- reader, err := NewGetter(tt.val, functions, testParsePath)
+ reader, err := NewGetter(tt.val, functions, testParsePath, testParseEnum)
  assert.NoError(t, err)
  val := reader.Get(tqltest.TestTransformContext{
  Item: tt.want,
@@ -114,7 +121,7 @@ func Test_newGetter(t *testing.T) {
  }
 
  t.Run("empty value", func(t *testing.T) {
- _, err := NewGetter(Value{}, functions, testParsePath)
+ _, err := NewGetter(Value{}, functions, testParsePath, testParseEnum)
  assert.Error(t, err)
  })
 }