[processor/transform] Add new delete functions (open-telemetry#11824)

* Add new delete functions

* Updated changelog

processor/transformprocessor/README.md

@@ -34,6 +34,10 @@ Supported functions:
 e.g., `set(attributes["http.path"], "/foo")`, `set(name, attributes["http.route"])`, `set(trace_state["svc"], "example")`, `set(attributes["source"], trace_state["source"])`. If `value` resolves to `nil`, e.g.
 it references an unset map value, there will be no action.
 
+- `delete_key(target, key)` - `target` is a path expression to a map type field. `key` is a string that is a key in the map. The key will be deleted from the map. e.g., `delete_key(attributes, "http.request.header.authorization")`
+
+- `delete_matching_keys(target, pattern)` - `target` is a path expression to a map type field. `pattern` is a regex string. All keys that match the pattern will be deleted from the map. e.g., `delete_matching_keys(attributes, ".*\.header\.authorization")`
+
 - `keep_keys(target, string...)` - `target` is a path expression to a map type field. The map will be mutated to only contain
 the fields specified by the list of strings. e.g., `keep_keys(attributes, "http.method")`, `keep_keys(attributes, "http.method", "http.route")`
 

processor/transformprocessor/internal/common/func_delete_key.go

@@ -0,0 +1,31 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common"
+
+import "go.opentelemetry.io/collector/pdata/pcommon"
+
+func deleteKey(target Getter, key string) (ExprFunc, error) {
+ return func(ctx TransformContext) interface{} {
+ val := target.Get(ctx)
+ if val == nil {
+ return nil
+ }
+
+ if attrs, ok := val.(pcommon.Map); ok {
+ attrs.Remove(key)
+ }
+ return nil
+ }, nil
+}

processor/transformprocessor/internal/common/func_delete_key_test.go

@@ -0,0 +1,137 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "go.opentelemetry.io/collector/pdata/pcommon"
+
+ "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common/testhelper"
+)
+
+func Test_deleteKey(t *testing.T) {
+ input := pcommon.NewMap()
+ input.InsertString("test", "hello world")
+ input.InsertInt("test2", 3)
+ input.InsertBool("test3", true)
+
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ return ctx.GetItem()
+ },
+ }
+
+ tests := []struct {
+ name string
+ target Getter
+ key string
+ want func(pcommon.Map)
+ }{
+ {
+ name: "delete test",
+ target: target,
+ key: "test",
+ want: func(expectedMap pcommon.Map) {
+ expectedMap.Clear()
+ expectedMap.InsertBool("test3", true)
+ expectedMap.InsertInt("test2", 3)
+ },
+ },
+ {
+ name: "delete test2",
+ target: target,
+ key: "test2",
+ want: func(expectedMap pcommon.Map) {
+ expectedMap.Clear()
+ expectedMap.InsertString("test", "hello world")
+ expectedMap.InsertBool("test3", true)
+ },
+ },
+ {
+ name: "delete nothing",
+ target: target,
+ key: "not a valid key",
+ want: func(expectedMap pcommon.Map) {
+ expectedMap.Clear()
+ expectedMap.InsertString("test", "hello world")
+ expectedMap.InsertInt("test2", 3)
+ expectedMap.InsertBool("test3", true)
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ scenarioMap := pcommon.NewMap()
+ input.CopyTo(scenarioMap)
+
+ ctx := testhelper.TestTransformContext{
+ Item: scenarioMap,
+ }
+
+ exprFunc, _ := deleteKey(tt.target, tt.key)
+ exprFunc(ctx)
+
+ expected := pcommon.NewMap()
+ tt.want(expected)
+
+ assert.Equal(t, expected, scenarioMap)
+ })
+ }
+}
+
+func Test_deleteKey_bad_input(t *testing.T) {
+ input := pcommon.NewValueString("not a map")
+ ctx := testhelper.TestTransformContext{
+ Item: input,
+ }
+
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ return ctx.GetItem()
+ },
+ setter: func(ctx TransformContext, val interface{}) {
+ t.Errorf("nothing should be set in this scenario")
+ },
+ }
+
+ key := "anything"
+
+ exprFunc, _ := deleteKey(target, key)
+ exprFunc(ctx)
+
+ assert.Equal(t, pcommon.NewValueString("not a map"), input)
+}
+
+func Test_deleteKey_get_nil(t *testing.T) {
+ ctx := testhelper.TestTransformContext{
+ Item: nil,
+ }
+
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ return ctx.GetItem()
+ },
+ setter: func(ctx TransformContext, val interface{}) {
+ t.Errorf("nothing should be set in this scenario")
+ },
+ }
+
+ key := "anything"
+
+ exprFunc, _ := deleteKey(target, key)
+ exprFunc(ctx)
+}

processor/transformprocessor/internal/common/func_delete_matching_keys.go

@@ -0,0 +1,42 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common"
+
+import (
+ "fmt"
+ "regexp"
+
+ "go.opentelemetry.io/collector/pdata/pcommon"
+)
+
+func deleteMatchingKeys(target Getter, pattern string) (ExprFunc, error) {
+ compiledPattern, err := regexp.Compile(pattern)
+ if err != nil {
+ return nil, fmt.Errorf("the regex pattern supplied to delete_matching_keys is not a valid pattern: %w", err)
+ }
+ return func(ctx TransformContext) interface{} {
+ val := target.Get(ctx)
+ if val == nil {
+ return nil
+ }
+
+ if attrs, ok := val.(pcommon.Map); ok {
+ attrs.RemoveIf(func(key string, _ pcommon.Value) bool {
+ return compiledPattern.MatchString(key)
+ })
+ }
+ return nil
+ }, nil
+}

processor/transformprocessor/internal/common/func_delete_matching_keys_test.go

@@ -0,0 +1,140 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "go.opentelemetry.io/collector/pdata/pcommon"
+
+ "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common/testhelper"
+)
+
+func Test_deleteMatchingKeys(t *testing.T) {
+ input := pcommon.NewMap()
+ input.InsertString("test", "hello world")
+ input.InsertInt("test2", 3)
+ input.InsertBool("test3", true)
+
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ return ctx.GetItem()
+ },
+ }
+
+ tests := []struct {
+ name string
+ target Getter
+ pattern string
+ want func(pcommon.Map)
+ }{
+ {
+ name: "delete everything",
+ target: target,
+ pattern: "test.*",
+ want: func(expectedMap pcommon.Map) {
+ expectedMap.Clear()
+ expectedMap.EnsureCapacity(3)
+ },
+ },
+ {
+ name: "delete attributes that end in a number",
+ target: target,
+ pattern: "\\d$",
+ want: func(expectedMap pcommon.Map) {
+ expectedMap.Clear()
+ expectedMap.InsertString("test", "hello world")
+ },
+ },
+ {
+ name: "delete nothing",
+ target: target,
+ pattern: "not a matching pattern",
+ want: func(expectedMap pcommon.Map) {
+ expectedMap.Clear()
+ expectedMap.InsertString("test", "hello world")
+ expectedMap.InsertInt("test2", 3)
+ expectedMap.InsertBool("test3", true)
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ scenarioMap := pcommon.NewMap()
+ input.CopyTo(scenarioMap)
+
+ ctx := testhelper.TestTransformContext{
+ Item: scenarioMap,
+ }
+
+ exprFunc, _ := deleteMatchingKeys(tt.target, tt.pattern)
+ exprFunc(ctx)
+
+ expected := pcommon.NewMap()
+ tt.want(expected)
+
+ assert.Equal(t, expected, scenarioMap)
+ })
+ }
+}
+
+func Test_deleteMatchingKeys_bad_input(t *testing.T) {
+ input := pcommon.NewValueInt(1)
+ ctx := testhelper.TestTransformContext{
+ Item: input,
+ }
+
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ return ctx.GetItem()
+ },
+ }
+
+ exprFunc, err := deleteMatchingKeys(target, "anything")
+ assert.Nil(t, err)
+ exprFunc(ctx)
+
+ assert.Equal(t, pcommon.NewValueInt(1), input)
+}
+
+func Test_deleteMatchingKeys_get_nil(t *testing.T) {
+ ctx := testhelper.TestTransformContext{
+ Item: nil,
+ }
+
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ return ctx.GetItem()
+ },
+ }
+
+ exprFunc, _ := deleteMatchingKeys(target, "anything")
+ exprFunc(ctx)
+}
+
+func Test_deleteMatchingKeys_invalid_pattern(t *testing.T) {
+ target := &testGetSetter{
+ getter: func(ctx TransformContext) interface{} {
+ t.Errorf("nothing should be received in this scenario")
+ return nil
+ },
+ }
+
+ invalidRegexPattern := "*"
+ exprFunc, err := deleteMatchingKeys(target, invalidRegexPattern)
+ assert.Nil(t, exprFunc)
+ assert.Contains(t, err.Error(), "error parsing regexp:")
+}

processor/transformprocessor/internal/common/functions.go

@@ -31,6 +31,8 @@ var registry = map[string]interface{}{
  "replace_all_matches": replaceAllMatches,
  "replace_pattern": replacePattern,
  "replace_all_patterns": replaceAllPatterns,
+ "delete_key": deleteKey,
+ "delete_matching_keys": deleteMatchingKeys,
 }
 
 type PathExpressionParser func(*Path) (GetSetter, error)

processor/transformprocessor/internal/logs/processor_test.go

@@ -109,6 +109,21 @@ func TestProcess(t *testing.T) {
  td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("test", "pass")
  },
  },
+ {
+ query: `delete_key(attributes, "http.url") where body == "operationA"`,
+ want: func(td plog.Logs) {
+ td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear()
+ td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.method", "get")
+ td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.path", "/health")
+ },
+ },
+ {
+ query: `delete_matching_keys(attributes, "http.*t.*") where body == "operationA"`,
+ want: func(td plog.Logs) {
+ td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear()
+ td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.url", "http:https://localhost/health")
+ },
+ },
  }
 
  for _, tt := range tests {