diff --git a/examples/logline-filtering/README.md b/examples/logline-filtering/README.md new file mode 100644 index 0000000000000..9ae695a33ae12 --- /dev/null +++ b/examples/logline-filtering/README.md @@ -0,0 +1,13 @@ +## Filtering log messages based on content + +Filelog receiver provides support for filtering logs based on their content. This can be achieved by using +the [filter operator](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/filter.md), +configured with matching regular expressions. + +With this happening at the collection point, a lot of resources at the destination backend +can be saved since no additional processing would need to take place. + +A full configuration example on how to filter out logs that start with the `INFO:` pattern is +provided in the [example config](./otel-col-config-filter-out-logs.yaml). +A full configuration example on how to only collect logs that start with the `WARN:` pattern is provided in +the [example config](./otel-col-config-filter-in-logs.yaml) \ No newline at end of file diff --git a/examples/logline-filtering/otel-col-config-filter-in-logs.yaml b/examples/logline-filtering/otel-col-config-filter-in-logs.yaml new file mode 100644 index 0000000000000..a03f7b08b36ff --- /dev/null +++ b/examples/logline-filtering/otel-col-config-filter-in-logs.yaml @@ -0,0 +1,17 @@ +receivers: + filelog: + include: [/var/log/busybox/simple.log] + operators: + - type: filter + expr: 'body not matches "^WARN:"' + +service: + pipelines: + logs: + receivers: [filelog] + exporters: [otlp/custom] + processors: [] + +exporters: + otlp/custom: + endpoint: http://0.0.0.0:4242 diff --git a/examples/logline-filtering/otel-col-config-filter-out-logs.yaml b/examples/logline-filtering/otel-col-config-filter-out-logs.yaml new file mode 100644 index 0000000000000..ff19b50c68898 --- /dev/null +++ b/examples/logline-filtering/otel-col-config-filter-out-logs.yaml @@ -0,0 +1,17 @@ +receivers: + filelog: + include: [/var/log/busybox/simple.log] + operators: + - type: filter + expr: 'body matches "^INFO:"' + +service: + pipelines: + logs: + receivers: [filelog] + exporters: [otlp/custom] + processors: [] + +exporters: + otlp/custom: + endpoint: http://0.0.0.0:4242 diff --git a/pkg/stanza/operator/transformer/filter/filter_test.go b/pkg/stanza/operator/transformer/filter/filter_test.go index 5f9eed013b8f5..ced34d202e115 100644 --- a/pkg/stanza/operator/transformer/filter/filter_test.go +++ b/pkg/stanza/operator/transformer/filter/filter_test.go @@ -46,6 +46,26 @@ func TestTransformer(t *testing.T) { `body.message == "test_message"`, false, }, + { + "FilterOutRegexp", + &entry.Entry{ + Body: map[string]any{ + "message": "INFO: this is an info message", + }, + }, + `body.message matches "^INFO:"`, + true, + }, + { + "FilterInRegexp", + &entry.Entry{ + Body: map[string]any{ + "message": "WARN: this is a warning message", + }, + }, + `body.message not matches "^WARN:"`, + false, + }, { "MatchAttribute", &entry.Entry{