add ability to control matching undefined severity records

internal/coreinternal/processor/filterconfig/config.go

@@ -105,6 +105,12 @@ type MatchProperties struct {
  // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match.
  LogMinSeverity plog.SeverityNumber `mapstructure:"log_min_severity"`
 
+ // LogMatchUndefinedSeverity controls whether logs with "undefined" severity matches.
+ // If this is true, entries with undefined severity will match.
+ // This field is only applicable if LogMinSeverity is specified;
+ // If LogMinSeverity is not specified, this field does nothing.
+ LogMatchUndefinedSeverity bool `mapstructure:"log_match_undefined_severity"`
+
  // MetricNames is a list of strings to match metric name against.
  // A match occurs if metric name matches at least one item in the list.
  // This field is optional.

internal/coreinternal/processor/filterlog/filterlog.go

@@ -80,7 +80,7 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) {
 
  var severityNumberMatcher Matcher
  if mp.LogMinSeverity != plog.SeverityNumberUNDEFINED {
- severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity)
+ severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity, mp.LogMatchUndefinedSeverity)
  }
 
  return &propertiesMatcher{

internal/coreinternal/processor/filterlog/severity_matcher.go

@@ -8,19 +8,21 @@ import (
 // severtiyNumberMatcher is a Matcher that matches if the input log record has a severity higher than
 // the minSeverityNumber.
 type severityNumberMatcher struct {
+ matchUndefined bool
  minSeverityNumber plog.SeverityNumber
 }
 
-func newSeverityNumberMatcher(sn plog.SeverityNumber) severityNumberMatcher {
+func newSeverityNumberMatcher(minSeverity plog.SeverityNumber, matchUndefined bool) severityNumberMatcher {
  return severityNumberMatcher{
- minSeverityNumber: sn,
+ minSeverityNumber: minSeverity,
+ matchUndefined: matchUndefined,
  }
 }
 
 func (snm severityNumberMatcher) MatchLogRecord(lr plog.LogRecord, _ pcommon.Resource, _ pcommon.InstrumentationScope) bool {
  // We explicitly do not match UNDEFINED severity.
  if lr.SeverityNumber() == plog.SeverityNumberUNDEFINED {
- return false
+ return snm.matchUndefined
  }
 
  // If the log records severity is greater than or equal to the desired severity, it matches

internal/coreinternal/processor/filterlog/severity_matcher_test.go

@@ -10,10 +10,11 @@ import (
 
 func TestSeverityMatcher_MatchLogRecord(t *testing.T) {
  testCases := []struct {
- name string
- minSeverity plog.SeverityNumber
- inputSeverity plog.SeverityNumber
- matches bool
+ name string
+ minSeverity plog.SeverityNumber
+ matchUndefined bool
+ inputSeverity plog.SeverityNumber
+ matches bool
  }{
  {
  name: "INFO matches if TRACE is min",
@@ -57,11 +58,18 @@ func TestSeverityMatcher_MatchLogRecord(t *testing.T) {
  inputSeverity: plog.SeverityNumberUNDEFINED,
  matches: false,
  },
+ {
+ name: "UNDEFINED matches if matchUndefined is true",
+ minSeverity: plog.SeverityNumberUNDEFINED,
+ matchUndefined: true,
+ inputSeverity: plog.SeverityNumberUNDEFINED,
+ matches: false,
+ },
  }
 
  for _, tc := range testCases {
  t.Run(tc.name, func(t *testing.T) {
- matcher := newSeverityNumberMatcher(tc.minSeverity)
+ matcher := newSeverityNumberMatcher(tc.minSeverity, tc.matchUndefined)
 
  r := pcommon.NewResource()
  i := pcommon.NewInstrumentationScope()

processor/filterprocessor/README.md

@@ -39,6 +39,7 @@ For logs:
  e.g. if this is "INFO", all log records with "INFO" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched.
  If this option is specified, no logs with "DEFAULT" severity will be matched.
  The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity)
+- `match_undefined_severity`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. If `min_severity` is not specified, this option does nothing. If `match_undefined_severity` is set to true, log records with no severity will be matched. If set to false, log records with no severity will not be matched.
 
 For metrics:
 
@@ -104,9 +105,11 @@ processors:
  - WARN[2-4]?
  - ERROR[2-4]?
  # Filter out logs below INFO (no DEBUG or TRACE level logs)
+ # log records
  logs/severity_number:
  include:
  min_severity: "INFO"
+ match_undefined_severity: true
  logs/bodies:
  include:
  match_type: regexp

processor/filterprocessor/config.go

@@ -167,6 +167,11 @@ type LogMatchProperties struct {
  // this field is case-insensitive ("INFO" == "info")
  MinSeverity logSeverity `mapstructure:"min_severity"`
 
+ // MatchUndefinedSeverity lets logs records with "unknown" severity match.
+ // This is only applied if MinSeverity is set.
+ // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity.
+ MatchUndefinedSeverity bool `mapstructure:"match_undefined_severity"`
+
  // LogBodies is a list of strings that the LogRecord's body field must match
  // against.
  LogBodies []string `mapstructure:"bodies"`
@@ -190,11 +195,12 @@ func (lmp LogMatchProperties) matchProperties() *filterconfig.MatchProperties {
  Config: filterset.Config{
  MatchType: filterset.MatchType(lmp.LogMatchType),
  },
- Resources: lmp.ResourceAttributes,
- Attributes: lmp.RecordAttributes,
- LogSeverityTexts: lmp.SeverityTexts,
- LogBodies: lmp.LogBodies,
- LogMinSeverity: lmp.MinSeverity.severityNumber(),
+ Resources: lmp.ResourceAttributes,
+ Attributes: lmp.RecordAttributes,
+ LogSeverityTexts: lmp.SeverityTexts,
+ LogBodies: lmp.LogBodies,
+ LogMinSeverity: lmp.MinSeverity.severityNumber(),
+ LogMatchUndefinedSeverity: lmp.MatchUndefinedSeverity,
  }
 }
 

processor/filterprocessor/config_test.go

@@ -445,7 +445,8 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) {
 // TestLoadingConfigMinSeverityNumberLogs tests loading testdata/config_logs_min_severity.yaml
 func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) {
  testDataLogPropertiesInclude := &LogMatchProperties{
- MinSeverity: logSeverity("INFO"),
+ MinSeverity: logSeverity("INFO"),
+ MatchUndefinedSeverity: true,
  }
 
  testDataLogPropertiesExclude := &LogMatchProperties{

processor/filterprocessor/filter_processor_logs_test.go

@@ -513,6 +513,18 @@ var (
  {"log3"},
  },
  },
+ {
+ name: "includeMinSeverityFATAL+undefined",
+ inc: &LogMatchProperties{
+ LogMatchType: Regexp,
+ MinSeverity: logSeverity("FATAL"),
+ MatchUndefinedSeverity: true,
+ },
+ inLogs: testResourceLogs(inLogForSeverityNumber),
+ outLN: [][]string{
+ {"log4"},
+ },
+ },
  {
  name: "excludeMinSeverityINFO",
  exc: &LogMatchProperties{
@@ -536,6 +548,18 @@ var (
  {"log4"},
  },
  },
+ {
+ name: "excludeMinSeverityINFO+undefined",
+ exc: &LogMatchProperties{
+ LogMatchType: Regexp,
+ MinSeverity: logSeverity("INFO"),
+ MatchUndefinedSeverity: true,
+ },
+ inLogs: testResourceLogs(inLogForSeverityNumber),
+ outLN: [][]string{
+ {"log1"},
+ },
+ },
  }
 )
 

processor/filterprocessor/testdata/config_logs_min_severity.yaml

@@ -6,8 +6,10 @@ processors:
  logs:
  # any logs NOT matching filters are excluded from remainder of pipeline
  # This filters out all logs below "INFO" level (the whole DEBUG and TRACE ranges)
+ # Logs with no defined severity will also be matched.
  include:
  min_severity: "INFO"
+ match_undefined_severity: true
  filter/exclude:
  logs:
  # any logs matching filters are excluded from remainder of pipeline
@@ -18,9 +20,11 @@ processors:
  filter/includeexclude:
  logs:
  # if both include and exclude are specified, include filters are applied first
- # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass
+ # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass,
+ # as well as logs with undefined severity
  include:
  min_severity: "INFO"
+ match_undefined_severity: true
 
  exclude:
  min_severity: "ERROR"