Move config for log severity_number matching to new object

internal/coreinternal/processor/filterconfig/config.go

@@ -102,15 +102,8 @@ type MatchProperties struct {
  // against.
  LogSeverityTexts []string `mapstructure:"log_severity_texts"`
 
- // LogMinSeverity is the lowest severity that may be matched.
- // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match.
- LogMinSeverity plog.SeverityNumber `mapstructure:"log_min_severity"`
-
- // LogMatchUndefinedSeverity controls whether logs with "undefined" severity matches.
- // If this is true, entries with undefined severity will match.
- // This field is only applicable if LogMinSeverity is specified;
- // If LogMinSeverity is not specified, this field does nothing.
- LogMatchUndefinedSeverity bool `mapstructure:"log_match_undefined_severity"`
+ // LogSeverityNumber defines how to match against a log record's SeverityNumber, if defined.
+ LogSeverityNumber *LogSeverityNumberMatchProperties `mapstructure:"log_severity_number"`
 
  // MetricNames is a list of strings to match metric name against.
  // A match occurs if metric name matches at least one item in the list.
@@ -144,6 +137,10 @@ func (mp *MatchProperties) ValidateForSpans() error {
  return errors.New("log_severity_texts should not be specified for trace spans")
  }
 
+ if mp.LogSeverityNumber != nil {
+ return errors.New("log_severity_number should not be specified for trace spans")
+ }
+
  if len(mp.Services) == 0 && len(mp.SpanNames) == 0 && len(mp.Attributes) == 0 &&
  len(mp.Libraries) == 0 && len(mp.Resources) == 0 {
  return errors.New(`at least one of "services", "span_names", "attributes", "libraries" or "resources" field must be specified`)
@@ -160,8 +157,8 @@ func (mp *MatchProperties) ValidateForLogs() error {
 
  if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 &&
  len(mp.Resources) == 0 && len(mp.LogBodies) == 0 &&
- len(mp.LogSeverityTexts) == 0 && mp.LogMinSeverity == plog.SeverityNumberUNDEFINED {
- return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`)
+ len(mp.LogSeverityTexts) == 0 && mp.LogSeverityNumber == nil {
+ return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`)
  }
 
  return nil
@@ -190,3 +187,14 @@ type InstrumentationLibrary struct {
  // 1 1 yes
  Version *string `mapstructure:"version"`
 }
+
+// LogSeverityNumberMatchProperties defines how to match based on a log record's SeverityNumber field.
+type LogSeverityNumberMatchProperties struct {
+ // Min is the lowest severity that may be matched.
+ // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match.
+ Min plog.SeverityNumber `mapstructure:"min"`
+
+ // MatchUndefined controls whether logs with "undefined" severity matches.
+ // If this is true, entries with undefined severity will match.
+ MatchUndefined bool `mapstructure:"match_undefined"`
+}

internal/coreinternal/processor/filterlog/filterlog.go

@@ -79,8 +79,8 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) {
  }
 
  var severityNumberMatcher Matcher
- if mp.LogMinSeverity != plog.SeverityNumberUNDEFINED {
- severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity, mp.LogMatchUndefinedSeverity)
+ if mp.LogSeverityNumber != nil {
+ severityNumberMatcher = newSeverityNumberMatcher(mp.LogSeverityNumber.Min, mp.LogSeverityNumber.MatchUndefined)
  }
 
  return &propertiesMatcher{

internal/coreinternal/processor/filterlog/filterlog_test.go

@@ -41,15 +41,15 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) {
  {
  name: "empty_property",
  property: filterconfig.MatchProperties{},
- errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`,
+ errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`,
  },
  {
  name: "empty_log_bodies_and_attributes",
  property: filterconfig.MatchProperties{
  LogBodies: []string{},
  LogSeverityTexts: []string{},
  },
- errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`,
+ errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`,
  },
  {
  name: "span_properties",
@@ -126,8 +126,10 @@ func TestLogRecord_Matching_False(t *testing.T) {
  {
  name: "log_min_severity_trace_dont_match",
  properties: &filterconfig.MatchProperties{
- Config: *createConfig(filterset.Regexp),
- LogMinSeverity: plog.SeverityNumberINFO,
+ Config: *createConfig(filterset.Regexp),
+ LogSeverityNumber: &filterconfig.LogSeverityNumberMatchProperties{
+ Min: plog.SeverityNumberINFO,
+ },
  },
  },
  }
@@ -184,8 +186,10 @@ func TestLogRecord_Matching_True(t *testing.T) {
  {
  name: "log_min_severity_match",
  properties: &filterconfig.MatchProperties{
- Config: *createConfig(filterset.Regexp),
- LogMinSeverity: plog.SeverityNumberDEBUG,
+ Config: *createConfig(filterset.Regexp),
+ LogSeverityNumber: &filterconfig.LogSeverityNumberMatchProperties{
+ Min: plog.SeverityNumberDEBUG,
+ },
  },
  },
  }

processor/filterprocessor/README.md

@@ -35,11 +35,13 @@ For logs:
  A match occurs if the record matches any expression in this given list.
 - `bodies`: Bodies defines a list of possible log bodies to match the logs against.
  A match occurs if the record matches any expression in this given list.
-- `min_severity`: MinSeverity defines the minimum severity with which a log record should match.
- e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched.
- The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity)
- By default, logs with undefined severity are not matched.
-- `match_undefined_severity`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. If `min_severity` is not specified, this option does nothing. If `match_undefined_severity` is set to true, log records with no severity will be matched. If set to false, log records with no severity will not be matched.
+- `severity_number`: SeverityNumber defines how to match a record based on its SeverityNumber.
+ The following can be configured for matching a log record's SeverityNumber:
+ - `min`: Min defines the minimum severity with which a log record should match.
+ e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched.
+ The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity). You may use either the numerical "SeverityNumber" or the "Short Name"
+ - `match_undefined`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option.
+ By default, this is `false`.
 
 For metrics:
 
@@ -104,12 +106,13 @@ processors:
  - INFO[2-4]?
  - WARN[2-4]?
  - ERROR[2-4]?
- # Filter out logs below INFO (no DEBUG or TRACE level logs)
- # log records
+ # Filter out logs below INFO (no DEBUG or TRACE level logs),
+ # retaining logs with undefined severity
  logs/severity_number:
  include:
- min_severity: "INFO"
- match_undefined_severity: true
+ severity_number:
+ min: "INFO"
+ match_undefined: true
  logs/bodies:
  include:
  match_type: regexp

processor/filterprocessor/config.go

@@ -185,16 +185,8 @@ type LogMatchProperties struct {
  // against.
  SeverityTexts []string `mapstructure:"severity_texts"`
 
- // MinSeverity is the minimum severity needed for the log record to match.
- // This corresponds to the short names specified here:
- // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity
- // this field is case-insensitive ("INFO" == "info")
- MinSeverity logSeverity `mapstructure:"min_severity"`
-
- // MatchUndefinedSeverity lets logs records with "unknown" severity match.
- // This is only applied if MinSeverity is set.
- // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity.
- MatchUndefinedSeverity bool `mapstructure:"match_undefined_severity"`
+ // SeverityNumberProperties defines how to match against a log record's SeverityNumber, if defined.
+ SeverityNumberProperties *LogSeverityNumberMatchProperties `mapstructure:"severity_number"`
 
  // LogBodies is a list of strings that the LogRecord's body field must match
  // against.
@@ -203,29 +195,58 @@ type LogMatchProperties struct {
 
 // validate checks that the LogMatchProperties is valid
 func (lmp LogMatchProperties) validate() error {
- return lmp.MinSeverity.validate()
+ if lmp.SeverityNumberProperties != nil {
+ return lmp.SeverityNumberProperties.validate()
+ }
+ return nil
 }
 
 // isEmpty returns true if the properties is "empty" (meaning, there are no filters specified)
 // if this is the case, the filter should be ignored.
 func (lmp LogMatchProperties) isEmpty() bool {
  return len(lmp.ResourceAttributes) == 0 && len(lmp.RecordAttributes) == 0 &&
- len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 && lmp.MinSeverity == ""
+ len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 &&
+ lmp.SeverityNumberProperties == nil
 }
 
 // matchProperties converts the LogMatchProperties to a corresponding filterconfig.MatchProperties
 func (lmp LogMatchProperties) matchProperties() *filterconfig.MatchProperties {
- return &filterconfig.MatchProperties{
+ mp := &filterconfig.MatchProperties{
  Config: filterset.Config{
  MatchType: filterset.MatchType(lmp.LogMatchType),
  },
- Resources: lmp.ResourceAttributes,
- Attributes: lmp.RecordAttributes,
- LogSeverityTexts: lmp.SeverityTexts,
- LogBodies: lmp.LogBodies,
- LogMinSeverity: lmp.MinSeverity.severityNumber(),
- LogMatchUndefinedSeverity: lmp.MatchUndefinedSeverity,
+ Resources: lmp.ResourceAttributes,
+ Attributes: lmp.RecordAttributes,
+ LogSeverityTexts: lmp.SeverityTexts,
+ LogBodies: lmp.LogBodies,
  }
+
+ // Include SeverityNumberProperties if defined
+ if lmp.SeverityNumberProperties != nil {
+ mp.LogSeverityNumber = &filterconfig.LogSeverityNumberMatchProperties{
+ Min: lmp.SeverityNumberProperties.Min.severityNumber(),
+ MatchUndefined: lmp.SeverityNumberProperties.MatchUndefined,
+ }
+ }
+
+ return mp
+}
+
+type LogSeverityNumberMatchProperties struct {
+ // Min is the minimum severity needed for the log record to match.
+ // This corresponds to the short names specified here:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity
+ // this field is case-insensitive ("INFO" == "info")
+ Min logSeverity `mapstructure:"min"`
+
+ // MatchUndefined lets logs records with "unknown" severity match.
+ // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity.
+ MatchUndefined bool `mapstructure:"match_undefined"`
+}
+
+// validate checks that the LogMatchProperties is valid
+func (lmp LogSeverityNumberMatchProperties) validate() error {
+ return lmp.Min.validate()
 }
 
 var _ config.Processor = (*Config)(nil)

processor/filterprocessor/config_test.go

@@ -445,12 +445,16 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) {
 // TestLoadingConfigMinSeverityNumberLogs tests loading testdata/config_logs_min_severity.yaml
 func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) {
  testDataLogPropertiesInclude := &LogMatchProperties{
- MinSeverity: logSeverity("INFO"),
- MatchUndefinedSeverity: true,
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("INFO"),
+ MatchUndefined: true,
+ },
  }
 
  testDataLogPropertiesExclude := &LogMatchProperties{
- MinSeverity: logSeverity("ERROR"),
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("ERROR"),
+ },
  }
 
  factories, err := componenttest.NopFactories()

processor/filterprocessor/filter_processor_logs_test.go

@@ -492,7 +492,9 @@ var (
  name: "includeMinSeverityINFO",
  inc: &LogMatchProperties{
  LogMatchType: Regexp,
- MinSeverity: logSeverity("INFO"),
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("INFO"),
+ },
  },
  inLogs: testResourceLogs(inLogForSeverityNumber),
  outLN: [][]string{
@@ -504,7 +506,9 @@ var (
  name: "includeMinSeverityDEBUG",
  inc: &LogMatchProperties{
  LogMatchType: Regexp,
- MinSeverity: logSeverity("DEBUG"),
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("DEBUG"),
+ },
  },
  inLogs: testResourceLogs(inLogForSeverityNumber),
  outLN: [][]string{
@@ -516,9 +520,11 @@ var (
  {
  name: "includeMinSeverityFATAL+undefined",
  inc: &LogMatchProperties{
- LogMatchType: Regexp,
- MinSeverity: logSeverity("FATAL"),
- MatchUndefinedSeverity: true,
+ LogMatchType: Regexp,
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("FATAL"),
+ MatchUndefined: true,
+ },
  },
  inLogs: testResourceLogs(inLogForSeverityNumber),
  outLN: [][]string{
@@ -529,7 +535,9 @@ var (
  name: "excludeMinSeverityINFO",
  exc: &LogMatchProperties{
  LogMatchType: Regexp,
- MinSeverity: logSeverity("INFO"),
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("INFO"),
+ },
  },
  inLogs: testResourceLogs(inLogForSeverityNumber),
  outLN: [][]string{
@@ -541,7 +549,9 @@ var (
  name: "excludeMinSeverityTRACE",
  exc: &LogMatchProperties{
  LogMatchType: Regexp,
- MinSeverity: logSeverity("TRACE"),
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("TRACE"),
+ },
  },
  inLogs: testResourceLogs(inLogForSeverityNumber),
  outLN: [][]string{
@@ -551,9 +561,11 @@ var (
  {
  name: "excludeMinSeverityINFO+undefined",
  exc: &LogMatchProperties{
- LogMatchType: Regexp,
- MinSeverity: logSeverity("INFO"),
- MatchUndefinedSeverity: true,
+ LogMatchType: Regexp,
+ SeverityNumberProperties: &LogSeverityNumberMatchProperties{
+ Min: logSeverity("INFO"),
+ MatchUndefined: true,
+ },
  },
  inLogs: testResourceLogs(inLogForSeverityNumber),
  outLN: [][]string{

processor/filterprocessor/testdata/config_logs_min_severity.yaml

@@ -8,26 +8,30 @@ processors:
  # This filters out all logs below "INFO" level (the whole DEBUG and TRACE ranges)
  # Logs with no defined severity will also be matched.
  include:
- min_severity: "INFO"
- match_undefined_severity: true
+ severity_number:
+ min: "INFO"
+ match_undefined: true
  filter/exclude:
  logs:
  # any logs matching filters are excluded from remainder of pipeline
  # This will filter out the "ERROR" and "FATAL" ranges
  exclude:
- min_severity: "ERROR"
+ severity_number:
+ min: "ERROR"
 
  filter/includeexclude:
  logs:
  # if both include and exclude are specified, include filters are applied first
  # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass,
  # as well as logs with undefined severity
  include:
- min_severity: "INFO"
- match_undefined_severity: true
+ severity_number:
+ min: "INFO"
+ match_undefined: true
 
  exclude:
- min_severity: "ERROR"
+ severity_number:
+ min: "ERROR"
 
 exporters:
  nop: