From c402a2c574309b6aa500e7fa8ab747d863529c46 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Tue, 9 Aug 2022 11:49:48 -0400 Subject: [PATCH] Move config for log severity_number matching to new object --- .../processor/filterconfig/config.go | 30 ++++++---- .../processor/filterlog/filterlog.go | 4 +- .../processor/filterlog/filterlog_test.go | 16 +++-- processor/filterprocessor/README.md | 21 ++++--- processor/filterprocessor/config.go | 59 +++++++++++++------ processor/filterprocessor/config_test.go | 10 +++- .../filter_processor_logs_test.go | 32 ++++++---- .../testdata/config_logs_min_severity.yaml | 16 +++-- 8 files changed, 122 insertions(+), 66 deletions(-) diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index ae141cbcd509a..6f8038553371c 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -102,15 +102,8 @@ type MatchProperties struct { // against. LogSeverityTexts []string `mapstructure:"log_severity_texts"` - // LogMinSeverity is the lowest severity that may be matched. - // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match. - LogMinSeverity plog.SeverityNumber `mapstructure:"log_min_severity"` - - // LogMatchUndefinedSeverity controls whether logs with "undefined" severity matches. - // If this is true, entries with undefined severity will match. - // This field is only applicable if LogMinSeverity is specified; - // If LogMinSeverity is not specified, this field does nothing. - LogMatchUndefinedSeverity bool `mapstructure:"log_match_undefined_severity"` + // LogSeverityNumber defines how to match against a log record's SeverityNumber, if defined. + LogSeverityNumber *LogSeverityNumberMatchProperties `mapstructure:"log_severity_number"` // MetricNames is a list of strings to match metric name against. // A match occurs if metric name matches at least one item in the list. @@ -144,6 +137,10 @@ func (mp *MatchProperties) ValidateForSpans() error { return errors.New("log_severity_texts should not be specified for trace spans") } + if mp.LogSeverityNumber != nil { + return errors.New("log_severity_number should not be specified for trace spans") + } + if len(mp.Services) == 0 && len(mp.SpanNames) == 0 && len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 { return errors.New(`at least one of "services", "span_names", "attributes", "libraries" or "resources" field must be specified`) @@ -160,8 +157,8 @@ func (mp *MatchProperties) ValidateForLogs() error { if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && - len(mp.LogSeverityTexts) == 0 && mp.LogMinSeverity == plog.SeverityNumberUNDEFINED { - return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`) + len(mp.LogSeverityTexts) == 0 && mp.LogSeverityNumber == nil { + return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`) } return nil @@ -190,3 +187,14 @@ type InstrumentationLibrary struct { // 1 1 yes Version *string `mapstructure:"version"` } + +// LogSeverityNumberMatchProperties defines how to match based on a log record's SeverityNumber field. +type LogSeverityNumberMatchProperties struct { + // Min is the lowest severity that may be matched. + // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match. + Min plog.SeverityNumber `mapstructure:"min"` + + // MatchUndefined controls whether logs with "undefined" severity matches. + // If this is true, entries with undefined severity will match. + MatchUndefined bool `mapstructure:"match_undefined"` +} diff --git a/internal/coreinternal/processor/filterlog/filterlog.go b/internal/coreinternal/processor/filterlog/filterlog.go index cd2158fcba7b6..0e673c7dd83bc 100644 --- a/internal/coreinternal/processor/filterlog/filterlog.go +++ b/internal/coreinternal/processor/filterlog/filterlog.go @@ -79,8 +79,8 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) { } var severityNumberMatcher Matcher - if mp.LogMinSeverity != plog.SeverityNumberUNDEFINED { - severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity, mp.LogMatchUndefinedSeverity) + if mp.LogSeverityNumber != nil { + severityNumberMatcher = newSeverityNumberMatcher(mp.LogSeverityNumber.Min, mp.LogSeverityNumber.MatchUndefined) } return &propertiesMatcher{ diff --git a/internal/coreinternal/processor/filterlog/filterlog_test.go b/internal/coreinternal/processor/filterlog/filterlog_test.go index 69dcf01ac1bc5..7d41990fb167a 100644 --- a/internal/coreinternal/processor/filterlog/filterlog_test.go +++ b/internal/coreinternal/processor/filterlog/filterlog_test.go @@ -41,7 +41,7 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { { name: "empty_property", property: filterconfig.MatchProperties{}, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`, }, { name: "empty_log_bodies_and_attributes", @@ -49,7 +49,7 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { LogBodies: []string{}, LogSeverityTexts: []string{}, }, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`, }, { name: "span_properties", @@ -126,8 +126,10 @@ func TestLogRecord_Matching_False(t *testing.T) { { name: "log_min_severity_trace_dont_match", properties: &filterconfig.MatchProperties{ - Config: *createConfig(filterset.Regexp), - LogMinSeverity: plog.SeverityNumberINFO, + Config: *createConfig(filterset.Regexp), + LogSeverityNumber: &filterconfig.LogSeverityNumberMatchProperties{ + Min: plog.SeverityNumberINFO, + }, }, }, } @@ -184,8 +186,10 @@ func TestLogRecord_Matching_True(t *testing.T) { { name: "log_min_severity_match", properties: &filterconfig.MatchProperties{ - Config: *createConfig(filterset.Regexp), - LogMinSeverity: plog.SeverityNumberDEBUG, + Config: *createConfig(filterset.Regexp), + LogSeverityNumber: &filterconfig.LogSeverityNumberMatchProperties{ + Min: plog.SeverityNumberDEBUG, + }, }, }, } diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 10de464269635..478ed725bba05 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -35,11 +35,13 @@ For logs: A match occurs if the record matches any expression in this given list. - `bodies`: Bodies defines a list of possible log bodies to match the logs against. A match occurs if the record matches any expression in this given list. -- `min_severity`: MinSeverity defines the minimum severity with which a log record should match. - e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. - The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity) - By default, logs with undefined severity are not matched. -- `match_undefined_severity`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. If `min_severity` is not specified, this option does nothing. If `match_undefined_severity` is set to true, log records with no severity will be matched. If set to false, log records with no severity will not be matched. +- `severity_number`: SeverityNumber defines how to match a record based on its SeverityNumber. + The following can be configured for matching a log record's SeverityNumber: + - `min`: Min defines the minimum severity with which a log record should match. + e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. + The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity). You may use either the numerical "SeverityNumber" or the "Short Name" + - `match_undefined`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. + By default, this is `false`. For metrics: @@ -104,12 +106,13 @@ processors: - INFO[2-4]? - WARN[2-4]? - ERROR[2-4]? - # Filter out logs below INFO (no DEBUG or TRACE level logs) - # log records + # Filter out logs below INFO (no DEBUG or TRACE level logs), + # retaining logs with undefined severity logs/severity_number: include: - min_severity: "INFO" - match_undefined_severity: true + severity_number: + min: "INFO" + match_undefined: true logs/bodies: include: match_type: regexp diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 451c6cdf91da4..b356f84dcb2b9 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -185,16 +185,8 @@ type LogMatchProperties struct { // against. SeverityTexts []string `mapstructure:"severity_texts"` - // MinSeverity is the minimum severity needed for the log record to match. - // This corresponds to the short names specified here: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity - // this field is case-insensitive ("INFO" == "info") - MinSeverity logSeverity `mapstructure:"min_severity"` - - // MatchUndefinedSeverity lets logs records with "unknown" severity match. - // This is only applied if MinSeverity is set. - // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity. - MatchUndefinedSeverity bool `mapstructure:"match_undefined_severity"` + // SeverityNumberProperties defines how to match against a log record's SeverityNumber, if defined. + SeverityNumberProperties *LogSeverityNumberMatchProperties `mapstructure:"severity_number"` // LogBodies is a list of strings that the LogRecord's body field must match // against. @@ -203,29 +195,58 @@ type LogMatchProperties struct { // validate checks that the LogMatchProperties is valid func (lmp LogMatchProperties) validate() error { - return lmp.MinSeverity.validate() + if lmp.SeverityNumberProperties != nil { + return lmp.SeverityNumberProperties.validate() + } + return nil } // isEmpty returns true if the properties is "empty" (meaning, there are no filters specified) // if this is the case, the filter should be ignored. func (lmp LogMatchProperties) isEmpty() bool { return len(lmp.ResourceAttributes) == 0 && len(lmp.RecordAttributes) == 0 && - len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 && lmp.MinSeverity == "" + len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 && + lmp.SeverityNumberProperties == nil } // matchProperties converts the LogMatchProperties to a corresponding filterconfig.MatchProperties func (lmp LogMatchProperties) matchProperties() *filterconfig.MatchProperties { - return &filterconfig.MatchProperties{ + mp := &filterconfig.MatchProperties{ Config: filterset.Config{ MatchType: filterset.MatchType(lmp.LogMatchType), }, - Resources: lmp.ResourceAttributes, - Attributes: lmp.RecordAttributes, - LogSeverityTexts: lmp.SeverityTexts, - LogBodies: lmp.LogBodies, - LogMinSeverity: lmp.MinSeverity.severityNumber(), - LogMatchUndefinedSeverity: lmp.MatchUndefinedSeverity, + Resources: lmp.ResourceAttributes, + Attributes: lmp.RecordAttributes, + LogSeverityTexts: lmp.SeverityTexts, + LogBodies: lmp.LogBodies, } + + // Include SeverityNumberProperties if defined + if lmp.SeverityNumberProperties != nil { + mp.LogSeverityNumber = &filterconfig.LogSeverityNumberMatchProperties{ + Min: lmp.SeverityNumberProperties.Min.severityNumber(), + MatchUndefined: lmp.SeverityNumberProperties.MatchUndefined, + } + } + + return mp +} + +type LogSeverityNumberMatchProperties struct { + // Min is the minimum severity needed for the log record to match. + // This corresponds to the short names specified here: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity + // this field is case-insensitive ("INFO" == "info") + Min logSeverity `mapstructure:"min"` + + // MatchUndefined lets logs records with "unknown" severity match. + // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity. + MatchUndefined bool `mapstructure:"match_undefined"` +} + +// validate checks that the LogMatchProperties is valid +func (lmp LogSeverityNumberMatchProperties) validate() error { + return lmp.Min.validate() } var _ config.Processor = (*Config)(nil) diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index 382fc2938b9a8..61747319bb44f 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -445,12 +445,16 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) { // TestLoadingConfigMinSeverityNumberLogs tests loading testdata/config_logs_min_severity.yaml func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) { testDataLogPropertiesInclude := &LogMatchProperties{ - MinSeverity: logSeverity("INFO"), - MatchUndefinedSeverity: true, + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + MatchUndefined: true, + }, } testDataLogPropertiesExclude := &LogMatchProperties{ - MinSeverity: logSeverity("ERROR"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("ERROR"), + }, } factories, err := componenttest.NopFactories() diff --git a/processor/filterprocessor/filter_processor_logs_test.go b/processor/filterprocessor/filter_processor_logs_test.go index 8baa1322043b2..2a785f1313fad 100644 --- a/processor/filterprocessor/filter_processor_logs_test.go +++ b/processor/filterprocessor/filter_processor_logs_test.go @@ -492,7 +492,9 @@ var ( name: "includeMinSeverityINFO", inc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("INFO"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -504,7 +506,9 @@ var ( name: "includeMinSeverityDEBUG", inc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("DEBUG"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("DEBUG"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -516,9 +520,11 @@ var ( { name: "includeMinSeverityFATAL+undefined", inc: &LogMatchProperties{ - LogMatchType: Regexp, - MinSeverity: logSeverity("FATAL"), - MatchUndefinedSeverity: true, + LogMatchType: Regexp, + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("FATAL"), + MatchUndefined: true, + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -529,7 +535,9 @@ var ( name: "excludeMinSeverityINFO", exc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("INFO"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -541,7 +549,9 @@ var ( name: "excludeMinSeverityTRACE", exc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("TRACE"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("TRACE"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -551,9 +561,11 @@ var ( { name: "excludeMinSeverityINFO+undefined", exc: &LogMatchProperties{ - LogMatchType: Regexp, - MinSeverity: logSeverity("INFO"), - MatchUndefinedSeverity: true, + LogMatchType: Regexp, + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + MatchUndefined: true, + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ diff --git a/processor/filterprocessor/testdata/config_logs_min_severity.yaml b/processor/filterprocessor/testdata/config_logs_min_severity.yaml index ed987977de28c..f7ae58e36e001 100644 --- a/processor/filterprocessor/testdata/config_logs_min_severity.yaml +++ b/processor/filterprocessor/testdata/config_logs_min_severity.yaml @@ -8,14 +8,16 @@ processors: # This filters out all logs below "INFO" level (the whole DEBUG and TRACE ranges) # Logs with no defined severity will also be matched. include: - min_severity: "INFO" - match_undefined_severity: true + severity_number: + min: "INFO" + match_undefined: true filter/exclude: logs: # any logs matching filters are excluded from remainder of pipeline # This will filter out the "ERROR" and "FATAL" ranges exclude: - min_severity: "ERROR" + severity_number: + min: "ERROR" filter/includeexclude: logs: @@ -23,11 +25,13 @@ processors: # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass, # as well as logs with undefined severity include: - min_severity: "INFO" - match_undefined_severity: true + severity_number: + min: "INFO" + match_undefined: true exclude: - min_severity: "ERROR" + severity_number: + min: "ERROR" exporters: nop: