From e5359979a41187a3c0f0431419b62d2e9a858382 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Fri, 5 Aug 2022 17:00:28 -0400 Subject: [PATCH 1/9] add severity number filtering --- .../processor/filterconfig/config.go | 11 +- .../processor/filterlog/filterlog.go | 18 +- .../processor/filterlog/filterlog_test.go | 21 ++- .../processor/filterlog/severity_matcher.go | 32 ++++ .../filterlog/severity_matcher_test.go | 74 ++++++++ processor/filterprocessor/README.md | 13 +- processor/filterprocessor/config.go | 85 ++++++++- processor/filterprocessor/config_test.go | 170 ++++++++++++++++++ .../filter_processor_logs_test.go | 81 ++++++++- .../testdata/config_logs_min_severity.yaml | 36 ++++ .../filterprocessor-sev-num-filtering.yaml | 11 ++ 11 files changed, 535 insertions(+), 17 deletions(-) create mode 100644 internal/coreinternal/processor/filterlog/severity_matcher.go create mode 100644 internal/coreinternal/processor/filterlog/severity_matcher_test.go create mode 100644 processor/filterprocessor/testdata/config_logs_min_severity.yaml create mode 100755 unreleased/filterprocessor-sev-num-filtering.yaml diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index 9702e3f918083..55eccf72d2dbb 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -18,6 +18,7 @@ import ( "errors" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filterset" + "go.opentelemetry.io/collector/pdata/plog" ) // MatchConfig has two optional MatchProperties one to define what is processed @@ -100,6 +101,10 @@ type MatchProperties struct { // against. LogSeverityTexts []string `mapstructure:"log_severity_texts"` + // LogMinSeverity is the lowest severity that may be matched. + // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match. + LogMinSeverity plog.SeverityNumber `mapstructure:"log_min_severity"` + // MetricNames is a list of strings to match metric name against. // A match occurs if metric name matches at least one item in the list. // This field is optional. @@ -146,8 +151,10 @@ func (mp *MatchProperties) ValidateForLogs() error { return errors.New("neither services nor span_names should be specified for log records") } - if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && len(mp.LogSeverityTexts) == 0 { - return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severity_texts" field must be specified`) + if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && + len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && + len(mp.LogSeverityTexts) == 0 && mp.LogMinSeverity == plog.SeverityNumberUNDEFINED { + return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`) } return nil diff --git a/internal/coreinternal/processor/filterlog/filterlog.go b/internal/coreinternal/processor/filterlog/filterlog.go index 71a4e34068f3c..60519c1197b03 100644 --- a/internal/coreinternal/processor/filterlog/filterlog.go +++ b/internal/coreinternal/processor/filterlog/filterlog.go @@ -43,6 +43,9 @@ type propertiesMatcher struct { // log severity texts to compare to severityTextFilters filterset.FilterSet + + // matcher for severity number + severityNumberMatcher Matcher } // NewMatcher creates a LogRecord Matcher that matches based on the given MatchProperties. @@ -75,10 +78,16 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) { } } + var severityNumberMatcher Matcher + if mp.LogMinSeverity != plog.SeverityNumberUNDEFINED { + severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity) + } + return &propertiesMatcher{ - PropertiesMatcher: rm, - bodyFilters: bodyFS, - severityTextFilters: severitytextFS, + PropertiesMatcher: rm, + bodyFilters: bodyFS, + severityTextFilters: severitytextFS, + severityNumberMatcher: severityNumberMatcher, }, nil } @@ -97,6 +106,9 @@ func (mp *propertiesMatcher) MatchLogRecord(lr plog.LogRecord, resource pcommon. if mp.severityTextFilters != nil && !mp.severityTextFilters.Matches(lr.SeverityText()) { return false } + if mp.severityNumberMatcher != nil && !mp.severityNumberMatcher.MatchLogRecord(lr, resource, library) { + return false + } return mp.PropertiesMatcher.Match(lr.Attributes(), resource, library) } diff --git a/internal/coreinternal/processor/filterlog/filterlog_test.go b/internal/coreinternal/processor/filterlog/filterlog_test.go index 63614899ee71f..69dcf01ac1bc5 100644 --- a/internal/coreinternal/processor/filterlog/filterlog_test.go +++ b/internal/coreinternal/processor/filterlog/filterlog_test.go @@ -41,7 +41,7 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { { name: "empty_property", property: filterconfig.MatchProperties{}, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severity_texts" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`, }, { name: "empty_log_bodies_and_attributes", @@ -49,7 +49,7 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { LogBodies: []string{}, LogSeverityTexts: []string{}, }, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severity_texts" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`, }, { name: "span_properties", @@ -123,9 +123,18 @@ func TestLogRecord_Matching_False(t *testing.T) { LogSeverityTexts: []string{"debug.*"}, }, }, + { + name: "log_min_severity_trace_dont_match", + properties: &filterconfig.MatchProperties{ + Config: *createConfig(filterset.Regexp), + LogMinSeverity: plog.SeverityNumberINFO, + }, + }, } lr := plog.NewLogRecord() + lr.SetSeverityNumber(plog.SeverityNumberTRACE) + for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { matcher, err := NewMatcher(tc.properties) @@ -172,12 +181,20 @@ func TestLogRecord_Matching_True(t *testing.T) { LogSeverityTexts: []string{"debug.*"}, }, }, + { + name: "log_min_severity_match", + properties: &filterconfig.MatchProperties{ + Config: *createConfig(filterset.Regexp), + LogMinSeverity: plog.SeverityNumberDEBUG, + }, + }, } lr := plog.NewLogRecord() lr.Attributes().InsertString("abc", "def") lr.Body().SetStringVal("AUTHENTICATION FAILED") lr.SetSeverityText("debug") + lr.SetSeverityNumber(plog.SeverityNumberDEBUG) for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/coreinternal/processor/filterlog/severity_matcher.go b/internal/coreinternal/processor/filterlog/severity_matcher.go new file mode 100644 index 0000000000000..5f9bab8d16677 --- /dev/null +++ b/internal/coreinternal/processor/filterlog/severity_matcher.go @@ -0,0 +1,32 @@ +package filterlog + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" +) + +// severtiyNumberMatcher is a Matcher that matches if the input log record has a severity higher than +// the minSeverityNumber. +type severityNumberMatcher struct { + minSeverityNumber plog.SeverityNumber +} + +func newSeverityNumberMatcher(sn plog.SeverityNumber) severityNumberMatcher { + return severityNumberMatcher{ + minSeverityNumber: sn, + } +} + +func (snm severityNumberMatcher) MatchLogRecord(lr plog.LogRecord, _ pcommon.Resource, _ pcommon.InstrumentationScope) bool { + // We explicitly do not match UNDEFINED severity. + if lr.SeverityNumber() == plog.SeverityNumberUNDEFINED { + return false + } + + // If the log records severity is greater than or equal to the desired severity, it matches + if lr.SeverityNumber() >= snm.minSeverityNumber { + return true + } + + return false +} diff --git a/internal/coreinternal/processor/filterlog/severity_matcher_test.go b/internal/coreinternal/processor/filterlog/severity_matcher_test.go new file mode 100644 index 0000000000000..8c3b196222bbf --- /dev/null +++ b/internal/coreinternal/processor/filterlog/severity_matcher_test.go @@ -0,0 +1,74 @@ +package filterlog + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" +) + +func TestSeverityMatcher_MatchLogRecord(t *testing.T) { + testCases := []struct { + name string + minSeverity plog.SeverityNumber + inputSeverity plog.SeverityNumber + matches bool + }{ + { + name: "INFO matches if TRACE is min", + minSeverity: plog.SeverityNumberTRACE, + inputSeverity: plog.SeverityNumberINFO, + matches: true, + }, + { + name: "INFO matches if INFO is min", + minSeverity: plog.SeverityNumberINFO, + inputSeverity: plog.SeverityNumberINFO, + matches: true, + }, + { + name: "INFO does not match if WARN is min", + minSeverity: plog.SeverityNumberWARN, + inputSeverity: plog.SeverityNumberINFO, + matches: false, + }, + { + name: "INFO does not match if INFO2 is min", + minSeverity: plog.SeverityNumberINFO2, + inputSeverity: plog.SeverityNumberINFO, + matches: false, + }, + { + name: "INFO2 matches if INFO is min", + minSeverity: plog.SeverityNumberINFO, + inputSeverity: plog.SeverityNumberINFO2, + matches: true, + }, + { + name: "UNDEFINED does not match if TRACE is min", + minSeverity: plog.SeverityNumberTRACE, + inputSeverity: plog.SeverityNumberUNDEFINED, + matches: false, + }, + { + name: "UNDEFINED does not match if UNDEFINED is min", + minSeverity: plog.SeverityNumberUNDEFINED, + inputSeverity: plog.SeverityNumberUNDEFINED, + matches: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + matcher := newSeverityNumberMatcher(tc.minSeverity) + + r := pcommon.NewResource() + i := pcommon.NewInstrumentationScope() + lr := plog.NewLogRecord() + lr.SetSeverityNumber(tc.inputSeverity) + + require.Equal(t, tc.matches, matcher.MatchLogRecord(lr, r, i)) + }) + } +} diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 9b1a79722825e..2f7d31e2909ba 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -35,6 +35,10 @@ For logs: A match occurs if the record matches any expression in this given list. - `bodies`: Bodies defines a list of possible log bodies to match the logs against. A match occurs if the record matches any expression in this given list. +- `min_severity`: MinSeverity defines the minimum severity with which a log record should match. + e.g. if this is "INFO", all log records with "INFO" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. + If this option is specified, no logs with "DEFAULT" severity will be matched. + The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity) For metrics: @@ -91,18 +95,23 @@ processors: record_attributes: - Key: record_attr Value: prefix_.* - logs/severity: + # Filter on severity text field + logs/severity_text: include: match_type: regexp severity_texts: - INFO[2-4]? - WARN[2-4]? - ERROR[2-4]? + # Filter out logs below INFO (no DEBUG or TRACE level logs) + logs/severity_number: + include: + min_severity: "INFO" logs/bodies: include: match_type: regexp bodies: - - ^IMPORTANT RECORD + - ^IMPORTANT RECORD ``` Refer to the config files in [testdata](./testdata) for detailed diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 9df77adea3edb..2266f5101fdaf 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -15,7 +15,13 @@ package filterprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" import ( + "errors" + "fmt" + "strings" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/multierr" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filterconfig" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filtermetric" @@ -85,6 +91,58 @@ const ( Regexp = LogMatchType(filterset.Regexp) ) +var severityToNumber = map[string]plog.SeverityNumber{ + "TRACE": plog.SeverityNumberTRACE, + "TRACE2": plog.SeverityNumberTRACE2, + "TRACE3": plog.SeverityNumberTRACE3, + "TRACE4": plog.SeverityNumberTRACE4, + "DEBUG": plog.SeverityNumberDEBUG, + "DEBUG2": plog.SeverityNumberDEBUG2, + "DEBUG3": plog.SeverityNumberDEBUG3, + "DEBUG4": plog.SeverityNumberDEBUG4, + "INFO": plog.SeverityNumberINFO, + "INFO2": plog.SeverityNumberINFO2, + "INFO3": plog.SeverityNumberINFO3, + "INFO4": plog.SeverityNumberINFO4, + "WARN": plog.SeverityNumberWARN, + "WARN2": plog.SeverityNumberWARN2, + "WARN3": plog.SeverityNumberWARN3, + "WARN4": plog.SeverityNumberWARN4, + "ERROR": plog.SeverityNumberERROR, + "ERROR2": plog.SeverityNumberERROR2, + "ERROR3": plog.SeverityNumberERROR3, + "ERROR4": plog.SeverityNumberERROR4, + "FATAL": plog.SeverityNumberFATAL, + "FATAL2": plog.SeverityNumberFATAL2, + "FATAL3": plog.SeverityNumberFATAL3, + "FATAL4": plog.SeverityNumberFATAL4, +} + +var errInvalidSeverity = errors.New("not a valid severity") + +// logSeverity is a type that represents a SeverityNumber as a string +type logSeverity string + +// validate checks that the logSeverity is valid +func (l logSeverity) validate() error { + if l == "" { + // No severity specified, which means to ignore this field. + return nil + } + + capsSeverity := strings.ToUpper(string(l)) + if _, ok := severityToNumber[capsSeverity]; !ok { + return fmt.Errorf("'%s' is not a valid severity: %w", string(l), errInvalidSeverity) + } + return nil +} + +// severityNumber returns the severity number that the logSeverity represents +func (l logSeverity) severityNumber() plog.SeverityNumber { + capsSeverity := strings.ToUpper(string(l)) + return severityToNumber[capsSeverity] +} + // LogMatchProperties specifies the set of properties in a log to match against and the // type of string pattern matching to use. type LogMatchProperties struct { @@ -103,15 +161,27 @@ type LogMatchProperties struct { // against. SeverityTexts []string `mapstructure:"severity_texts"` + // MinSeverity is the minimum severity needed for the log record to match. + // This corresponds to the short names specified here: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity + // this field is case-insensitive ("INFO" == "info") + MinSeverity logSeverity `mapstructure:"min_severity"` + // LogBodies is a list of strings that the LogRecord's body field must match // against. LogBodies []string `mapstructure:"bodies"` } +// validate checks that the LogMatchProperties is valid +func (l LogMatchProperties) validate() error { + return l.MinSeverity.validate() +} + // isEmpty returns true if the properties is "empty" (meaning, there are no filters specified) // if this is the case, the filter should be ignored. func (lmp LogMatchProperties) isEmpty() bool { - return len(lmp.ResourceAttributes) == 0 && len(lmp.RecordAttributes) == 0 && len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 + return len(lmp.ResourceAttributes) == 0 && len(lmp.RecordAttributes) == 0 && + len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 && lmp.MinSeverity == "" } // matchProperties converts the LogMatchProperties to a corresponding filterconfig.MatchProperties @@ -124,6 +194,7 @@ func (lmp LogMatchProperties) matchProperties() *filterconfig.MatchProperties { Attributes: lmp.RecordAttributes, LogSeverityTexts: lmp.SeverityTexts, LogBodies: lmp.LogBodies, + LogMinSeverity: lmp.MinSeverity.severityNumber(), } } @@ -131,5 +202,15 @@ var _ config.Processor = (*Config)(nil) // Validate checks if the processor configuration is valid func (cfg *Config) Validate() error { - return nil + var err error + + if cfg.Logs.Include != nil { + err = multierr.Append(err, cfg.Logs.Include.validate()) + } + + if cfg.Logs.Exclude != nil { + err = multierr.Append(err, cfg.Logs.Exclude.validate()) + } + + return err } diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index bf7077ef498ce..a9dd3e433258e 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/service/servicetest" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filterconfig" @@ -441,6 +442,67 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) { } } +// TestLoadingConfigMinSeverityNumberLogs tests loading testdata/config_logs_min_severity.yaml +func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) { + testDataLogPropertiesInclude := &LogMatchProperties{ + MinSeverity: logSeverity("INFO"), + } + + testDataLogPropertiesExclude := &LogMatchProperties{ + MinSeverity: logSeverity("ERROR"), + } + + factories, err := componenttest.NopFactories() + assert.Nil(t, err) + + factory := NewFactory() + factories.Processors[typeStr] = factory + cfg, err := servicetest.LoadConfigAndValidate(filepath.Join("testdata", "config_logs_min_severity.yaml"), factories) + + assert.Nil(t, err) + require.NotNil(t, cfg) + + tests := []struct { + filterID config.ComponentID + expCfg *Config + }{ + { + filterID: config.NewComponentIDWithName("filter", "include"), + expCfg: &Config{ + ProcessorSettings: config.NewProcessorSettings(config.NewComponentIDWithName(typeStr, "include")), + Logs: LogFilters{ + Include: testDataLogPropertiesInclude, + }, + }, + }, { + filterID: config.NewComponentIDWithName("filter", "exclude"), + expCfg: &Config{ + ProcessorSettings: config.NewProcessorSettings(config.NewComponentIDWithName(typeStr, "exclude")), + Logs: LogFilters{ + Exclude: testDataLogPropertiesExclude, + }, + }, + }, { + filterID: config.NewComponentIDWithName("filter", "includeexclude"), + expCfg: &Config{ + ProcessorSettings: config.NewProcessorSettings(config.NewComponentIDWithName(typeStr, "includeexclude")), + Logs: LogFilters{ + Include: testDataLogPropertiesInclude, + Exclude: testDataLogPropertiesExclude, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.filterID.String(), func(t *testing.T) { + cfg := cfg.Processors[test.filterID] + assert.Equal(t, test.expCfg, cfg) + assert.NoError(t, test.expCfg.Validate(), "Failed to validate config") + }) + } +} + // TestLoadingConfigRegexp tests loading testdata/config_regexp.yaml func TestLoadingConfigRegexp(t *testing.T) { // list of filters used repeatedly on testdata/config.yaml @@ -648,3 +710,111 @@ func TestLoadingConfigExpr(t *testing.T) { }) } } + +func TestLogSeverity_severityNumber(t *testing.T) { + testCases := []struct { + name string + sev logSeverity + num plog.SeverityNumber + }{ + { + name: "INFO severity", + sev: logSeverity("INFO"), + num: plog.SeverityNumberINFO, + }, + { + name: "info severity", + sev: logSeverity("info"), + num: plog.SeverityNumberINFO, + }, + { + name: "info3 severity", + sev: logSeverity("info3"), + num: plog.SeverityNumberINFO3, + }, + { + name: "DEBUG severity", + sev: logSeverity("DEBUG"), + num: plog.SeverityNumberDEBUG, + }, + { + name: "ERROR severity", + sev: logSeverity("ERROR"), + num: plog.SeverityNumberERROR, + }, + { + name: "WARN severity", + sev: logSeverity("WARN"), + num: plog.SeverityNumberWARN, + }, + { + name: "unknown severity", + sev: logSeverity("unknown"), + num: plog.SeverityNumberUNDEFINED, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + num := tc.sev.severityNumber() + require.Equal(t, tc.num, num) + }) + } +} + +func TestLogSeverity_severityValidate(t *testing.T) { + testCases := []struct { + name string + sev logSeverity + expectedErr error + }{ + { + name: "INFO severity", + sev: logSeverity("INFO"), + }, + { + name: "info severity", + sev: logSeverity("info"), + }, + { + name: "info3 severity", + sev: logSeverity("info3"), + }, + { + name: "DEBUG severity", + sev: logSeverity("DEBUG"), + }, + { + name: "ERROR severity", + sev: logSeverity("ERROR"), + }, + { + name: "WARN severity", + sev: logSeverity("WARN"), + }, + { + name: "FATAL severity", + sev: logSeverity("FATAL"), + }, + { + name: "unknown severity", + sev: logSeverity("unknown"), + expectedErr: errInvalidSeverity, + }, + { + name: "empty severity is valid", + sev: logSeverity(""), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.sev.validate() + if tc.expectedErr != nil { + require.ErrorIs(t, err, tc.expectedErr) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/processor/filterprocessor/filter_processor_logs_test.go b/processor/filterprocessor/filter_processor_logs_test.go index 396b0bd899fa9..a14a38b100e73 100644 --- a/processor/filterprocessor/filter_processor_logs_test.go +++ b/processor/filterprocessor/filter_processor_logs_test.go @@ -43,6 +43,7 @@ type logWithResource struct { recordAttributes map[string]interface{} severityText string body string + severityNumber plog.SeverityNumber } var ( @@ -154,7 +155,7 @@ var ( }, } - inLogForSeverity = []logWithResource{ + inLogForSeverityText = []logWithResource{ { logNames: []string{"log1"}, severityText: "DEBUG", @@ -192,6 +193,25 @@ var ( }, } + inLogForSeverityNumber = []logWithResource{ + { + logNames: []string{"log1"}, + severityNumber: plog.SeverityNumberDEBUG, + }, + { + logNames: []string{"log2"}, + severityNumber: plog.SeverityNumberINFO, + }, + { + logNames: []string{"log3"}, + severityNumber: plog.SeverityNumberERROR, + }, + { + logNames: []string{"log4"}, + severityNumber: plog.SeverityNumberUNDEFINED, + }, + } + standardLogTests = []logNameTest{ { name: "emptyFilterInclude", @@ -379,7 +399,7 @@ var ( LogMatchType: Strict, SeverityTexts: []string{"INFO", "DEBUG2"}, }, - inLogs: testResourceLogs(inLogForSeverity), + inLogs: testResourceLogs(inLogForSeverityText), outLN: [][]string{ {"log2"}, {"log3"}, @@ -391,7 +411,7 @@ var ( LogMatchType: Regexp, SeverityTexts: []string{"DEBUG[1-4]?"}, }, - inLogs: testResourceLogs(inLogForSeverity), + inLogs: testResourceLogs(inLogForSeverityText), outLN: [][]string{ {"log1"}, {"log2"}, @@ -403,7 +423,7 @@ var ( LogMatchType: Strict, SeverityTexts: []string{"INFO", "DEBUG"}, }, - inLogs: testResourceLogs(inLogForSeverity), + inLogs: testResourceLogs(inLogForSeverityText), outLN: [][]string{ {"log2"}, {"log4"}, @@ -415,7 +435,7 @@ var ( LogMatchType: Regexp, SeverityTexts: []string{"^[DI]"}, }, - inLogs: testResourceLogs(inLogForSeverity), + inLogs: testResourceLogs(inLogForSeverityText), outLN: [][]string{ {"log4"}, }, @@ -468,6 +488,54 @@ var ( {"log4"}, }, }, + { + name: "includeMinSeverityINFO", + inc: &LogMatchProperties{ + LogMatchType: Regexp, + MinSeverity: logSeverity("INFO"), + }, + inLogs: testResourceLogs(inLogForSeverityNumber), + outLN: [][]string{ + {"log2"}, + {"log3"}, + }, + }, + { + name: "includeMinSeverityDEBUG", + inc: &LogMatchProperties{ + LogMatchType: Regexp, + MinSeverity: logSeverity("DEBUG"), + }, + inLogs: testResourceLogs(inLogForSeverityNumber), + outLN: [][]string{ + {"log1"}, + {"log2"}, + {"log3"}, + }, + }, + { + name: "excludeMinSeverityINFO", + exc: &LogMatchProperties{ + LogMatchType: Regexp, + MinSeverity: logSeverity("INFO"), + }, + inLogs: testResourceLogs(inLogForSeverityNumber), + outLN: [][]string{ + {"log1"}, + {"log4"}, + }, + }, + { + name: "excludeMinSeverityTRACE", + exc: &LogMatchProperties{ + LogMatchType: Regexp, + MinSeverity: logSeverity("TRACE"), + }, + inLogs: testResourceLogs(inLogForSeverityNumber), + outLN: [][]string{ + {"log4"}, + }, + }, } ) @@ -534,9 +602,10 @@ func testResourceLogs(lwrs []logWithResource) plog.Logs { // Add record level attributes pcommon.NewMapFromRaw(lwrs[i].recordAttributes).CopyTo(l.Attributes()) l.Attributes().InsertString("name", name) - // Set body & severity text + // Set body & severity fields l.Body().SetStringVal(lwr.body) l.SetSeverityText(lwr.severityText) + l.SetSeverityNumber(lwr.severityNumber) } } return ld diff --git a/processor/filterprocessor/testdata/config_logs_min_severity.yaml b/processor/filterprocessor/testdata/config_logs_min_severity.yaml new file mode 100644 index 0000000000000..b1626ecab7fca --- /dev/null +++ b/processor/filterprocessor/testdata/config_logs_min_severity.yaml @@ -0,0 +1,36 @@ +receivers: + nop: + +processors: + filter/include: + logs: + # any logs NOT matching filters are excluded from remainder of pipeline + # This filters out all logs below "INFO" level (the whole DEBUG and TRACE ranges) + include: + min_severity: "INFO" + filter/exclude: + logs: + # any logs matching filters are excluded from remainder of pipeline + # This will filter out the "ERROR" and "FATAL" ranges + exclude: + min_severity: "ERROR" + + filter/includeexclude: + logs: + # if both include and exclude are specified, include filters are applied first + # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass + include: + min_severity: "INFO" + + exclude: + min_severity: "ERROR" + +exporters: + nop: + +service: + pipelines: + logs: + receivers: [nop] + processors: [filter/include, filter/exclude, filter/includeexclude] + exporters: [nop] diff --git a/unreleased/filterprocessor-sev-num-filtering.yaml b/unreleased/filterprocessor-sev-num-filtering.yaml new file mode 100755 index 0000000000000..e8d7a28fb8bea --- /dev/null +++ b/unreleased/filterprocessor-sev-num-filtering.yaml @@ -0,0 +1,11 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: filterprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add log filtering based on SeverityNumber" + +# One or more tracking issues related to the change +issues: [12959] From ef27a3d684d9c2794fbdc968d37e071c4dea14a7 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 09:56:21 -0400 Subject: [PATCH 2/9] add ability to control matching undefined severity records --- .../processor/filterconfig/config.go | 6 +++++ .../processor/filterlog/filterlog.go | 2 +- .../processor/filterlog/severity_matcher.go | 8 ++++--- .../filterlog/severity_matcher_test.go | 18 ++++++++++---- processor/filterprocessor/README.md | 3 +++ processor/filterprocessor/config.go | 16 +++++++++---- processor/filterprocessor/config_test.go | 3 ++- .../filter_processor_logs_test.go | 24 +++++++++++++++++++ .../testdata/config_logs_min_severity.yaml | 6 ++++- 9 files changed, 70 insertions(+), 16 deletions(-) diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index 55eccf72d2dbb..bc1e0d1a3c688 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -105,6 +105,12 @@ type MatchProperties struct { // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match. LogMinSeverity plog.SeverityNumber `mapstructure:"log_min_severity"` + // LogMatchUndefinedSeverity controls whether logs with "undefined" severity matches. + // If this is true, entries with undefined severity will match. + // This field is only applicable if LogMinSeverity is specified; + // If LogMinSeverity is not specified, this field does nothing. + LogMatchUndefinedSeverity bool `mapstructure:"log_match_undefined_severity"` + // MetricNames is a list of strings to match metric name against. // A match occurs if metric name matches at least one item in the list. // This field is optional. diff --git a/internal/coreinternal/processor/filterlog/filterlog.go b/internal/coreinternal/processor/filterlog/filterlog.go index 60519c1197b03..cd2158fcba7b6 100644 --- a/internal/coreinternal/processor/filterlog/filterlog.go +++ b/internal/coreinternal/processor/filterlog/filterlog.go @@ -80,7 +80,7 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) { var severityNumberMatcher Matcher if mp.LogMinSeverity != plog.SeverityNumberUNDEFINED { - severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity) + severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity, mp.LogMatchUndefinedSeverity) } return &propertiesMatcher{ diff --git a/internal/coreinternal/processor/filterlog/severity_matcher.go b/internal/coreinternal/processor/filterlog/severity_matcher.go index 5f9bab8d16677..122da1822f36f 100644 --- a/internal/coreinternal/processor/filterlog/severity_matcher.go +++ b/internal/coreinternal/processor/filterlog/severity_matcher.go @@ -8,19 +8,21 @@ import ( // severtiyNumberMatcher is a Matcher that matches if the input log record has a severity higher than // the minSeverityNumber. type severityNumberMatcher struct { + matchUndefined bool minSeverityNumber plog.SeverityNumber } -func newSeverityNumberMatcher(sn plog.SeverityNumber) severityNumberMatcher { +func newSeverityNumberMatcher(minSeverity plog.SeverityNumber, matchUndefined bool) severityNumberMatcher { return severityNumberMatcher{ - minSeverityNumber: sn, + minSeverityNumber: minSeverity, + matchUndefined: matchUndefined, } } func (snm severityNumberMatcher) MatchLogRecord(lr plog.LogRecord, _ pcommon.Resource, _ pcommon.InstrumentationScope) bool { // We explicitly do not match UNDEFINED severity. if lr.SeverityNumber() == plog.SeverityNumberUNDEFINED { - return false + return snm.matchUndefined } // If the log records severity is greater than or equal to the desired severity, it matches diff --git a/internal/coreinternal/processor/filterlog/severity_matcher_test.go b/internal/coreinternal/processor/filterlog/severity_matcher_test.go index 8c3b196222bbf..a3d20532943de 100644 --- a/internal/coreinternal/processor/filterlog/severity_matcher_test.go +++ b/internal/coreinternal/processor/filterlog/severity_matcher_test.go @@ -10,10 +10,11 @@ import ( func TestSeverityMatcher_MatchLogRecord(t *testing.T) { testCases := []struct { - name string - minSeverity plog.SeverityNumber - inputSeverity plog.SeverityNumber - matches bool + name string + minSeverity plog.SeverityNumber + matchUndefined bool + inputSeverity plog.SeverityNumber + matches bool }{ { name: "INFO matches if TRACE is min", @@ -57,11 +58,18 @@ func TestSeverityMatcher_MatchLogRecord(t *testing.T) { inputSeverity: plog.SeverityNumberUNDEFINED, matches: false, }, + { + name: "UNDEFINED matches if matchUndefined is true", + minSeverity: plog.SeverityNumberUNDEFINED, + matchUndefined: true, + inputSeverity: plog.SeverityNumberUNDEFINED, + matches: false, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - matcher := newSeverityNumberMatcher(tc.minSeverity) + matcher := newSeverityNumberMatcher(tc.minSeverity, tc.matchUndefined) r := pcommon.NewResource() i := pcommon.NewInstrumentationScope() diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 2f7d31e2909ba..163c1b1a06055 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -39,6 +39,7 @@ For logs: e.g. if this is "INFO", all log records with "INFO" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. If this option is specified, no logs with "DEFAULT" severity will be matched. The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity) +- `match_undefined_severity`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. If `min_severity` is not specified, this option does nothing. If `match_undefined_severity` is set to true, log records with no severity will be matched. If set to false, log records with no severity will not be matched. For metrics: @@ -104,9 +105,11 @@ processors: - WARN[2-4]? - ERROR[2-4]? # Filter out logs below INFO (no DEBUG or TRACE level logs) + # log records logs/severity_number: include: min_severity: "INFO" + match_undefined_severity: true logs/bodies: include: match_type: regexp diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 2266f5101fdaf..18543e2d2cf38 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -167,6 +167,11 @@ type LogMatchProperties struct { // this field is case-insensitive ("INFO" == "info") MinSeverity logSeverity `mapstructure:"min_severity"` + // MatchUndefinedSeverity lets logs records with "unknown" severity match. + // This is only applied if MinSeverity is set. + // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity. + MatchUndefinedSeverity bool `mapstructure:"match_undefined_severity"` + // LogBodies is a list of strings that the LogRecord's body field must match // against. LogBodies []string `mapstructure:"bodies"` @@ -190,11 +195,12 @@ func (lmp LogMatchProperties) matchProperties() *filterconfig.MatchProperties { Config: filterset.Config{ MatchType: filterset.MatchType(lmp.LogMatchType), }, - Resources: lmp.ResourceAttributes, - Attributes: lmp.RecordAttributes, - LogSeverityTexts: lmp.SeverityTexts, - LogBodies: lmp.LogBodies, - LogMinSeverity: lmp.MinSeverity.severityNumber(), + Resources: lmp.ResourceAttributes, + Attributes: lmp.RecordAttributes, + LogSeverityTexts: lmp.SeverityTexts, + LogBodies: lmp.LogBodies, + LogMinSeverity: lmp.MinSeverity.severityNumber(), + LogMatchUndefinedSeverity: lmp.MatchUndefinedSeverity, } } diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index a9dd3e433258e..5c63e5541dc44 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -445,7 +445,8 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) { // TestLoadingConfigMinSeverityNumberLogs tests loading testdata/config_logs_min_severity.yaml func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) { testDataLogPropertiesInclude := &LogMatchProperties{ - MinSeverity: logSeverity("INFO"), + MinSeverity: logSeverity("INFO"), + MatchUndefinedSeverity: true, } testDataLogPropertiesExclude := &LogMatchProperties{ diff --git a/processor/filterprocessor/filter_processor_logs_test.go b/processor/filterprocessor/filter_processor_logs_test.go index a14a38b100e73..8baa1322043b2 100644 --- a/processor/filterprocessor/filter_processor_logs_test.go +++ b/processor/filterprocessor/filter_processor_logs_test.go @@ -513,6 +513,18 @@ var ( {"log3"}, }, }, + { + name: "includeMinSeverityFATAL+undefined", + inc: &LogMatchProperties{ + LogMatchType: Regexp, + MinSeverity: logSeverity("FATAL"), + MatchUndefinedSeverity: true, + }, + inLogs: testResourceLogs(inLogForSeverityNumber), + outLN: [][]string{ + {"log4"}, + }, + }, { name: "excludeMinSeverityINFO", exc: &LogMatchProperties{ @@ -536,6 +548,18 @@ var ( {"log4"}, }, }, + { + name: "excludeMinSeverityINFO+undefined", + exc: &LogMatchProperties{ + LogMatchType: Regexp, + MinSeverity: logSeverity("INFO"), + MatchUndefinedSeverity: true, + }, + inLogs: testResourceLogs(inLogForSeverityNumber), + outLN: [][]string{ + {"log1"}, + }, + }, } ) diff --git a/processor/filterprocessor/testdata/config_logs_min_severity.yaml b/processor/filterprocessor/testdata/config_logs_min_severity.yaml index b1626ecab7fca..ed987977de28c 100644 --- a/processor/filterprocessor/testdata/config_logs_min_severity.yaml +++ b/processor/filterprocessor/testdata/config_logs_min_severity.yaml @@ -6,8 +6,10 @@ processors: logs: # any logs NOT matching filters are excluded from remainder of pipeline # This filters out all logs below "INFO" level (the whole DEBUG and TRACE ranges) + # Logs with no defined severity will also be matched. include: min_severity: "INFO" + match_undefined_severity: true filter/exclude: logs: # any logs matching filters are excluded from remainder of pipeline @@ -18,9 +20,11 @@ processors: filter/includeexclude: logs: # if both include and exclude are specified, include filters are applied first - # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass + # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass, + # as well as logs with undefined severity include: min_severity: "INFO" + match_undefined_severity: true exclude: min_severity: "ERROR" From d3e8dbd12b4bafa1f6d60e3a8b38ac5e04bdf1bd Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 10:29:45 -0400 Subject: [PATCH 3/9] add license, fix test --- .../processor/filterlog/severity_matcher.go | 16 +++++++++++++++- .../processor/filterlog/severity_matcher_test.go | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/internal/coreinternal/processor/filterlog/severity_matcher.go b/internal/coreinternal/processor/filterlog/severity_matcher.go index 122da1822f36f..d62362c93b118 100644 --- a/internal/coreinternal/processor/filterlog/severity_matcher.go +++ b/internal/coreinternal/processor/filterlog/severity_matcher.go @@ -1,3 +1,17 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package filterlog import ( @@ -20,7 +34,7 @@ func newSeverityNumberMatcher(minSeverity plog.SeverityNumber, matchUndefined bo } func (snm severityNumberMatcher) MatchLogRecord(lr plog.LogRecord, _ pcommon.Resource, _ pcommon.InstrumentationScope) bool { - // We explicitly do not match UNDEFINED severity. + // behavior on SeverityNumberUNDEFINED is explicitly defined by matchUndefined if lr.SeverityNumber() == plog.SeverityNumberUNDEFINED { return snm.matchUndefined } diff --git a/internal/coreinternal/processor/filterlog/severity_matcher_test.go b/internal/coreinternal/processor/filterlog/severity_matcher_test.go index a3d20532943de..5dd4cb55ad863 100644 --- a/internal/coreinternal/processor/filterlog/severity_matcher_test.go +++ b/internal/coreinternal/processor/filterlog/severity_matcher_test.go @@ -1,3 +1,17 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package filterlog import ( @@ -63,7 +77,7 @@ func TestSeverityMatcher_MatchLogRecord(t *testing.T) { minSeverity: plog.SeverityNumberUNDEFINED, matchUndefined: true, inputSeverity: plog.SeverityNumberUNDEFINED, - matches: false, + matches: true, }, } From 788a34ccecc449fbd8c10052f1da791ad19a6363 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 12:07:43 -0400 Subject: [PATCH 4/9] lint --- internal/coreinternal/processor/filterconfig/config.go | 3 ++- processor/filterprocessor/config.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index bc1e0d1a3c688..ae141cbcd509a 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -17,8 +17,9 @@ package filterconfig // import "github.com/open-telemetry/opentelemetry-collecto import ( "errors" - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filterset" "go.opentelemetry.io/collector/pdata/plog" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filterset" ) // MatchConfig has two optional MatchProperties one to define what is processed diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 18543e2d2cf38..0ea97650012a4 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -178,8 +178,8 @@ type LogMatchProperties struct { } // validate checks that the LogMatchProperties is valid -func (l LogMatchProperties) validate() error { - return l.MinSeverity.validate() +func (lmp LogMatchProperties) validate() error { + return lmp.MinSeverity.validate() } // isEmpty returns true if the properties is "empty" (meaning, there are no filters specified) From ef11c8f8788629d11868c0cc7a860423474bdb9c Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 12:18:20 -0400 Subject: [PATCH 5/9] fix incorrect explanation in docs --- processor/filterprocessor/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 163c1b1a06055..7b5c946c2ea01 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -36,9 +36,9 @@ For logs: - `bodies`: Bodies defines a list of possible log bodies to match the logs against. A match occurs if the record matches any expression in this given list. - `min_severity`: MinSeverity defines the minimum severity with which a log record should match. - e.g. if this is "INFO", all log records with "INFO" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. - If this option is specified, no logs with "DEFAULT" severity will be matched. + e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity) + By default, logs with undefined severity are not matched. - `match_undefined_severity`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. If `min_severity` is not specified, this option does nothing. If `match_undefined_severity` is set to true, log records with no severity will be matched. If set to false, log records with no severity will not be matched. For metrics: From c049ce91dc23a6f31a59b0c6ba1ff76347686179 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 12:21:47 -0400 Subject: [PATCH 6/9] remove added space in README --- processor/filterprocessor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 7b5c946c2ea01..10de464269635 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -114,7 +114,7 @@ processors: include: match_type: regexp bodies: - - ^IMPORTANT RECORD + - ^IMPORTANT RECORD ``` Refer to the config files in [testdata](./testdata) for detailed From 3573046d91dbd7fe8202f5324f39bb7678b3d9e1 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 12:28:44 -0400 Subject: [PATCH 7/9] porto --- internal/coreinternal/processor/filterlog/severity_matcher.go | 2 +- processor/filterprocessor/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/coreinternal/processor/filterlog/severity_matcher.go b/internal/coreinternal/processor/filterlog/severity_matcher.go index d62362c93b118..bdbe8540f8b85 100644 --- a/internal/coreinternal/processor/filterlog/severity_matcher.go +++ b/internal/coreinternal/processor/filterlog/severity_matcher.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package filterlog +package filterlog // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/processor/filterlog" import ( "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/processor/filterprocessor/go.mod b/processor/filterprocessor/go.mod index 240ba013a73c6..59d6faff9de95 100644 --- a/processor/filterprocessor/go.mod +++ b/processor/filterprocessor/go.mod @@ -7,6 +7,7 @@ require ( github.com/stretchr/testify v1.8.0 go.opentelemetry.io/collector v0.58.0 go.opentelemetry.io/collector/pdata v0.58.0 + go.uber.org/multierr v1.8.0 go.uber.org/zap v1.22.0 ) @@ -31,7 +32,6 @@ require ( go.opentelemetry.io/otel/metric v0.31.0 // indirect go.opentelemetry.io/otel/trace v1.9.0 // indirect go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.8.0 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect From b90692a01e36c00cf6a7c9058f2d1cdf83d79b0e Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 8 Aug 2022 15:02:29 -0400 Subject: [PATCH 8/9] add numeric severities --- processor/filterprocessor/config.go | 24 ++++++++++++++++++++++++ processor/filterprocessor/config_test.go | 5 +++++ 2 files changed, 29 insertions(+) diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 0ea97650012a4..451c6cdf91da4 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -92,6 +92,30 @@ const ( ) var severityToNumber = map[string]plog.SeverityNumber{ + "1": plog.SeverityNumberTRACE, + "2": plog.SeverityNumberTRACE2, + "3": plog.SeverityNumberTRACE3, + "4": plog.SeverityNumberTRACE4, + "5": plog.SeverityNumberDEBUG, + "6": plog.SeverityNumberDEBUG2, + "7": plog.SeverityNumberDEBUG3, + "8": plog.SeverityNumberDEBUG4, + "9": plog.SeverityNumberINFO, + "10": plog.SeverityNumberINFO2, + "11": plog.SeverityNumberINFO3, + "12": plog.SeverityNumberINFO4, + "13": plog.SeverityNumberWARN, + "14": plog.SeverityNumberWARN2, + "15": plog.SeverityNumberWARN3, + "16": plog.SeverityNumberWARN4, + "17": plog.SeverityNumberERROR, + "18": plog.SeverityNumberERROR2, + "19": plog.SeverityNumberERROR3, + "20": plog.SeverityNumberERROR4, + "21": plog.SeverityNumberFATAL, + "22": plog.SeverityNumberFATAL2, + "23": plog.SeverityNumberFATAL3, + "24": plog.SeverityNumberFATAL4, "TRACE": plog.SeverityNumberTRACE, "TRACE2": plog.SeverityNumberTRACE2, "TRACE3": plog.SeverityNumberTRACE3, diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index 5c63e5541dc44..382fc2938b9a8 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -753,6 +753,11 @@ func TestLogSeverity_severityNumber(t *testing.T) { sev: logSeverity("unknown"), num: plog.SeverityNumberUNDEFINED, }, + { + name: "Numeric Severity", + sev: logSeverity("9"), + num: plog.SeverityNumberINFO, + }, } for _, tc := range testCases { From c402a2c574309b6aa500e7fa8ab747d863529c46 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Tue, 9 Aug 2022 11:49:48 -0400 Subject: [PATCH 9/9] Move config for log severity_number matching to new object --- .../processor/filterconfig/config.go | 30 ++++++---- .../processor/filterlog/filterlog.go | 4 +- .../processor/filterlog/filterlog_test.go | 16 +++-- processor/filterprocessor/README.md | 21 ++++--- processor/filterprocessor/config.go | 59 +++++++++++++------ processor/filterprocessor/config_test.go | 10 +++- .../filter_processor_logs_test.go | 32 ++++++---- .../testdata/config_logs_min_severity.yaml | 16 +++-- 8 files changed, 122 insertions(+), 66 deletions(-) diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index ae141cbcd509a..6f8038553371c 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -102,15 +102,8 @@ type MatchProperties struct { // against. LogSeverityTexts []string `mapstructure:"log_severity_texts"` - // LogMinSeverity is the lowest severity that may be matched. - // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match. - LogMinSeverity plog.SeverityNumber `mapstructure:"log_min_severity"` - - // LogMatchUndefinedSeverity controls whether logs with "undefined" severity matches. - // If this is true, entries with undefined severity will match. - // This field is only applicable if LogMinSeverity is specified; - // If LogMinSeverity is not specified, this field does nothing. - LogMatchUndefinedSeverity bool `mapstructure:"log_match_undefined_severity"` + // LogSeverityNumber defines how to match against a log record's SeverityNumber, if defined. + LogSeverityNumber *LogSeverityNumberMatchProperties `mapstructure:"log_severity_number"` // MetricNames is a list of strings to match metric name against. // A match occurs if metric name matches at least one item in the list. @@ -144,6 +137,10 @@ func (mp *MatchProperties) ValidateForSpans() error { return errors.New("log_severity_texts should not be specified for trace spans") } + if mp.LogSeverityNumber != nil { + return errors.New("log_severity_number should not be specified for trace spans") + } + if len(mp.Services) == 0 && len(mp.SpanNames) == 0 && len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 { return errors.New(`at least one of "services", "span_names", "attributes", "libraries" or "resources" field must be specified`) @@ -160,8 +157,8 @@ func (mp *MatchProperties) ValidateForLogs() error { if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && - len(mp.LogSeverityTexts) == 0 && mp.LogMinSeverity == plog.SeverityNumberUNDEFINED { - return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`) + len(mp.LogSeverityTexts) == 0 && mp.LogSeverityNumber == nil { + return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`) } return nil @@ -190,3 +187,14 @@ type InstrumentationLibrary struct { // 1 1 yes Version *string `mapstructure:"version"` } + +// LogSeverityNumberMatchProperties defines how to match based on a log record's SeverityNumber field. +type LogSeverityNumberMatchProperties struct { + // Min is the lowest severity that may be matched. + // e.g. if this is plog.SeverityNumberINFO, INFO, WARN, ERROR, and FATAL logs will match. + Min plog.SeverityNumber `mapstructure:"min"` + + // MatchUndefined controls whether logs with "undefined" severity matches. + // If this is true, entries with undefined severity will match. + MatchUndefined bool `mapstructure:"match_undefined"` +} diff --git a/internal/coreinternal/processor/filterlog/filterlog.go b/internal/coreinternal/processor/filterlog/filterlog.go index cd2158fcba7b6..0e673c7dd83bc 100644 --- a/internal/coreinternal/processor/filterlog/filterlog.go +++ b/internal/coreinternal/processor/filterlog/filterlog.go @@ -79,8 +79,8 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) { } var severityNumberMatcher Matcher - if mp.LogMinSeverity != plog.SeverityNumberUNDEFINED { - severityNumberMatcher = newSeverityNumberMatcher(mp.LogMinSeverity, mp.LogMatchUndefinedSeverity) + if mp.LogSeverityNumber != nil { + severityNumberMatcher = newSeverityNumberMatcher(mp.LogSeverityNumber.Min, mp.LogSeverityNumber.MatchUndefined) } return &propertiesMatcher{ diff --git a/internal/coreinternal/processor/filterlog/filterlog_test.go b/internal/coreinternal/processor/filterlog/filterlog_test.go index 69dcf01ac1bc5..7d41990fb167a 100644 --- a/internal/coreinternal/processor/filterlog/filterlog_test.go +++ b/internal/coreinternal/processor/filterlog/filterlog_test.go @@ -41,7 +41,7 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { { name: "empty_property", property: filterconfig.MatchProperties{}, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`, }, { name: "empty_log_bodies_and_attributes", @@ -49,7 +49,7 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { LogBodies: []string{}, LogSeverityTexts: []string{}, }, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_min_severity" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies", "log_severity_texts" or "log_severity_number" field must be specified`, }, { name: "span_properties", @@ -126,8 +126,10 @@ func TestLogRecord_Matching_False(t *testing.T) { { name: "log_min_severity_trace_dont_match", properties: &filterconfig.MatchProperties{ - Config: *createConfig(filterset.Regexp), - LogMinSeverity: plog.SeverityNumberINFO, + Config: *createConfig(filterset.Regexp), + LogSeverityNumber: &filterconfig.LogSeverityNumberMatchProperties{ + Min: plog.SeverityNumberINFO, + }, }, }, } @@ -184,8 +186,10 @@ func TestLogRecord_Matching_True(t *testing.T) { { name: "log_min_severity_match", properties: &filterconfig.MatchProperties{ - Config: *createConfig(filterset.Regexp), - LogMinSeverity: plog.SeverityNumberDEBUG, + Config: *createConfig(filterset.Regexp), + LogSeverityNumber: &filterconfig.LogSeverityNumberMatchProperties{ + Min: plog.SeverityNumberDEBUG, + }, }, }, } diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 10de464269635..478ed725bba05 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -35,11 +35,13 @@ For logs: A match occurs if the record matches any expression in this given list. - `bodies`: Bodies defines a list of possible log bodies to match the logs against. A match occurs if the record matches any expression in this given list. -- `min_severity`: MinSeverity defines the minimum severity with which a log record should match. - e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. - The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity) - By default, logs with undefined severity are not matched. -- `match_undefined_severity`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. If `min_severity` is not specified, this option does nothing. If `match_undefined_severity` is set to true, log records with no severity will be matched. If set to false, log records with no severity will not be matched. +- `severity_number`: SeverityNumber defines how to match a record based on its SeverityNumber. + The following can be configured for matching a log record's SeverityNumber: + - `min`: Min defines the minimum severity with which a log record should match. + e.g. if this is "WARN", all log records with "WARN" severity and above (WARN[2-4], ERROR[2-4], FATAL[2-4]) are matched. + The list of valid severities that may be used for this option can be found [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity). You may use either the numerical "SeverityNumber" or the "Short Name" + - `match_undefined`: MatchUndefinedSeverity defines whether to match logs with undefined severity or not when using the `min_severity` matching option. + By default, this is `false`. For metrics: @@ -104,12 +106,13 @@ processors: - INFO[2-4]? - WARN[2-4]? - ERROR[2-4]? - # Filter out logs below INFO (no DEBUG or TRACE level logs) - # log records + # Filter out logs below INFO (no DEBUG or TRACE level logs), + # retaining logs with undefined severity logs/severity_number: include: - min_severity: "INFO" - match_undefined_severity: true + severity_number: + min: "INFO" + match_undefined: true logs/bodies: include: match_type: regexp diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 451c6cdf91da4..b356f84dcb2b9 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -185,16 +185,8 @@ type LogMatchProperties struct { // against. SeverityTexts []string `mapstructure:"severity_texts"` - // MinSeverity is the minimum severity needed for the log record to match. - // This corresponds to the short names specified here: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity - // this field is case-insensitive ("INFO" == "info") - MinSeverity logSeverity `mapstructure:"min_severity"` - - // MatchUndefinedSeverity lets logs records with "unknown" severity match. - // This is only applied if MinSeverity is set. - // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity. - MatchUndefinedSeverity bool `mapstructure:"match_undefined_severity"` + // SeverityNumberProperties defines how to match against a log record's SeverityNumber, if defined. + SeverityNumberProperties *LogSeverityNumberMatchProperties `mapstructure:"severity_number"` // LogBodies is a list of strings that the LogRecord's body field must match // against. @@ -203,29 +195,58 @@ type LogMatchProperties struct { // validate checks that the LogMatchProperties is valid func (lmp LogMatchProperties) validate() error { - return lmp.MinSeverity.validate() + if lmp.SeverityNumberProperties != nil { + return lmp.SeverityNumberProperties.validate() + } + return nil } // isEmpty returns true if the properties is "empty" (meaning, there are no filters specified) // if this is the case, the filter should be ignored. func (lmp LogMatchProperties) isEmpty() bool { return len(lmp.ResourceAttributes) == 0 && len(lmp.RecordAttributes) == 0 && - len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 && lmp.MinSeverity == "" + len(lmp.SeverityTexts) == 0 && len(lmp.LogBodies) == 0 && + lmp.SeverityNumberProperties == nil } // matchProperties converts the LogMatchProperties to a corresponding filterconfig.MatchProperties func (lmp LogMatchProperties) matchProperties() *filterconfig.MatchProperties { - return &filterconfig.MatchProperties{ + mp := &filterconfig.MatchProperties{ Config: filterset.Config{ MatchType: filterset.MatchType(lmp.LogMatchType), }, - Resources: lmp.ResourceAttributes, - Attributes: lmp.RecordAttributes, - LogSeverityTexts: lmp.SeverityTexts, - LogBodies: lmp.LogBodies, - LogMinSeverity: lmp.MinSeverity.severityNumber(), - LogMatchUndefinedSeverity: lmp.MatchUndefinedSeverity, + Resources: lmp.ResourceAttributes, + Attributes: lmp.RecordAttributes, + LogSeverityTexts: lmp.SeverityTexts, + LogBodies: lmp.LogBodies, } + + // Include SeverityNumberProperties if defined + if lmp.SeverityNumberProperties != nil { + mp.LogSeverityNumber = &filterconfig.LogSeverityNumberMatchProperties{ + Min: lmp.SeverityNumberProperties.Min.severityNumber(), + MatchUndefined: lmp.SeverityNumberProperties.MatchUndefined, + } + } + + return mp +} + +type LogSeverityNumberMatchProperties struct { + // Min is the minimum severity needed for the log record to match. + // This corresponds to the short names specified here: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity + // this field is case-insensitive ("INFO" == "info") + Min logSeverity `mapstructure:"min"` + + // MatchUndefined lets logs records with "unknown" severity match. + // If MinSeverity is not set, this field is ignored, as fields are not matched based on severity. + MatchUndefined bool `mapstructure:"match_undefined"` +} + +// validate checks that the LogMatchProperties is valid +func (lmp LogSeverityNumberMatchProperties) validate() error { + return lmp.Min.validate() } var _ config.Processor = (*Config)(nil) diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index 382fc2938b9a8..61747319bb44f 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -445,12 +445,16 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) { // TestLoadingConfigMinSeverityNumberLogs tests loading testdata/config_logs_min_severity.yaml func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) { testDataLogPropertiesInclude := &LogMatchProperties{ - MinSeverity: logSeverity("INFO"), - MatchUndefinedSeverity: true, + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + MatchUndefined: true, + }, } testDataLogPropertiesExclude := &LogMatchProperties{ - MinSeverity: logSeverity("ERROR"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("ERROR"), + }, } factories, err := componenttest.NopFactories() diff --git a/processor/filterprocessor/filter_processor_logs_test.go b/processor/filterprocessor/filter_processor_logs_test.go index 8baa1322043b2..2a785f1313fad 100644 --- a/processor/filterprocessor/filter_processor_logs_test.go +++ b/processor/filterprocessor/filter_processor_logs_test.go @@ -492,7 +492,9 @@ var ( name: "includeMinSeverityINFO", inc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("INFO"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -504,7 +506,9 @@ var ( name: "includeMinSeverityDEBUG", inc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("DEBUG"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("DEBUG"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -516,9 +520,11 @@ var ( { name: "includeMinSeverityFATAL+undefined", inc: &LogMatchProperties{ - LogMatchType: Regexp, - MinSeverity: logSeverity("FATAL"), - MatchUndefinedSeverity: true, + LogMatchType: Regexp, + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("FATAL"), + MatchUndefined: true, + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -529,7 +535,9 @@ var ( name: "excludeMinSeverityINFO", exc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("INFO"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -541,7 +549,9 @@ var ( name: "excludeMinSeverityTRACE", exc: &LogMatchProperties{ LogMatchType: Regexp, - MinSeverity: logSeverity("TRACE"), + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("TRACE"), + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ @@ -551,9 +561,11 @@ var ( { name: "excludeMinSeverityINFO+undefined", exc: &LogMatchProperties{ - LogMatchType: Regexp, - MinSeverity: logSeverity("INFO"), - MatchUndefinedSeverity: true, + LogMatchType: Regexp, + SeverityNumberProperties: &LogSeverityNumberMatchProperties{ + Min: logSeverity("INFO"), + MatchUndefined: true, + }, }, inLogs: testResourceLogs(inLogForSeverityNumber), outLN: [][]string{ diff --git a/processor/filterprocessor/testdata/config_logs_min_severity.yaml b/processor/filterprocessor/testdata/config_logs_min_severity.yaml index ed987977de28c..f7ae58e36e001 100644 --- a/processor/filterprocessor/testdata/config_logs_min_severity.yaml +++ b/processor/filterprocessor/testdata/config_logs_min_severity.yaml @@ -8,14 +8,16 @@ processors: # This filters out all logs below "INFO" level (the whole DEBUG and TRACE ranges) # Logs with no defined severity will also be matched. include: - min_severity: "INFO" - match_undefined_severity: true + severity_number: + min: "INFO" + match_undefined: true filter/exclude: logs: # any logs matching filters are excluded from remainder of pipeline # This will filter out the "ERROR" and "FATAL" ranges exclude: - min_severity: "ERROR" + severity_number: + min: "ERROR" filter/includeexclude: logs: @@ -23,11 +25,13 @@ processors: # the following will only allow records with severity in the "INFO" and "WARN" ranges to pass, # as well as logs with undefined severity include: - min_severity: "INFO" - match_undefined_severity: true + severity_number: + min: "INFO" + match_undefined: true exclude: - min_severity: "ERROR" + severity_number: + min: "ERROR" exporters: nop: