Add header_delimiter option to the csv_parser.

.chloggen/csv-header-delimiter.yaml

@@ -0,0 +1,11 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/stanza
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `header_delimiter` option to the `csv_parser`. 
+
+# One or more tracking issues related to the change
+issues: [18198]

pkg/stanza/docs/operators/csv_parser.md

@@ -11,6 +11,7 @@ The `csv_parser` operator parses the string-type field selected by `parse_from`
 | `header` | required when `header_attribute` not set | A string of delimited field names |
 | `header_attribute` | required when `header` not set | An attribute name to read the header field from, to support dynamic field names |
 | `delimiter` | `,` | A character that will be used as a delimiter. Values `\r` and `\n` cannot be used as a delimiter. |
+| `header_delimiter` | value of `delimiter` | A character that will be used as a delimiter for headers. Values `\r` and `\n` cannot be used as a delimiter. |
 | `lazy_quotes` | `false` | If true, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field. Cannot be true if `ignore_quotes` is true. |
 | `ignore_quotes` | `false` | If true, all quotes are ignored, and fields are simply split on the delimiter. Cannot be true if `lazy_quotes` is true. |
 | `parse_from` | `body` | The [field](../types/field.md) from which the value will be parsed. |

pkg/stanza/operator/parser/csv/config_test.go

@@ -66,6 +66,15 @@ func TestConfig(t *testing.T) {
  return p
  }(),
  },
+ {
+ Name: "header_delimiter",
+ Expect: func() *Config {
+ p := NewConfig()
+ p.Header = "id\tseverity\tmessage"
+ p.HeaderDelimiter = "\t"
+ return p
+ }(),
+ },
  {
  Name: "header_attribute",
  Expect: func() *Config {

pkg/stanza/operator/parser/csv/csv.go

@@ -51,6 +51,7 @@ type Config struct {
  helper.ParserConfig `mapstructure:",squash"`
 
  Header string `mapstructure:"header"`
+ HeaderDelimiter string `mapstructure:"header_delimiter"`
  HeaderAttribute string `mapstructure:"header_attribute"`
  FieldDelimiter string `mapstructure:"delimiter"`
  LazyQuotes bool `mapstructure:"lazy_quotes"`
@@ -68,33 +69,43 @@ func (c Config) Build(logger *zap.SugaredLogger) (operator.Operator, error) {
  c.FieldDelimiter = ","
  }
 
+ if c.HeaderDelimiter == "" {
+ c.HeaderDelimiter = c.FieldDelimiter
+ }
+
  if c.IgnoreQuotes && c.LazyQuotes {
  return nil, errors.New("only one of 'ignore_quotes' or 'lazy_quotes' can be true")
  }
 
  fieldDelimiter := []rune(c.FieldDelimiter)[0]
+ headerDelimiter := []rune(c.HeaderDelimiter)[0]
 
  if len([]rune(c.FieldDelimiter)) != 1 {
  return nil, fmt.Errorf("invalid 'delimiter': '%s'", c.FieldDelimiter)
  }
 
+ if len([]rune(c.HeaderDelimiter)) != 1 {
+ return nil, fmt.Errorf("invalid 'header_delimiter': '%s'", c.HeaderDelimiter)
+ }
+
  var headers []string
  switch {
  case c.Header == "" && c.HeaderAttribute == "":
  return nil, errors.New("missing required field 'header' or 'header_attribute'")
  case c.Header != "" && c.HeaderAttribute != "":
  return nil, errors.New("only one header parameter can be set: 'header' or 'header_attribute'")
- case c.Header != "" && !strings.Contains(c.Header, c.FieldDelimiter):
+ case c.Header != "" && !strings.Contains(c.Header, c.HeaderDelimiter):
  return nil, errors.New("missing field delimiter in header")
  case c.Header != "":
- headers = strings.Split(c.Header, c.FieldDelimiter)
+ headers = strings.Split(c.Header, c.HeaderDelimiter)
  }
 
  return &Parser{
  ParserOperator: parserOperator,
  header: headers,
  headerAttribute: c.HeaderAttribute,
  fieldDelimiter: fieldDelimiter,
+ headerDelimiter: headerDelimiter,
  lazyQuotes: c.LazyQuotes,
  ignoreQuotes: c.IgnoreQuotes,
  parse: generateParseFunc(headers, fieldDelimiter, c.LazyQuotes, c.IgnoreQuotes),
@@ -105,6 +116,7 @@ func (c Config) Build(logger *zap.SugaredLogger) (operator.Operator, error) {
 type Parser struct {
  helper.ParserOperator
  fieldDelimiter rune
+ headerDelimiter rune
  header []string
  headerAttribute string
  lazyQuotes bool
@@ -132,7 +144,7 @@ func (r *Parser) Process(ctx context.Context, e *entry.Entry) error {
  r.Error(err)
  return err
  }
- headers := strings.Split(headerString, string([]rune{r.fieldDelimiter}))
+ headers := strings.Split(headerString, string([]rune{r.headerDelimiter}))
  parse = generateParseFunc(headers, r.fieldDelimiter, r.lazyQuotes, r.ignoreQuotes)
  }
 

pkg/stanza/operator/parser/csv/csv_test.go

@@ -140,6 +140,31 @@ func TestParserCSV(t *testing.T) {
  false,
  false,
  },
+ {
+ "basic-different-delimiters",
+ func(p *Config) {
+ p.Header = testHeader
+ p.HeaderDelimiter = ","
+ p.FieldDelimiter = "|"
+ },
+ []entry.Entry{
+ {
+ Body: "stanza|INFO|started agent",
+ },
+ },
+ []entry.Entry{
+ {
+ Body: "stanza|INFO|started agent",
+ Attributes: map[string]interface{}{
+ "name": "stanza",
+ "sev": "INFO",
+ "msg": "started agent",
+ },
+ },
+ },
+ false,
+ false,
+ },
  {
  "basic-multiple-static-bodies",
  func(p *Config) {
@@ -562,6 +587,32 @@ func TestParserCSV(t *testing.T) {
  true,
  false,
  },
+ {
+ "invalid-header-delimiter",
+ func(p *Config) {
+ // expect []rune of length 1
+ p.Header = "name,,age,,height,,number"
+ p.HeaderDelimiter = ",,"
+ },
+ []entry.Entry{
+ {
+ Body: "stanza,1,400,555-555-5555",
+ },
+ },
+ []entry.Entry{
+ {
+ Attributes: map[string]interface{}{
+ "name": "stanza",
+ "age": "1",
+ "height": "400",
+ "number": "555-555-5555",
+ },
+ Body: "stanza,1,400,555-555-5555",
+ },
+ },
+ true,
+ false,
+ },
  {
  "parse-failure-num-fields-mismatch",
  func(p *Config) {

pkg/stanza/operator/parser/csv/testdata/config.yaml

@@ -7,6 +7,10 @@ delimiter:
  parse_from: body.message
  header: id,severity,message
  delimiter: "\t"
+header_delimiter:
+ type: csv_parser
+ header: "id\tseverity\tmessage"
+ header_delimiter: "\t"
 header_attribute:
  type: csv_parser
  parse_from: body.message