From afe06861559838a96807e6221831f0f54eae981d Mon Sep 17 00:00:00 2001 From: An Quach Date: Sat, 6 Jan 2024 20:30:57 +0700 Subject: [PATCH] [exporter/awss3] add marsheler config to pick only body content of logs update chlog-new update readme --- .chloggen/feat_allow_pick_only_log_body.yaml | 27 ++++++++++ exporter/awss3exporter/README.md | 2 + exporter/awss3exporter/config.go | 1 + exporter/awss3exporter/logbody_marshaler.go | 52 ++++++++++++++++++++ exporter/awss3exporter/marshaler.go | 4 ++ exporter/awss3exporter/marshaler_test.go | 6 +++ 6 files changed, 92 insertions(+) create mode 100755 .chloggen/feat_allow_pick_only_log_body.yaml create mode 100644 exporter/awss3exporter/logbody_marshaler.go diff --git a/.chloggen/feat_allow_pick_only_log_body.yaml b/.chloggen/feat_allow_pick_only_log_body.yaml new file mode 100755 index 0000000000000..d9c2ecc6fad99 --- /dev/null +++ b/.chloggen/feat_allow_pick_only_log_body.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: awss3exporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add config option is only pick the body of logs for simplify AWS Athena connect and query + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [30318] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/awss3exporter/README.md b/exporter/awss3exporter/README.md index 93a90b4d0141d..d50df65ca57f4 100644 --- a/exporter/awss3exporter/README.md +++ b/exporter/awss3exporter/README.md @@ -42,6 +42,8 @@ Marshaler determines the format of data sent to AWS S3. Currently, the following - `otlp_json` (default): the [OpenTelemetry Protocol format](https://github.com/open-telemetry/opentelemetry-proto), represented as json. - `sumo_ic`: the [Sumo Logic Installed Collector Archive format](https://help.sumologic.com/docs/manage/data-archiving/archive/). **This format is supported only for logs.** +- `body_log`: In case you only want to wrap logs body for simple to use Athena query. + **This format is supported only for logs.** # Example Configuration diff --git a/exporter/awss3exporter/config.go b/exporter/awss3exporter/config.go index 50a6079919333..a20f6ad2db4e8 100644 --- a/exporter/awss3exporter/config.go +++ b/exporter/awss3exporter/config.go @@ -28,6 +28,7 @@ type MarshalerType string const ( OtlpJSON MarshalerType = "otlp_json" SumoIC MarshalerType = "sumo_ic" + BodyLog MarshalerType = "body_log" ) // Config contains the main configuration options for the s3 exporter diff --git a/exporter/awss3exporter/logbody_marshaler.go b/exporter/awss3exporter/logbody_marshaler.go new file mode 100644 index 0000000000000..9fc909c382f13 --- /dev/null +++ b/exporter/awss3exporter/logbody_marshaler.go @@ -0,0 +1,52 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter" + +import ( + "bytes" + "fmt" + + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +type bodylogMarshaler struct{} + +func (*bodylogMarshaler) format() string { + return "log" +} + +func newbodylogMarshaler() bodylogMarshaler { + return bodylogMarshaler{} +} + +func (bodylogMarshaler) MarshalLogs(ld plog.Logs) ([]byte, error) { + buf := bytes.Buffer{} + rls := ld.ResourceLogs() + for i := 0; i < rls.Len(); i++ { + rl := rls.At(i) + + ills := rl.ScopeLogs() + for j := 0; j < ills.Len(); j++ { + ils := ills.At(j) + logs := ils.LogRecords() + for k := 0; k < logs.Len(); k++ { + lr := logs.At(k) + body := lr.Body() + buf.WriteString(body.Str()) + buf.WriteString("\n") + } + } + } + return buf.Bytes(), nil +} + +func (s bodylogMarshaler) MarshalTraces(_ ptrace.Traces) ([]byte, error) { + return nil, fmt.Errorf("traces can't be marshaled into %s format", s.format()) +} + +func (s bodylogMarshaler) MarshalMetrics(_ pmetric.Metrics) ([]byte, error) { + return nil, fmt.Errorf("metrics can't be marshaled into %s format", s.format()) +} diff --git a/exporter/awss3exporter/marshaler.go b/exporter/awss3exporter/marshaler.go index 30d18bfd3b862..7ecb7cdc0a29a 100644 --- a/exporter/awss3exporter/marshaler.go +++ b/exporter/awss3exporter/marshaler.go @@ -35,6 +35,10 @@ func newMarshaler(mType MarshalerType, logger *zap.Logger) (marshaler, error) { sumomarshaler := newSumoICMarshaler() marshaler.logsMarshaler = &sumomarshaler marshaler.fileFormat = "json.gz" + case BodyLog: + bodylogMarshaler := newbodylogMarshaler() + marshaler.logsMarshaler = &bodylogMarshaler + marshaler.fileFormat = bodylogMarshaler.format() default: return nil, ErrUnknownMarshaler } diff --git a/exporter/awss3exporter/marshaler_test.go b/exporter/awss3exporter/marshaler_test.go index d0ede1990099c..2133d74b17166 100644 --- a/exporter/awss3exporter/marshaler_test.go +++ b/exporter/awss3exporter/marshaler_test.go @@ -29,4 +29,10 @@ func TestMarshaler(t *testing.T) { assert.Error(t, err) require.Nil(t, m) } + { + m, err := newMarshaler("body_log", zap.NewNop()) + assert.NoError(t, err) + require.NotNil(t, m) + assert.Equal(t, m.format(), "log") + } }