[chore] [exporter/s3exporter] aws s3 exporter initial version (First …

…PR, config + exporter stub) (#9979)

* aws s3 exporter initial version

* fixup! Prerelease collector contrib v0.51.0 (#9975)

* [chore] update changelog.

* update dependabot.yml

* interfaces compliance

* fixing go.mod

* update dependabot.yml

* fixing issue templates

* update CODEOWNERS

* update go.mod

* update CHANGELOG

* fixing CHANGELOG.md

* Minor change to changelog

---------

Co-authored-by: Antoine Toulme <[email protected]>
Co-authored-by: Pablo Baeyens <[email protected]>

.chloggen/s3exporter-1.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awss3exporter
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add new AWS S3 exporter.
+
+# One or more tracking issues related to the change
+issues: [9979]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.github/CODEOWNERS

@@ -38,6 +38,7 @@ exporter/alibabacloudlogserviceexporter/ @open-telemetry/collect
 exporter/awscloudwatchlogsexporter/ @open-telemetry/collector-contrib-approvers @boostchicken
 exporter/awsemfexporter/ @open-telemetry/collector-contrib-approvers @Aneurysm9 @shaochengwang @mxiamxia
 exporter/awskinesisexporter/ @open-telemetry/collector-contrib-approvers @Aneurysm9 @MovieStoreGuy
+exporter/awss3exporter/ @open-telemetry/collector-contrib-approvers @atoulme @pdelewski
 exporter/awsxrayexporter/ @open-telemetry/collector-contrib-approvers @willarmiros
 exporter/azuremonitorexporter/ @open-telemetry/collector-contrib-approvers @pcwiese
 exporter/azuredataexplorerexporter/ @open-telemetry/collector-contrib-approvers @asaharan @ag-ramachandran

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -32,6 +32,7 @@ body:
  - exporter/awscloudwatchlogs
  - exporter/awsemf
  - exporter/awskinesis
+ - exporter/awss3
  - exporter/awsxray
  - exporter/azuredataexplorer
  - exporter/azuremonitor

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -26,6 +26,7 @@ body:
  - exporter/awscloudwatchlogs
  - exporter/awsemf
  - exporter/awskinesis
+ - exporter/awss3
  - exporter/awsxray
  - exporter/azuredataexplorer
  - exporter/azuremonitor

.github/ISSUE_TEMPLATE/other.yaml

@@ -26,6 +26,7 @@ body:
  - exporter/awscloudwatchlogs
  - exporter/awsemf
  - exporter/awskinesis
+ - exporter/awss3
  - exporter/awsxray
  - exporter/azuredataexplorer
  - exporter/azuremonitor

.github/dependabot.yml

@@ -92,6 +92,11 @@ updates:
  schedule:
  interval: "weekly"
  day: "wednesday"
+ - package-ecosystem: "gomod"
+ directory: "/exporter/awss3exporter"
+ schedule:
+ interval: "weekly"
+ day: "wednesday"
  - package-ecosystem: "gomod"
  directory: "/exporter/awsxrayexporter"
  schedule:
@@ -1097,8 +1102,3 @@ updates:
  schedule:
  interval: "weekly"
  day: "wednesday"
- - package-ecosystem: "gomod"
- directory: "/receiver/zookeeperreceiver"
- schedule:
- interval: "weekly"
- day: "wednesday"

exporter/awss3exporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/awss3exporter/README.md

@@ -0,0 +1,51 @@
+# AWS S3 Exporter for OpenTelemetry Collector
+
+| Status | |
+| ------------------------ |-----------------------|
+| Stability | [in development] |
+| Supported pipeline types | traces, logs |
+| Distributions | [contrib] |
+
+## Schema supported
+This exporter targets to support proto/json and proto/binary format
+
+## Exporter Configuration
+
+The following exporter configuration parameters are supported. 
+
+| Name | Description | Default |
+| :--------------------- | :--------------------------------------------------------------------------------- | ------- |
+| `region` | AWS region. | |
+| `s3_bucket` | S3 bucket | |
+| `s3_prefix` | prefix for the S3 key (root directory inside bucket). | |
+| `s3_partition` | time granularity of S3 key: hour or minute |"minute" |
+| `file_prefix` | file prefix defined by user | |
+| `marshaler_name` | marshaler used to produce output data otlp_json or otlp_proto | |
+
+# Example Configuration
+
+Following example configuration defines to store output in 'eu-central' region and bucket named 'databucket'.
+
+```yaml
+exporters:
+ awss3:
+ s3uploader:
+ region: 'eu-central-1'
+ s3_bucket: 'databucket'
+ s3_prefix: 'metric'
+ s3_partition: 'minute'
+```
+
+Logs and traces will be stored inside 'databucket' in the following path format.
+
+```console
+metric/year=XXXX/month=XX/day=XX/hour=XX/minute=XX
+```
+
+## AWS Credential Configuration
+
+This exporter follows default credential resolution for the
+[aws-sdk-go](https://docs.aws.amazon.com/sdk-for-go/api/index.html).
+
+Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the
+credential configuration.

exporter/awss3exporter/config.go

@@ -0,0 +1,42 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import (
+ "go.uber.org/zap"
+)
+
+// S3UploaderConfig contains aws s3 uploader related config to controls things
+// like bucket, prefix, batching, connections, retries, etc.
+type S3UploaderConfig struct {
+ Region string `mapstructure:"region"`
+ S3Bucket string `mapstructure:"s3_bucket"`
+ S3Prefix string `mapstructure:"s3_prefix"`
+ S3Partition string `mapstructure:"s3_partition"`
+ FilePrefix string `mapstructure:"file_prefix"`
+}
+
+// Config contains the main configuration options for the awskinesis exporter
+type Config struct {
+ S3Uploader S3UploaderConfig `mapstructure:"s3uploader"`
+ MarshalerName string `mapstructure:"marshaler_name"`
+
+ // ResourceToTelemetrySettings is the option for converting resource attrihutes to telemetry attributes.
+ // "Enabled" - A boolean field to enable/disable this option. Default is `false`.
+ // If enabled, all the resource attributes will be converted to metric labels by default.
+ // exporterhelper.ResourceToTelemetrySettings `mapstructure:"resource_to_telemetry_conversion"`
+
+ logger *zap.Logger
+}

exporter/awss3exporter/config_test.go

@@ -0,0 +1,75 @@
+// Copyright 2021 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// shttp:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "go.opentelemetry.io/collector/component"
+ "go.opentelemetry.io/collector/otelcol/otelcoltest"
+)
+
+func TestLoadConfig(t *testing.T) {
+ factories, err := otelcoltest.NopFactories()
+ assert.NoError(t, err)
+
+ factory := NewFactory()
+ factories.Exporters[typeStr] = factory
+ cfg, err := otelcoltest.LoadConfigAndValidate(filepath.Join("testdata", "default.yaml"), factories)
+
+ require.NoError(t, err)
+ require.NotNil(t, cfg)
+
+ e := cfg.Exporters[component.NewID(typeStr)].(*Config)
+ assert.Equal(t, e,
+ &Config{
+ S3Uploader: S3UploaderConfig{
+ Region: "us-east-1",
+ S3Partition: "minute",
+ },
+ MarshalerName: "otlp_json",
+ },
+ )
+}
+
+func TestConfig(t *testing.T) {
+ factories, err := otelcoltest.NopFactories()
+ assert.Nil(t, err)
+
+ factory := NewFactory()
+ factories.Exporters[factory.Type()] = factory
+ cfg, err := otelcoltest.LoadConfigAndValidate(
+ filepath.Join("testdata", "config.yaml"), factories)
+
+ require.NoError(t, err)
+ require.NotNil(t, cfg)
+
+ e := cfg.Exporters[component.NewID(typeStr)].(*Config)
+
+ assert.Equal(t, e,
+ &Config{
+ S3Uploader: S3UploaderConfig{
+ Region: "us-east-1",
+ S3Bucket: "foo",
+ S3Prefix: "bar",
+ S3Partition: "minute",
+ },
+ MarshalerName: "otlp_json",
+ },
+ )
+}

exporter/awss3exporter/data_writer.go

@@ -0,0 +1,21 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import "context"
+
+type DataWriter interface {
+ WriteBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error
+}

exporter/awss3exporter/exporter.go

@@ -0,0 +1,82 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http:https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import (
+ "context"
+ "errors"
+
+ "go.opentelemetry.io/collector/component"
+ "go.opentelemetry.io/collector/consumer"
+ "go.opentelemetry.io/collector/exporter"
+ "go.opentelemetry.io/collector/pdata/plog"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+ "go.opentelemetry.io/collector/pdata/ptrace"
+ "go.uber.org/zap"
+)
+
+type S3Exporter struct {
+ dataWriter DataWriter
+ logger *zap.Logger
+ marshaler Marshaler
+}
+
+func NewS3Exporter(config *Config,
+ params exporter.CreateSettings) (*S3Exporter, error) {
+
+ if config == nil {
+ return nil, errors.New("s3 exporter config is nil")
+ }
+
+ logger := params.Logger
+ expConfig := config
+ expConfig.logger = logger
+
+ marshaler, err := NewMarshaler(expConfig.MarshalerName, logger)
+ if err != nil {
+ return nil, errors.New("unknown marshaler")
+ }
+
+ s3Exporter := &S3Exporter{
+ dataWriter: &S3Writer{},
+ logger: logger,
+ marshaler: marshaler,
+ }
+ return s3Exporter, nil
+}
+
+func (e *S3Exporter) Capabilities() consumer.Capabilities {
+ return consumer.Capabilities{MutatesData: false}
+}
+
+func (e *S3Exporter) Start(ctx context.Context, host component.Host) error {
+ return nil
+}
+
+func (e *S3Exporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error {
+ return nil
+}
+
+func (e *S3Exporter) ConsumeLogs(ctx context.Context, logs plog.Logs) error {
+ return nil
+}
+
+func (e *S3Exporter) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error {
+ return nil
+}
+
+func (e *S3Exporter) Shutdown(context.Context) error {
+ return nil
+}