From 6b988661cc01d01d99175b8f46b6858f0c4f3cf5 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 27 Apr 2023 07:58:24 -0700 Subject: [PATCH] [exporter/awss3] S3 Exporter (#20912) * aws s3 exporter initial version --------- Co-authored-by: Przemek Delewski --- .chloggen/add-s3-exporter-impl.yaml | 16 +++++ exporter/awss3exporter/README.md | 33 +++++---- exporter/awss3exporter/config.go | 23 +++---- exporter/awss3exporter/config_test.go | 8 +-- exporter/awss3exporter/data_writer.go | 6 +- exporter/awss3exporter/doc.go | 18 +++++ exporter/awss3exporter/exporter.go | 60 +++++++++------- exporter/awss3exporter/exporter_test.go | 59 ++++++++++++++++ exporter/awss3exporter/factory.go | 48 +++++++------ exporter/awss3exporter/factory_test.go | 61 ++++++++++++++++ exporter/awss3exporter/go.mod | 3 + exporter/awss3exporter/go.sum | 19 ++++- .../internal/metadata/generated_status.go | 12 ++++ exporter/awss3exporter/marshaler.go | 22 ++++-- exporter/awss3exporter/marshaler_test.go | 37 ++++++++++ exporter/awss3exporter/metadata.yaml | 7 ++ exporter/awss3exporter/s3_marshaler.go | 46 +++++++++++++ exporter/awss3exporter/s3_writer.go | 69 ++++++++++++++++++- exporter/awss3exporter/s3_writer_test.go | 52 ++++++++++++++ exporter/awss3exporter/testdata/config.yaml | 2 +- exporter/awss3exporter/testdata/default.yaml | 2 +- 21 files changed, 511 insertions(+), 92 deletions(-) create mode 100644 .chloggen/add-s3-exporter-impl.yaml create mode 100644 exporter/awss3exporter/doc.go create mode 100644 exporter/awss3exporter/exporter_test.go create mode 100644 exporter/awss3exporter/factory_test.go create mode 100644 exporter/awss3exporter/internal/metadata/generated_status.go create mode 100644 exporter/awss3exporter/marshaler_test.go create mode 100644 exporter/awss3exporter/metadata.yaml create mode 100644 exporter/awss3exporter/s3_marshaler.go create mode 100644 exporter/awss3exporter/s3_writer_test.go diff --git a/.chloggen/add-s3-exporter-impl.yaml b/.chloggen/add-s3-exporter-impl.yaml new file mode 100644 index 000000000000..903436abf4d0 --- /dev/null +++ b/.chloggen/add-s3-exporter-impl.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: awss3exporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add implementation of AWS S3 exporter + +# One or more tracking issues related to the change +issues: [2835] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/exporter/awss3exporter/README.md b/exporter/awss3exporter/README.md index 2129f0285cd1..3ec4dfa9114b 100644 --- a/exporter/awss3exporter/README.md +++ b/exporter/awss3exporter/README.md @@ -1,26 +1,31 @@ # AWS S3 Exporter for OpenTelemetry Collector -| Status | | -| ------------------------ |-----------------------| -| Stability | [in development] | -| Supported pipeline types | traces, logs | -| Distributions | [contrib] | + +| Status | | +| ------------------------ |-----------| +| Stability | [alpha] | +| Supported pipeline types | logs, metrics, traces | +| Distributions | [contrib] | + +[alpha]: https://github.com/open-telemetry/opentelemetry-collector#alpha +[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib + ## Schema supported -This exporter targets to support proto/json and proto/binary format +This exporter targets to support proto/json format. ## Exporter Configuration The following exporter configuration parameters are supported. -| Name | Description | Default | -| :--------------------- | :--------------------------------------------------------------------------------- | ------- | -| `region` | AWS region. | | -| `s3_bucket` | S3 bucket | | -| `s3_prefix` | prefix for the S3 key (root directory inside bucket). | | -| `s3_partition` | time granularity of S3 key: hour or minute |"minute" | -| `file_prefix` | file prefix defined by user | | -| `marshaler_name` | marshaler used to produce output data otlp_json or otlp_proto | | +| Name | Description | Default | +|:---------------|:------------------------------------------------------|----------| +| `region` | AWS region. | | +| `s3_bucket` | S3 bucket | | +| `s3_prefix` | prefix for the S3 key (root directory inside bucket). | | +| `s3_partition` | time granularity of S3 key: hour or minute | "minute" | +| `file_prefix` | file prefix defined by user | | +| `marshaler` | marshaler used to produce output data otlp_json | | # Example Configuration diff --git a/exporter/awss3exporter/config.go b/exporter/awss3exporter/config.go index 1e5326d2fdff..46587d115208 100644 --- a/exporter/awss3exporter/config.go +++ b/exporter/awss3exporter/config.go @@ -1,4 +1,4 @@ -// Copyright 2022 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,10 +14,6 @@ package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter" -import ( - "go.uber.org/zap" -) - // S3UploaderConfig contains aws s3 uploader related config to controls things // like bucket, prefix, batching, connections, retries, etc. type S3UploaderConfig struct { @@ -28,15 +24,16 @@ type S3UploaderConfig struct { FilePrefix string `mapstructure:"file_prefix"` } -// Config contains the main configuration options for the awskinesis exporter +type MarshalerType string + +const ( + OtlpJSON MarshalerType = "otlp_json" +) + +// Config contains the main configuration options for the s3 exporter type Config struct { S3Uploader S3UploaderConfig `mapstructure:"s3uploader"` - MarshalerName string `mapstructure:"marshaler_name"` - - // ResourceToTelemetrySettings is the option for converting resource attrihutes to telemetry attributes. - // "Enabled" - A boolean field to enable/disable this option. Default is `false`. - // If enabled, all the resource attributes will be converted to metric labels by default. - // exporterhelper.ResourceToTelemetrySettings `mapstructure:"resource_to_telemetry_conversion"` + MarshalerName MarshalerType `mapstructure:"marshaler"` - logger *zap.Logger + FileFormat string `mapstructure:"file_format"` } diff --git a/exporter/awss3exporter/config_test.go b/exporter/awss3exporter/config_test.go index e4230d7c0c2c..2f4a773a5231 100644 --- a/exporter/awss3exporter/config_test.go +++ b/exporter/awss3exporter/config_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,13 +29,13 @@ func TestLoadConfig(t *testing.T) { assert.NoError(t, err) factory := NewFactory() - factories.Exporters[typeStr] = factory + factories.Exporters["awss3"] = factory cfg, err := otelcoltest.LoadConfigAndValidate(filepath.Join("testdata", "default.yaml"), factories) require.NoError(t, err) require.NotNil(t, cfg) - e := cfg.Exporters[component.NewID(typeStr)].(*Config) + e := cfg.Exporters[component.NewID("awss3")].(*Config) assert.Equal(t, e, &Config{ S3Uploader: S3UploaderConfig{ @@ -59,7 +59,7 @@ func TestConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, cfg) - e := cfg.Exporters[component.NewID(typeStr)].(*Config) + e := cfg.Exporters[component.NewID("awss3")].(*Config) assert.Equal(t, e, &Config{ diff --git a/exporter/awss3exporter/data_writer.go b/exporter/awss3exporter/data_writer.go index 1bd57e9bee45..66cc4b5c2e84 100644 --- a/exporter/awss3exporter/data_writer.go +++ b/exporter/awss3exporter/data_writer.go @@ -1,4 +1,4 @@ -// Copyright 2022 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,6 @@ package awss3exporter // import "github.com/open-telemetry/opentelemetry-collect import "context" -type DataWriter interface { - WriteBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error +type dataWriter interface { + writeBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error } diff --git a/exporter/awss3exporter/doc.go b/exporter/awss3exporter/doc.go new file mode 100644 index 000000000000..48063e193486 --- /dev/null +++ b/exporter/awss3exporter/doc.go @@ -0,0 +1,18 @@ +// Copyright OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate mdatagen metadata.yaml + +// Package awss3exporter stores OpenTelemetry data as an AWS S3 exporter. +package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter" diff --git a/exporter/awss3exporter/exporter.go b/exporter/awss3exporter/exporter.go index 08db13c2ec87..607cefe4c130 100644 --- a/exporter/awss3exporter/exporter.go +++ b/exporter/awss3exporter/exporter.go @@ -1,4 +1,4 @@ -// Copyright 2022 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ import ( "context" "errors" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/plog" @@ -27,56 +26,65 @@ import ( "go.uber.org/zap" ) -type S3Exporter struct { - dataWriter DataWriter +type s3Exporter struct { + config *Config + dataWriter dataWriter logger *zap.Logger - marshaler Marshaler + marshaler marshaler } -func NewS3Exporter(config *Config, - params exporter.CreateSettings) (*S3Exporter, error) { +func newS3Exporter(config *Config, + params exporter.CreateSettings) (*s3Exporter, error) { if config == nil { return nil, errors.New("s3 exporter config is nil") } logger := params.Logger - expConfig := config - expConfig.logger = logger - marshaler, err := NewMarshaler(expConfig.MarshalerName, logger) + m, err := NewMarshaler(config.MarshalerName, logger) if err != nil { return nil, errors.New("unknown marshaler") } - s3Exporter := &S3Exporter{ - dataWriter: &S3Writer{}, + s3Exporter := &s3Exporter{ + config: config, + dataWriter: &s3Writer{}, logger: logger, - marshaler: marshaler, + marshaler: m, } return s3Exporter, nil } -func (e *S3Exporter) Capabilities() consumer.Capabilities { +func (e *s3Exporter) Capabilities() consumer.Capabilities { return consumer.Capabilities{MutatesData: false} } -func (e *S3Exporter) Start(ctx context.Context, host component.Host) error { - return nil -} +func (e *s3Exporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { + buf, err := e.marshaler.MarshalMetrics(md) -func (e *S3Exporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { - return nil -} + if err != nil { + return err + } -func (e *S3Exporter) ConsumeLogs(ctx context.Context, logs plog.Logs) error { - return nil + return e.dataWriter.writeBuffer(ctx, buf, e.config, "metrics", e.marshaler.format()) } -func (e *S3Exporter) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error { - return nil +func (e *s3Exporter) ConsumeLogs(ctx context.Context, logs plog.Logs) error { + buf, err := e.marshaler.MarshalLogs(logs) + + if err != nil { + return err + } + + return e.dataWriter.writeBuffer(ctx, buf, e.config, "logs", e.marshaler.format()) } -func (e *S3Exporter) Shutdown(context.Context) error { - return nil +func (e *s3Exporter) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error { + buf, err := e.marshaler.MarshalTraces(traces) + if err != nil { + return err + } + + return e.dataWriter.writeBuffer(ctx, buf, e.config, "traces", e.marshaler.format()) } diff --git a/exporter/awss3exporter/exporter_test.go b/exporter/awss3exporter/exporter_test.go new file mode 100644 index 000000000000..d5b35165607f --- /dev/null +++ b/exporter/awss3exporter/exporter_test.go @@ -0,0 +1,59 @@ +// Copyright OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awss3exporter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" +) + +var testLogs = []byte(`{"resourceLogs":[{"resource":{"attributes":[{"key":"_sourceCategory","value":{"stringValue":"logfile"}},{"key":"_sourceHost","value":{"stringValue":"host"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"observedTimeUnixNano":"1654257420681895000","body":{"stringValue":"2022-06-03 13:57:00.62739 +0200 CEST m=+14.018296742 log entry14"},"attributes":[{"key":"log.file.path_resolved","value":{"stringValue":"logwriter/data.log"}}],"traceId":"","spanId":""}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.6.1"}]}`) + +type TestWriter struct { + t *testing.T +} + +func (testWriter *TestWriter) writeBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error { + assert.Equal(testWriter.t, testLogs, buf) + return nil +} + +func getTestLogs(tb testing.TB) plog.Logs { + logsMarshaler := plog.JSONUnmarshaler{} + logs, err := logsMarshaler.UnmarshalLogs(testLogs) + assert.NoError(tb, err, "Can't unmarshal testing logs data -> %s", err) + return logs +} + +func getLogExporter(t *testing.T) *s3Exporter { + marshaler, _ := NewMarshaler("otlp_json", zap.NewNop()) + exporter := &s3Exporter{ + config: createDefaultConfig().(*Config), + dataWriter: &TestWriter{t}, + logger: zap.NewNop(), + marshaler: marshaler, + } + return exporter +} + +func TestLog(t *testing.T) { + logs := getTestLogs(t) + exporter := getLogExporter(t) + assert.NoError(t, exporter.ConsumeLogs(context.Background(), logs)) +} diff --git a/exporter/awss3exporter/factory.go b/exporter/awss3exporter/factory.go index 197efdfb8468..91bed578d8fd 100644 --- a/exporter/awss3exporter/factory.go +++ b/exporter/awss3exporter/factory.go @@ -1,4 +1,4 @@ -// Copyright 2022 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,22 +20,19 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" -) -const ( - // The value of "type" key in configuration. - typeStr = "awss3" - // The stability level of the exporter. - stability = component.StabilityLevelAlpha + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter/internal/metadata" ) // NewFactory creates a factory for S3 exporter. func NewFactory() exporter.Factory { return exporter.NewFactory( - typeStr, + metadata.Type, createDefaultConfig, - exporter.WithTraces(createTracesExporter, stability), - exporter.WithLogs(createLogsExporter, stability)) + exporter.WithTraces(createTracesExporter, metadata.Stability), + exporter.WithLogs(createLogsExporter, metadata.Stability), + exporter.WithMetrics(createMetricsExporter, metadata.Stability), + ) } func createDefaultConfig() component.Config { @@ -44,39 +41,48 @@ func createDefaultConfig() component.Config { Region: "us-east-1", S3Partition: "minute", }, - MarshalerName: "otlp_json", - logger: nil, } } func createLogsExporter(ctx context.Context, params exporter.CreateSettings, - config component.Config) (exp exporter.Logs, err error) { + config component.Config) (exporter.Logs, error) { - s3Exporter, err := NewS3Exporter(config.(*Config), params) + s3Exporter, err := newS3Exporter(config.(*Config), params) if err != nil { return nil, err } - return exporterhelper.NewLogsExporter( - context.TODO(), - params, + return exporterhelper.NewLogsExporter(ctx, params, config, s3Exporter.ConsumeLogs) } +func createMetricsExporter(ctx context.Context, + params exporter.CreateSettings, + config component.Config) (exporter.Metrics, error) { + + s3Exporter, err := newS3Exporter(config.(*Config), params) + if err != nil { + return nil, err + } + + return exporterhelper.NewMetricsExporter(ctx, params, + config, + s3Exporter.ConsumeMetrics) +} + func createTracesExporter(ctx context.Context, params exporter.CreateSettings, - config component.Config) (exp exporter.Traces, err error) { + config component.Config) (exporter.Traces, error) { - s3Exporter, err := NewS3Exporter(config.(*Config), params) + s3Exporter, err := newS3Exporter(config.(*Config), params) if err != nil { return nil, err } - return exporterhelper.NewTracesExporter( - context.TODO(), + return exporterhelper.NewTracesExporter(ctx, params, config, s3Exporter.ConsumeTraces) diff --git a/exporter/awss3exporter/factory_test.go b/exporter/awss3exporter/factory_test.go new file mode 100644 index 000000000000..c4c5ef7b4e36 --- /dev/null +++ b/exporter/awss3exporter/factory_test.go @@ -0,0 +1,61 @@ +// Copyright OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awss3exporter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/exporter/exportertest" +) + +func TestCreateDefaultConfig(t *testing.T) { + cfg := createDefaultConfig() + assert.NotNil(t, cfg, "failed to create default config") + assert.NoError(t, componenttest.CheckConfigStruct(cfg)) +} + +func TestCreateMetricsExporter(t *testing.T) { + cfg := createDefaultConfig() + exp, err := createMetricsExporter( + context.Background(), + exportertest.NewNopCreateSettings(), + cfg) + assert.NoError(t, err) + require.NotNil(t, exp) +} + +func TestCreateTracesExporter(t *testing.T) { + cfg := createDefaultConfig() + exp, err := createTracesExporter( + context.Background(), + exportertest.NewNopCreateSettings(), + cfg) + assert.NoError(t, err) + require.NotNil(t, exp) +} + +func TestCreateLogsExporter(t *testing.T) { + cfg := createDefaultConfig() + exp, err := createLogsExporter( + context.Background(), + exportertest.NewNopCreateSettings(), + cfg) + assert.NoError(t, err) + require.NotNil(t, exp) +} diff --git a/exporter/awss3exporter/go.mod b/exporter/awss3exporter/go.mod index 652016f3ad0e..dc7699405f45 100644 --- a/exporter/awss3exporter/go.mod +++ b/exporter/awss3exporter/go.mod @@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3e go 1.19 require ( + github.com/aws/aws-sdk-go v1.44.244 github.com/stretchr/testify v1.8.2 go.opentelemetry.io/collector v0.76.1-0.20230426191218-56daa378f504 go.opentelemetry.io/collector/component v0.76.1-0.20230426191218-56daa378f504 @@ -28,8 +29,10 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knadh/koanf v1.5.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/exporter/awss3exporter/go.sum b/exporter/awss3exporter/go.sum index 18aaa25bfa14..51a57a505d77 100644 --- a/exporter/awss3exporter/go.sum +++ b/exporter/awss3exporter/go.sum @@ -46,6 +46,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.44.244 h1:QzBWLD5HjZHdRZyTMTOWtD9Pobzf1n8/CeTJB4giXi0= +github.com/aws/aws-sdk-go v1.44.244/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= @@ -78,6 +80,7 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -227,7 +230,9 @@ github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEF github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -252,8 +257,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -392,6 +398,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= @@ -458,6 +465,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -491,6 +499,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -528,6 +537,8 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -549,6 +560,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -607,12 +619,15 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -622,6 +637,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -671,6 +687,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/exporter/awss3exporter/internal/metadata/generated_status.go b/exporter/awss3exporter/internal/metadata/generated_status.go new file mode 100644 index 000000000000..9cb577081706 --- /dev/null +++ b/exporter/awss3exporter/internal/metadata/generated_status.go @@ -0,0 +1,12 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +const ( + Type = "awss3" + Stability = component.StabilityLevelAlpha +) diff --git a/exporter/awss3exporter/marshaler.go b/exporter/awss3exporter/marshaler.go index f94808d9cafe..f8581f4c7306 100644 --- a/exporter/awss3exporter/marshaler.go +++ b/exporter/awss3exporter/marshaler.go @@ -1,4 +1,4 @@ -// Copyright 2022 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,20 +18,32 @@ import ( "errors" "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" ) -type Marshaler interface { +type marshaler interface { MarshalTraces(td ptrace.Traces) ([]byte, error) MarshalLogs(ld plog.Logs) ([]byte, error) - Format() string + MarshalMetrics(md pmetric.Metrics) ([]byte, error) + format() string } var ( ErrUnknownMarshaler = errors.New("unknown marshaler") ) -func NewMarshaler(name string, logger *zap.Logger) (Marshaler, error) { - return nil, nil +func NewMarshaler(mType MarshalerType, logger *zap.Logger) (marshaler, error) { + marshaler := &s3Marshaler{logger: logger} + switch mType { + case OtlpJSON: + marshaler.logsMarshaler = &plog.JSONMarshaler{} + marshaler.tracesMarshaler = &ptrace.JSONMarshaler{} + marshaler.metricsMarshaler = &pmetric.JSONMarshaler{} + marshaler.fileFormat = "json" + default: + return nil, ErrUnknownMarshaler + } + return marshaler, nil } diff --git a/exporter/awss3exporter/marshaler_test.go b/exporter/awss3exporter/marshaler_test.go new file mode 100644 index 000000000000..0e1ced0404bf --- /dev/null +++ b/exporter/awss3exporter/marshaler_test.go @@ -0,0 +1,37 @@ +// Copyright OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awss3exporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestMarshaler(t *testing.T) { + { + m, err := NewMarshaler("otlp_json", zap.NewNop()) + assert.NoError(t, err) + require.NotNil(t, m) + assert.Equal(t, m.format(), "json") + } + { + m, err := NewMarshaler("unknown", zap.NewNop()) + assert.Error(t, err) + require.Nil(t, m) + } +} diff --git a/exporter/awss3exporter/metadata.yaml b/exporter/awss3exporter/metadata.yaml new file mode 100644 index 000000000000..3fbe21b77094 --- /dev/null +++ b/exporter/awss3exporter/metadata.yaml @@ -0,0 +1,7 @@ +type: awss3 + +status: + class: exporter + stability: alpha + pipelines: [logs, metrics, traces] + distributions: [contrib] diff --git a/exporter/awss3exporter/s3_marshaler.go b/exporter/awss3exporter/s3_marshaler.go new file mode 100644 index 000000000000..f720cf612fc4 --- /dev/null +++ b/exporter/awss3exporter/s3_marshaler.go @@ -0,0 +1,46 @@ +// Copyright OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter" + +import ( + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +type s3Marshaler struct { + logsMarshaler plog.Marshaler + tracesMarshaler ptrace.Marshaler + metricsMarshaler pmetric.Marshaler + logger *zap.Logger + fileFormat string +} + +func (marshaler *s3Marshaler) MarshalTraces(td ptrace.Traces) ([]byte, error) { + return marshaler.tracesMarshaler.MarshalTraces(td) +} + +func (marshaler *s3Marshaler) MarshalLogs(ld plog.Logs) ([]byte, error) { + return marshaler.logsMarshaler.MarshalLogs(ld) +} + +func (marshaler *s3Marshaler) MarshalMetrics(md pmetric.Metrics) ([]byte, error) { + return marshaler.metricsMarshaler.MarshalMetrics(md) +} + +func (marshaler *s3Marshaler) format() string { + return marshaler.fileFormat +} diff --git a/exporter/awss3exporter/s3_writer.go b/exporter/awss3exporter/s3_writer.go index cd770d27af12..12cc43a0d7d5 100644 --- a/exporter/awss3exporter/s3_writer.go +++ b/exporter/awss3exporter/s3_writer.go @@ -1,4 +1,4 @@ -// Copyright 2022 OpenTelemetry Authors +// Copyright OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,12 +15,75 @@ package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter" import ( + "bytes" "context" + "fmt" + "math/rand" + "strconv" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" ) -type S3Writer struct { +type s3Writer struct { +} + +// generate the s3 time key based on partition configuration +func getTimeKey(time time.Time, partition string) string { + var timeKey string + year, month, day := time.Date() + hour, minute, _ := time.Clock() + + if partition == "hour" { + timeKey = fmt.Sprintf("year=%d/month=%02d/day=%02d/hour=%02d", year, month, day, hour) + } else { + timeKey = fmt.Sprintf("year=%d/month=%02d/day=%02d/hour=%02d/minute=%02d", year, month, day, hour, minute) + } + return timeKey +} + +func randomInRange(low, hi int) int { + return low + rand.Intn(hi-low) +} + +func getS3Key(time time.Time, keyPrefix string, partition string, filePrefix string, metadata string, fileformat string) string { + timeKey := getTimeKey(time, partition) + randomID := randomInRange(100000000, 999999999) + + s3Key := keyPrefix + "/" + timeKey + "/" + filePrefix + metadata + "_" + strconv.Itoa(randomID) + "." + fileformat + + return s3Key } -func (s3writer *S3Writer) WriteBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error { +func (s3writer *s3Writer) writeBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error { + now := time.Now() + key := getS3Key(now, + config.S3Uploader.S3Prefix, config.S3Uploader.S3Partition, + config.S3Uploader.FilePrefix, metadata, format) + + // create a reader from data data in memory + reader := bytes.NewReader(buf) + + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(config.S3Uploader.Region)}, + ) + + if err != nil { + return err + } + + uploader := s3manager.NewUploader(sess) + + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(config.S3Uploader.S3Bucket), + Key: aws.String(key), + Body: reader, + }) + if err != nil { + return err + } + return nil } diff --git a/exporter/awss3exporter/s3_writer_test.go b/exporter/awss3exporter/s3_writer_test.go new file mode 100644 index 000000000000..0d1d00348a9a --- /dev/null +++ b/exporter/awss3exporter/s3_writer_test.go @@ -0,0 +1,52 @@ +// Copyright OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awss3exporter + +import ( + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestS3TimeKey(t *testing.T) { + const layout = "2006-01-02" + + tm, err := time.Parse(layout, "2022-06-05") + timeKey := getTimeKey(tm, "hour") + + assert.NoError(t, err) + require.NotNil(t, tm) + assert.Equal(t, "year=2022/month=06/day=05/hour=00", timeKey) + + timeKey = getTimeKey(tm, "minute") + assert.Equal(t, "year=2022/month=06/day=05/hour=00/minute=00", timeKey) +} + +func TestS3Key(t *testing.T) { + const layout = "2006-01-02" + + tm, err := time.Parse(layout, "2022-06-05") + + assert.NoError(t, err) + require.NotNil(t, tm) + + re := regexp.MustCompile(`keyprefix/year=2022/month=06/day=05/hour=00/minute=00/fileprefixlogs_([0-9]+).json`) + s3Key := getS3Key(tm, "keyprefix", "minute", "fileprefix", "logs", "json") + matched := re.MatchString(s3Key) + assert.Equal(t, true, matched) +} diff --git a/exporter/awss3exporter/testdata/config.yaml b/exporter/awss3exporter/testdata/config.yaml index 48f2cb5620ec..1310a3bb4ad7 100644 --- a/exporter/awss3exporter/testdata/config.yaml +++ b/exporter/awss3exporter/testdata/config.yaml @@ -8,7 +8,7 @@ exporters: s3_bucket: 'foo' s3_prefix: 'bar' s3_partition: 'minute' - + processors: nop: diff --git a/exporter/awss3exporter/testdata/default.yaml b/exporter/awss3exporter/testdata/default.yaml index cc2c50cbb48a..006698ab55cb 100644 --- a/exporter/awss3exporter/testdata/default.yaml +++ b/exporter/awss3exporter/testdata/default.yaml @@ -6,7 +6,7 @@ exporters: s3uploader: region: 'us-east-1' s3_partition: 'minute' - + processors: nop: