Skip to content

Commit

Permalink
Merge pull request #1 from keyval-dev/add-s3-exporter
Browse files Browse the repository at this point in the history
add s3 exporter
  • Loading branch information
edeNFed authored Mar 24, 2023
2 parents 5bed9e9 + e4a91fa commit 5acb471
Show file tree
Hide file tree
Showing 17 changed files with 923 additions and 40 deletions.
2 changes: 2 additions & 0 deletions cmd/otelcontribcol/builder-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ exporters:
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudpubsubexporter v0.71.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudstorageexporter v0.71.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azureblobstorageexporter v0.71.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter v0.71.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlemanagedprometheusexporter v0.71.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/influxdbexporter v0.71.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/instanaexporter v0.71.0
Expand Down Expand Up @@ -196,6 +197,7 @@ replaces:
- github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudpubsubexporter => ../../exporter/googlecloudpubsubexporter
- github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudstorageexporter => ../../exporter/googlecloudstorageexporter
- github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azureblobstorageexporter => ../../exporter/azureblobstorageexporter
- github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter => ../../exporter/awss3exporter
- github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => ../../exporter/awsxrayexporter
- github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => ../../receiver/postgresqlreceiver
- github.com/open-telemetry/opentelemetry-collector-contrib/exporter/jaegerexporter => ../../exporter/jaegerexporter
Expand Down
2 changes: 2 additions & 0 deletions cmd/otelcontribcol/components.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 19 additions & 16 deletions cmd/otelcontribcol/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.71.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.71.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awskinesisexporter v0.71.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter v0.71.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.71.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azureblobstorageexporter v0.71.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azuremonitorexporter v0.71.0
Expand Down Expand Up @@ -254,25 +255,25 @@ require (
github.com/ardielle/ardielle-go v1.5.2 // indirect
github.com/armon/go-metrics v0.4.0 // indirect
github.com/aws/aws-sdk-go v1.44.196 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.6 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.24 // indirect
github.com/aws/aws-sdk-go-v2/service/kinesis v1.17.3 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
Expand Down Expand Up @@ -668,6 +669,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googl

replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azureblobstorageexporter => ../../exporter/azureblobstorageexporter

replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter => ../../exporter/awss3exporter

replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => ../../exporter/awsxrayexporter

replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => ../../receiver/postgresqlreceiver
Expand Down
56 changes: 32 additions & 24 deletions cmd/otelcontribcol/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions exporter/awss3exporter/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
include ../../Makefile.Common
2 changes: 2 additions & 0 deletions exporter/awss3exporter/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# AWS S3 Exporter
This exporter supports sending trace and metric data to AWS S3.
21 changes: 21 additions & 0 deletions exporter/awss3exporter/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package awss3exporter

import (
"go.uber.org/zap"
)

type AWSS3UploadConfig struct {
S3Bucket string `mapstructure:"bucket"`
S3Region string `mapstructure:"region"`
S3Prefix string `mapstructure:"prefix"`
S3Partition string `mapstructure:"partition"`
FilePrefix string `mapstructure:"file_prefix"`
}

// Config contains the main configuration options for the aws s3 exporter
type Config struct {
AWSS3UploadConfig AWSS3UploadConfig `mapstructure:"settings"`
MarshalerName string `mapstructure:"marshaler_name"`

logger *zap.Logger
}
7 changes: 7 additions & 0 deletions exporter/awss3exporter/data_writer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package awss3exporter

import "context"

type DataWriter interface {
WriteBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error
}
79 changes: 79 additions & 0 deletions exporter/awss3exporter/exporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package awss3exporter

import (
"context"
"errors"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"go.opentelemetry.io/collector/exporter"

"go.opentelemetry.io/collector/consumer"
"go.opentelemetry.io/collector/pdata/plog"
"go.opentelemetry.io/collector/pdata/ptrace"
"go.uber.org/zap"
)

type S3Exporter struct {
config *Config
dataWriter DataWriter
logger *zap.Logger
marshaler Marshaler
}

func NewS3Exporter(cfg *Config,
params exporter.CreateSettings) (*S3Exporter, error) {

if cfg == nil {
return nil, errors.New("s3 exporter config is nil")
}

logger := params.Logger
expConfig := cfg
expConfig.logger = logger

s3Config, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(cfg.AWSS3UploadConfig.S3Region))
if err != nil {
return nil, err
}

s3Client := s3.NewFromConfig(s3Config)

marshaler, err := NewMarshaler(expConfig.MarshalerName, logger)
if err != nil {
return nil, errors.New("unknown marshaler")
}

s3Exporter := &S3Exporter{
config: expConfig,
dataWriter: &S3Writer{
s3Client: s3Client,
},
logger: logger,
marshaler: marshaler,
}

return s3Exporter, nil
}

func (e *S3Exporter) Capabilities() consumer.Capabilities {
return consumer.Capabilities{MutatesData: false}
}

func (e *S3Exporter) ConsumeLogs(ctx context.Context, logs plog.Logs) error {
buf, err := e.marshaler.MarshalLogs(logs)

if err != nil {
return err
}

return e.dataWriter.WriteBuffer(ctx, buf, e.config, "logs", e.marshaler.Format())
}

func (e *S3Exporter) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error {
buf, err := e.marshaler.MarshalTraces(traces)
if err != nil {
return err
}

return e.dataWriter.WriteBuffer(ctx, buf, e.config, "traces", e.marshaler.Format())
}
Loading

0 comments on commit 5acb471

Please sign in to comment.