diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 82ef779..722a0c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "1.20" + go-version-file: go.mod - name: Unit tests run: | diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d5bb845..3ae0281 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -35,6 +35,10 @@ jobs: name: PR - GO lint steps: - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: go.mod - run: | go generate ./... diff --git a/client.go b/client.go index 4d392b9..6e83186 100644 --- a/client.go +++ b/client.go @@ -3,12 +3,11 @@ package s3client import ( "bufio" "context" + "errors" "fmt" "io" "net/http" "path/filepath" - "sync" - "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -18,7 +17,6 @@ import ( ) type ( - // Logger interface to represent a fluent-bit logging mechanism. Logger interface { Error(format string, a ...any) @@ -30,7 +28,7 @@ type ( // Client is the interface for interacting with an S3 bucket. Client interface { ListFiles(ctx context.Context, bucket, pattern string) ([]string, error) - ReadFiles(ctx context.Context, bucket string, files []string, concurrency int, ch chan<- Message) error + ReadFile(ctx context.Context, bucket string, file string, initialBufferSize int, maxBufferSize int) (<-chan string, <-chan error) } // DefaultClient is a concrete implementation of the Client interface that uses the AWS SDK for Go to interact with S3. DefaultClient struct { @@ -38,12 +36,6 @@ type ( Svc ifaces.Client Logger Logger } - - // Message is the msg format as specified by the plugin library (https://github.com/calyptia/plugin/blob/785e54918feb3efb78f9ddeadf135dc4f75fa5b0/plugin.go#L77C1-L81C2) - Message struct { - Time time.Time - Record any - } ) // New returns a new DefaultClient configured with the given options and using the provided logger. @@ -92,7 +84,7 @@ func (c *DefaultClient) ListFiles(ctx context.Context, bucket, pattern string) ( params.Prefix = &prefix } - c.Logger.Info("listing files on bucket: %q with prefix: %q that follows pattern: %q", bucket, prefix, pattern) + c.Logger.Debug("listing files on bucket: %q with prefix: %q that follows pattern: %q", bucket, prefix, pattern) p := s3.NewListObjectsV2Paginator(c.Svc, params) for p.HasMorePages() { @@ -108,7 +100,7 @@ func (c *DefaultClient) ListFiles(ctx context.Context, bucket, pattern string) ( } } } - c.Logger.Info("found: %d file(s) on bucket: %q that follows pattern: %q", len(files), bucket, pattern) + c.Logger.Debug("found: %d file(s) on bucket: %q that follows pattern: %q", len(files), bucket, pattern) return files, nil } @@ -126,137 +118,86 @@ func (c *DefaultClient) ListFiles(ctx context.Context, bucket, pattern string) ( return files, nil } -// ReadFiles reads a list of files from the specified bucket and sends their contents -// through the provided channel. -// -// The function takes in a context object to cancel long-running operations, the name -// of the bucket, a slice of strings representing the names of the files to read, and -// a channel of plugin.Message to send the contents of the files. -// -// The function returns an error if there is a problem reading the files or sending -// the messages. -func (c *DefaultClient) ReadFiles(ctx context.Context, bucket string, files []string, concurrency int, ch chan<- Message) error { - // Create a done channel to signal when all the files have been processed - done := make(chan bool) - // Create an error channel to handle any errors that occur while processing the files +// ReadFile reads the specified file from the given S3 bucket and sends its contents +// line by line through a channel. It uses an adaptive buffering mechanism to handle +// large lines of text up to a specified maximum size. +func (c *DefaultClient) ReadFile(ctx context.Context, bucket string, file string, initialBufferSize int, maxBufferSize int) (<-chan string, <-chan error) { + // Channels to return the file contents and any potential errors. + out := make(chan string) errChan := make(chan error) - // isCritical is a helper function to determine if an error is critical and should cause processing to stop - isCritical := func(err error) bool { - // TODO: add logic to determine if an error is critical - return false - } - - var wg sync.WaitGroup - - if concurrency == 0 { - concurrency = 1 - } - // TODO: make this configurable. - poolSize := concurrency - - // Create a semaphore to limit the number of files being processed concurrently - semaphore := make(chan struct{}, poolSize) - - // Iterate over the files and process them concurrently - for _, file := range files { - // Acquire a slot in the semaphore - semaphore <- struct{}{} - // Add a waiting group for the file being processed - wg.Add(1) + go func() { + // Always close the output channel when done. + defer close(out) - go func(filename string) { - // Release the slot in the semaphore and decrement the waiting group when the function exits - defer func() { - wg.Done() - <-semaphore - }() + // Log start of file processing. + c.Logger.Info("Started processing file: %s from bucket: %s", file, bucket) - // Log that the filename processing has started - c.Logger.Info("started processing filename: %s from bucket: %s", filename, bucket) - // Get the filename from the S3 bucket - resp, err := c.Svc.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &filename, - }) + // Get the specified file from the S3 bucket. + resp, err := c.Svc.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &file, + }) + if err != nil { + // On error, send to error channel and exit. + errChan <- err + return + } + // Ensure the file's body stream is closed when done. + // Close the filename body when the function exits + defer func(Body io.ReadCloser) { + err = Body.Close() if err != nil { // Send the error to the error channel errChan <- err return } - // Close the filename body when the function exits - defer func(Body io.ReadCloser) { - err = Body.Close() - if err != nil { - // Send the error to the error channel - errChan <- err - return - } - }(resp.Body) + }(resp.Body) - c.Logger.Info("getting a filename reader for filename: %q and content-type %q", filename, *resp.ContentType) - - // Get a reader for the filename based on its content type - reader, err := GetFileReader(filename)(resp.Body) + // Get a reader for the file based on its format/type. + reader, err := GetFileReader(file)(resp.Body) + if err != nil { + // On error, send to error channel and exit. + errChan <- err + return + } + // Ensure the reader is closed when done. + defer func(reader io.ReadCloser) { + err := reader.Close() if err != nil { - // Send the error to the error channel errChan <- err return } - defer func() { - err := reader.Close() - if err != nil { - c.Logger.Error("error closing reader fp: %w", err) - } - }() + }(reader) - // Use a buffered reader to read the filename line by line - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - // Send each line of the filename as a message on the channel - ch <- Message{ - Time: time.Now(), - Record: map[string]string{ - "_raw": scanner.Text(), - "bucket": bucket, - "file": filename, - }, - } - } - // If there is an error with the scanner, send the error to the errChan - // channel and return from the function. - if err := scanner.Err(); err != nil { - errChan <- err - return - } - c.Logger.Info("completed processing of filename: %s on bucket: %s", filename, bucket) - }(file) - } + // Create a scanner to read the file contents. + scanner := bufio.NewScanner(reader) - // Start a goroutine that waits for all the file processing goroutines to finish. - // When they are all done, send a message on the done channel. - go func() { - wg.Wait() - done <- true - }() + // Initialize a buffer for the scanner, setting its initial and maximum sizes. + buf := make([]byte, 0, initialBufferSize) + scanner.Buffer(buf, maxBufferSize) + + // Read the file line by line. + for scanner.Scan() { + out <- scanner.Text() + } - // Loop indefinitely until the context is done or the done channel is closed. - for { - select { - case <-ctx.Done(): - // If the context is done, return the context's error. - return ctx.Err() - case <-done: - // If the done channel is closed, close the done channel and return nil. - close(done) - return nil - case err := <-errChan: - // If there is an error on the errChan channel, log it and check if it is critical. - // If it is critical, return the error. - c.Logger.Error("error while processing file from s3 bucket:%w", err) - if isCritical(err) { - return err + // Check for any scanning errors. + if err := scanner.Err(); err != nil { + // If the error is due to a line being too long, log a specific message. + if errors.Is(err, bufio.ErrTooLong) { + c.Logger.Error("Encountered a line that was too long to read in file: %s from bucket: %s, exceeds > %d", file, bucket, maxBufferSize) } + + // Send the error to the error channel and exit. + errChan <- err + return } - } + + // Log completion of file processing. + c.Logger.Info("Completed processing of file: %s on bucket: %s", file, bucket) + }() + + // Return channels to the caller. + return out, errChan } diff --git a/client_test.go b/client_test.go index 54e6ff6..9f6a4fb 100644 --- a/client_test.go +++ b/client_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -25,61 +26,22 @@ func (n NullLogger) Warn(format string, a ...any) {} func (n NullLogger) Info(format string, a ...any) {} func (n NullLogger) Debug(format string, a ...any) {} -func msgsFromFile(filePath string) ([]Message, error) { - var out []Message - - pwd, err := os.Getwd() - if err != nil { - return out, err - } - - path := filepath.Join(pwd, filePath) - - //nolint gosec // this is a test, ignore - file, err := os.Open(path) - if err != nil { - return out, err - } - - reader, err := GetFileReader(path)(bufio.NewReader(file)) - if err != nil { - return out, err - } - defer reader.Close() - - var msgs []Message - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - msgs = append(msgs, Message{ - Record: map[string]string{ - "_raw": scanner.Text(), - "file": path, - }, - }) - } - - return msgs, nil -} - -func TestDefaultClient_ReadFiles(t *testing.T) { +func TestDefaultClient_ReadFile(t *testing.T) { ctx := context.TODO() t.Run("ok", func(t *testing.T) { - tt := []struct { + tt := []*struct { name string - clientMock ifaces.ClientMock - files []string - channel chan Message + clientMock *ifaces.ClientMock + file string expectedErr error - expectedMessages func() []Message bucket string + expectedMessages func() []string }{ { name: "single line", - files: []string{ - "single-line-file.txt", - }, - clientMock: ifaces.ClientMock{ + file: "single-line-file.txt", + clientMock: &ifaces.ClientMock{ GetObjectFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{ Body: io.NopCloser(bytes.NewReader([]byte("single line"))), @@ -87,28 +49,20 @@ func TestDefaultClient_ReadFiles(t *testing.T) { }, nil }, }, - channel: make(chan Message), - expectedMessages: func() []Message { - return []Message{{ - Record: map[string]string{ - "_raw": "single line", - "file": "single-line-file.txt", - }, - }} + expectedMessages: func() []string { + return []string{"single line"} }, }, { name: "compressed multiline", - files: []string{ - "testadata/large-file.csv.gz", - }, - clientMock: ifaces.ClientMock{ + file: "testdata/large-file.csv.gz", + clientMock: &ifaces.ClientMock{ GetObjectFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { pwd, err := os.Getwd() assert.NoError(t, err) filename := filepath.Join(pwd, "testdata/large-file.csv.gz") - //nolint gosec // this is a test, ignore + // nolint:gosec //ignore this is just a test. content, err := os.ReadFile(filename) assert.NoError(t, err) return &s3.GetObjectOutput{ @@ -117,26 +71,23 @@ func TestDefaultClient_ReadFiles(t *testing.T) { }, nil }, }, - channel: make(chan Message), - expectedMessages: func() []Message { - msgs, err := msgsFromFile("testdata/large-file.csv.gz") + expectedMessages: func() []string { + // assuming a helper function to read file content + lines, err := linesFromFile("testdata/large-file.csv.gz") assert.NoError(t, err) - assert.NotZero(t, msgs) - return msgs + return lines }, }, { name: "compressed with invalid content type", - files: []string{ - "testdata/large-file-invalid-content-type.csv.gz", - }, - clientMock: ifaces.ClientMock{ + file: "testdata/large-file-invalid-content-type.csv.gz", + clientMock: &ifaces.ClientMock{ GetObjectFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { pwd, err := os.Getwd() assert.NoError(t, err) filename := filepath.Join(pwd, "testdata/large-file-invalid-content-type.csv.gz") - //nolint gosec // this is a test, ignore + // nolint:gosec //ignore this is just a test. content, err := os.ReadFile(filename) assert.NoError(t, err) return &s3.GetObjectOutput{ @@ -145,54 +96,98 @@ func TestDefaultClient_ReadFiles(t *testing.T) { }, nil }, }, - channel: make(chan Message), - expectedMessages: func() []Message { - msgs, err := msgsFromFile("testdata/large-file-invalid-content-type.csv.gz") + expectedMessages: func() []string { + lines, err := linesFromFile("testdata/large-file-invalid-content-type.csv.gz") assert.NoError(t, err) - assert.NotZero(t, msgs) - return msgs + return lines + }, + }, + { + name: "line larger than 10MiB", + file: "very-large-line.txt", + clientMock: &ifaces.ClientMock{ + GetObjectFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + // Creating a string with more than 10MiB length. + veryLongLine := strings.Repeat("a", 11*1024*1024) // 11 MiB + + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader([]byte(veryLongLine))), + ContentType: aws.String("text/csv"), + }, nil + }, }, + expectedErr: bufio.ErrTooLong, // Expecting an error in this case + expectedMessages: func() []string { return nil }, // No valid messages to expect here }, } - //nolint copylocks //that's fine. for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { c := DefaultClient{ - Svc: &tc.clientMock, + Svc: tc.clientMock, Logger: NullLogger{}, } - withTimeout, cancel := context.WithTimeout(ctx, 5*time.Second) + withTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) defer cancel() + outCh, errCh := c.ReadFile(withTimeout, tc.bucket, tc.file, 64*1024, 10*1024*1024) + + expectedMessages := tc.expectedMessages() idx := 0 - go func() { - expectedMessages := tc.expectedMessages() - for { - select { - case msg := <-tc.channel: - receivedRecord, ok := msg.Record.(map[string]string) - assert.NotZero(t, ok) - expectedRecord, ok := expectedMessages[idx].Record.(map[string]string) - assert.NotZero(t, ok) - assert.Equal(t, receivedRecord["_raw"], expectedRecord["_raw"]) - assert.Equal(t, filepath.Base(receivedRecord["file"]), filepath.Base(expectedRecord["file"])) + for { + select { + case line := <-outCh: + // Ensure we don't go beyond the expected messages length + if expectedMessages != nil && idx < len(expectedMessages) { + assert.Equal(t, line, expectedMessages[idx], "Mismatch at index %d", idx) idx++ - case <-withTimeout.Done(): - return } + case err := <-errCh: + assert.Equal(t, err, tc.expectedErr) + return + case <-withTimeout.Done(): + return } - }() - - err := c.ReadFiles(withTimeout, tc.bucket, tc.files, 0, tc.channel) - assert.Equal(t, err, tc.expectedErr) - t.Logf("processed: %d messages for test case: %s", idx, tc.name) + } }) } }) } +func linesFromFile(filePath string) ([]string, error) { + var out []string + pwd, err := os.Getwd() + if err != nil { + return out, err + } + + path := filepath.Join(pwd, filePath) + + //nolint gosec // this is a test, ignore + file, err := os.Open(path) + if err != nil { + return out, err + } + + reader, err := GetFileReader(path)(bufio.NewReader(file)) + if err != nil { + return out, err + } + defer reader.Close() + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + out = append(out, scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return out, nil +} + func TestDefaultClient_ListFiles(t *testing.T) { ctx := context.TODO() diff --git a/go.mod b/go.mod index 7872f79..2de48d6 100644 --- a/go.mod +++ b/go.mod @@ -1,32 +1,32 @@ module github.com/calyptia/go-s3-client -go 1.20 +go 1.21.0 require ( github.com/alecthomas/assert/v2 v2.3.0 - github.com/aws/aws-sdk-go v1.44.317 - github.com/aws/aws-sdk-go-v2 v1.20.0 - github.com/aws/aws-sdk-go-v2/config v1.18.32 - github.com/aws/aws-sdk-go-v2/credentials v1.13.31 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 + github.com/aws/aws-sdk-go v1.45.10 + github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2/config v1.18.39 + github.com/aws/aws-sdk-go-v2/credentials v1.13.37 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/bmatcuk/doublestar v1.3.4 ) require ( github.com/alecthomas/repr v0.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect - github.com/aws/smithy-go v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect ) diff --git a/go.sum b/go.sum index 8c155c0..3cd4af3 100644 --- a/go.sum +++ b/go.sum @@ -2,44 +2,44 @@ github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVd github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/aws/aws-sdk-go v1.44.317 h1:+8XWrLmGMwPPXSRSLPzhgcGnzJ2mYkgkrcB9C/GnSOU= -github.com/aws/aws-sdk-go v1.44.317/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.20.0 h1:INUDpYLt4oiPOJl0XwZDK2OVAVf0Rzo+MGVTv9f+gy8= -github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 h1:/MS8AzqYNAhhRNalOmxUvYs8VEbNGifTnzhPFdcRQkQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11/go.mod h1:va22++AdXht4ccO3kH2SHkHHYvZ2G9Utz+CXKmm2CaU= -github.com/aws/aws-sdk-go-v2/config v1.18.32 h1:tqEOvkbTxwEV7hToRcJ1xZRjcATqwDVsWbAscgRKyNI= -github.com/aws/aws-sdk-go-v2/config v1.18.32/go.mod h1:U3ZF0fQRRA4gnbn9GGvOWLoT2EzzZfAWeKwnVrm1rDc= -github.com/aws/aws-sdk-go-v2/credentials v1.13.31 h1:vJyON3lG7R8VOErpJJBclBADiWTwzcwdkQpTKx8D2sk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.31/go.mod h1:T4sESjBtY2lNxLgkIASmeP57b5j7hTQqCbqG0tWnxC4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 h1:X3H6+SU21x+76LRglk21dFRgMTJMa5QcpW+SqUf5BBg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7/go.mod h1:3we0V09SwcJBzNlnyovrR2wWJhWmVdqAsmVs4uronv8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 h1:zr/gxAZkMcvP71ZhQOcvdm8ReLjFgIXnIn0fw5AM7mo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 h1:0HCMIkAkVY9KMgueD8tf4bRTUanzEYvhw7KkPXIMpO0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 h1:+i1DOFrW3YZ3apE45tCal9+aDKK6kNEbW6Ib7e1nFxE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38/go.mod h1:1/jLp0OgOaWIetycOmycW+vYTYgTZFPttJQRgsI1PoU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 h1:uAiiHnWihGP2rVp64fHwzLDrswGjEjsPszwRYMiYQPU= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12/go.mod h1:fUTHpOXqRQpXvEpDPSa3zxCc2fnpW6YnBoba+eQr+Bg= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 h1:kvN1jPHr9UffqqG3bSgZ8tx4+1zKVHz/Ktw/BwW6hX8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32/go.mod h1:QmMEM7es84EUkbYWcpnkx8i5EW2uERPfrTFeOch128Y= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 h1:auGDJ0aLZahF5SPvkJ6WcUuX7iQ7kyl2MamV7Tm8QBk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31/go.mod h1:3+lloe3sZuBQw1aBc5MyndvodzQlyqCZ7x1QPDHaWP4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeuqgPCHIQtbZ87wsgom7S5F8obreg+c= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 h1:pAOJj+80tC8sPVgSDHzMYD6KLWsaLQ1kZw31PTeORbs= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKBczhsbhj2p/YY7qeJezJ3CI= -github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io= -github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go v1.45.10 h1:GoqAm25t0qrs4rrXAjqt3luZnM9mV0lzfNwzgaCKpm4= +github.com/aws/aws-sdk-go v1.45.10/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2/config v1.18.39 h1:oPVyh6fuu/u4OiW4qcuQyEtk7U7uuNBmHmJSLg1AJsQ= +github.com/aws/aws-sdk-go-v2/config v1.18.39/go.mod h1:+NH/ZigdPckFpgB1TRcRuWCB/Kbbvkxc/iNAKTq5RhE= +github.com/aws/aws-sdk-go-v2/credentials v1.13.37 h1:BvEdm09+ZEh2XtN+PVHPcYwKY3wIeB6pw7vPRM4M9/U= +github.com/aws/aws-sdk-go-v2/credentials v1.13.37/go.mod h1:ACLrdkd4CLZyXOghZ8IYumQbcooAcp2jo/s2xsFH8IM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.6/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 h1:pSB560BbVj9ZlJZF4WYj5zsytWHWKxg+NgyGV4B2L58= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/utils.go b/utils.go index 90ece47..d251908 100644 --- a/utils.go +++ b/utils.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "errors" "io" "mime" "path/filepath" @@ -55,7 +56,7 @@ func GetFileReader(filename string) func(io.Reader) (io.ReadCloser, error) { // The default HTTP transports that the AWS SDK uses will decompress objects transparently // if the Content Encoding is gzip. Not everyone or everything properly sets the Content-Encoding // header on their S3 objects, so we could be trying to process gzipped objects and not know it. - if err == gzip.ErrHeader { + if errors.Is(err, gzip.ErrHeader) { rc := io.NopCloser(bytes.NewReader(orig)) return rc, nil }