diff --git a/storage/s3.go b/storage/s3.go index 58bd96a29..0f7d1f9b6 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -311,8 +311,8 @@ func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object { go func() { defer close(objCh) objectFound := false - var now time.Time + var serverDate string err := s.api.ListObjectsV2PagesWithContext(ctx, &listInput, func(p *s3.ListObjectsV2Output, lastPage bool) bool { for _, c := range p.CommonPrefixes { @@ -330,10 +330,20 @@ func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object { objectFound = true } + // track the instant object iteration began, // so it can be used to bypass objects created after this instant if now.IsZero() { - now = time.Now().UTC() + if serverDate != "" { + n, err := http.ParseTime(serverDate) + if err != nil { + now = time.Now().UTC() + } else { + now = n + } + } else { + now = time.Now().UTC() + } } for _, c := range p.Contents { @@ -370,7 +380,7 @@ func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object { } return !lastPage - }) + }, request.WithGetResponseHeader("date", &serverDate)) if err != nil { objCh <- &Object{Err: err} diff --git a/storage/s3_test.go b/storage/s3_test.go index d101946b4..9372c2cdb 100644 --- a/storage/s3_test.go +++ b/storage/s3_test.go @@ -227,6 +227,13 @@ func TestS3ListURL(t *testing.T) { } mockApi.Handlers.Send.Clear() + mockApi.Handlers.Send.PushBack(func(r *request.Request) { + header := http.Header{} + r.HTTPResponse = &http.Response{ + StatusCode: http.StatusOK, + Header: header, + } + }) mockApi.Handlers.Unmarshal.Clear() mockApi.Handlers.UnmarshalMeta.Clear() mockApi.Handlers.ValidateResponse.Clear() @@ -329,6 +336,13 @@ func TestS3ListNoItemFound(t *testing.T) { } mockApi.Handlers.Send.Clear() + mockApi.Handlers.Send.PushBack(func(r *request.Request) { + header := http.Header{} + r.HTTPResponse = &http.Response{ + StatusCode: http.StatusOK, + Header: header, + } + }) mockApi.Handlers.Unmarshal.Clear() mockApi.Handlers.UnmarshalMeta.Clear() mockApi.Handlers.ValidateResponse.Clear() @@ -367,6 +381,14 @@ func TestS3ListContextCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() + mockApi.Handlers.Complete.Clear() + mockApi.Handlers.Complete.PushBack(func(r *request.Request) { + header := http.Header{} + r.HTTPResponse = &http.Response{ + StatusCode: http.StatusOK, + Header: header, + } + }) mockApi.Handlers.Unmarshal.Clear() mockApi.Handlers.UnmarshalMeta.Clear() mockApi.Handlers.ValidateResponse.Clear() @@ -1164,6 +1186,14 @@ func TestS3ListObjectsAPIVersions(t *testing.T) { mockS3 := &S3{api: mockApi} mockApi.Handlers.Send.Clear() + mockApi.Handlers.Send.PushBack(func(r *request.Request) { + header := http.Header{} + header.Set("Date", "Fri, 12 May 2023 20:36:22 GMT") + r.HTTPResponse = &http.Response{ + StatusCode: http.StatusOK, + Header: header, + } + }) mockApi.Handlers.Unmarshal.Clear() mockApi.Handlers.UnmarshalMeta.Clear() mockApi.Handlers.ValidateResponse.Clear()