Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add failFast flag to UploadFiles(), UploadFilesWithSummary() #1021

Merged
merged 4 commits into from
Sep 16, 2024
Merged

Add failFast flag to UploadFiles(), UploadFilesWithSummary() #1021

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
14073d1
Add failFast flag to UploadFiles(), UploadFilesWithSummary()
omerzi Sep 15, 2024
c24c54a
README
omerzi Sep 15, 2024
0155ae7
CR
omerzi Sep 15, 2024
46bbfd8
Merge branch 'dev' into failfast-uploadfiles
omerzi Sep 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,9 @@ params.TargetProps = targetProps
TargetPathInArchive := "archive/path/"
// Size limit for files to be uploaded.
SizeLimit= &fspatterns.SizeThreshold{SizeInBytes: 10000, Condition: fspatterns.LessThan}
totalUploaded, totalFailed, err := rtManager.UploadFiles(params)
// Set to true to fail the upload operation if any of the files fail to upload
failFast := false
totalUploaded, totalFailed, err := rtManager.UploadFiles(failFast, params)
```

#### Downloading Files from Artifactory
Expand Down Expand Up @@ -547,8 +549,10 @@ calling `Close()` on the OperationSummary struct.
params := services.NewUploadParams()
params.Pattern = "repo/*/*.zip"
params.Target = "repo/path/"
// Set to true to fail the upload operation if any of the files fail to upload
failFast := false

summary, err := rtManager.UploadFilesWithSummary(params)
summary, err := rtManager.UploadFilesWithSummary(failFast, params)
defer summary.Close()
reader, totalDownloaded, totalExpected, err := rtManager.DownloadFilesWithResultReader(params)

Expand Down
13 changes: 4 additions & 9 deletions artifactory/emptymanager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,8 @@ type ArtifactoryServicesManager interface {
SetProps(params services.PropsParams) (int, error)
DeleteProps(params services.PropsParams) (int, error)
GetItemProps(relativePath string) (*utils.ItemProperties, error)
UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error)
UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error)
UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error)
UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error)
UploadFiles(failFast bool, params ...services.UploadParams) (totalUploaded, totalFailed int, err error)
omerzi marked this conversation as resolved.
Show resolved Hide resolved
Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error)
Move(params ...services.MoveCopyParams) (successCount, failedCount int, err error)
PublishGoProject(params _go.GoParams) (*utils.OperationSummary, error)
Expand Down Expand Up @@ -263,15 +262,11 @@ func (esm *EmptyArtifactoryServicesManager) GetItemProps(string) (*utils.ItemPro
panic("Failed: Method is not implemented")
}

func (esm *EmptyArtifactoryServicesManager) UploadFiles(...services.UploadParams) (int, int, error) {
func (esm *EmptyArtifactoryServicesManager) UploadFiles(_ bool, _ ...services.UploadParams) (int, int, error) {
panic("Failed: Method is not implemented")
}

func (esm *EmptyArtifactoryServicesManager) UploadFilesWithFailFast(...services.UploadParams) (int, int, error) {
panic("Failed: Method is not implemented")
}

func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(...services.UploadParams) (*utils.OperationSummary, error) {
func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(_ bool, _ ...services.UploadParams) (*utils.OperationSummary, error) {
panic("Failed: Method is not implemented")
}

Expand Down
17 changes: 4 additions & 13 deletions artifactory/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,28 +313,19 @@ func (sm *ArtifactoryServicesManagerImp) initUploadService() *services.UploadSer
return uploadService
}

func (sm *ArtifactoryServicesManagerImp) UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) {
return sm.uploadFiles(sm.initUploadService(), params...)
}

func (sm *ArtifactoryServicesManagerImp) UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) {
func (sm *ArtifactoryServicesManagerImp) UploadFiles(failFast bool, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) {
uploadService := sm.initUploadService()
uploadService.SetFailFast(true)
return sm.uploadFiles(uploadService, params...)
}

func (sm *ArtifactoryServicesManagerImp) uploadFiles(uploadService *services.UploadService, uploadParams ...services.UploadParams) (totalUploaded, totalFailed int, err error) {
summary, err := uploadService.UploadFiles(uploadParams...)
summary, err := uploadService.UploadFiles(failFast, params...)
if summary == nil {
return 0, 0, err
}
return summary.TotalSucceeded, summary.TotalFailed, err
}

func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) {
func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) {
uploadService := sm.initUploadService()
uploadService.SetSaveSummary(true)
return uploadService.UploadFiles(params...)
return uploadService.UploadFiles(failFast, params...)
omerzi marked this conversation as resolved.
Show resolved Hide resolved
}

func (sm *ArtifactoryServicesManagerImp) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) {
Expand Down
9 changes: 2 additions & 7 deletions artifactory/services/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ type UploadService struct {
MultipartUpload *utils.MultipartUpload
DryRun bool
saveSummary bool
failFast bool
Threads int
resultsManager *resultsManager
}
Expand Down Expand Up @@ -78,10 +77,6 @@ func (us *UploadService) SetSaveSummary(saveSummary bool) {
us.saveSummary = saveSummary
}

func (us *UploadService) SetFailFast(failFast bool) {
us.failFast = failFast
}

func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *utils.OperationSummary {
if !us.saveSummary {
return &utils.OperationSummary{
Expand All @@ -92,10 +87,10 @@ func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *u
return us.resultsManager.getOperationSummary(totalSucceeded, totalFailed)
}

func (us *UploadService) UploadFiles(uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) {
func (us *UploadService) UploadFiles(failFast bool, uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) {
// Uploading threads are using this struct to report upload results.
uploadSummary := utils.NewResult(us.Threads)
producerConsumer := parallel.NewRunner(us.Threads, 20000, us.failFast)
producerConsumer := parallel.NewRunner(us.Threads, 20000, failFast)
errorsQueue := clientutils.NewErrorsQueue(1)
if us.saveSummary {
us.resultsManager, err = newResultManager()
Expand Down
2 changes: 1 addition & 1 deletion tests/artifactorymultipartupload_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func multipartUpload(t *testing.T) {
up.MinSplitSize = bigFileSize

// Upload file and verify success
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
assert.NoError(t, err)
assert.Equal(t, 1, summary.TotalSucceeded)
assert.Zero(t, summary.TotalFailed)
Expand Down
18 changes: 9 additions & 9 deletions tests/artifactoryupload_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func flatUpload(t *testing.T) {
up := services.NewUploadParams()
up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo()}
up.Flat = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -85,7 +85,7 @@ func recursiveUpload(t *testing.T) {
up := services.NewUploadParams()
up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo()}
up.Flat = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -128,7 +128,7 @@ func placeholderUpload(t *testing.T) {
up := services.NewUploadParams()
up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "{1}"}
up.Flat = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -171,7 +171,7 @@ func includeDirsUpload(t *testing.T) {
up := services.NewUploadParams()
up.CommonParams = &utils.CommonParams{Pattern: pattern, IncludeDirs: true, Recursive: false, Target: getRtTargetRepo()}
up.Flat = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -227,7 +227,7 @@ func explodeUpload(t *testing.T) {
up.CommonParams = &utils.CommonParams{Pattern: pattern, IncludeDirs: true, Recursive: false, Target: getRtTargetRepo()}
up.Flat = true
up.ExplodeArchive = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -274,7 +274,7 @@ func propsUpload(t *testing.T) {
up := services.NewUploadParams()
up.CommonParams = &utils.CommonParams{Pattern: pattern, Target: getRtTargetRepo(), TargetProps: targetProps}
up.Flat = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
assert.NoError(t, err)
assert.Equal(t, 1, summary.TotalSucceeded)
assert.Equal(t, 0, summary.TotalFailed)
Expand Down Expand Up @@ -311,7 +311,7 @@ func summaryUpload(t *testing.T) {
up.Flat = true
testsUploadService.SetSaveSummary(true)
defer testsUploadService.SetSaveSummary(false)
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -352,7 +352,7 @@ func archiveUpload(t *testing.T) {
up.CommonParams = &utils.CommonParams{Pattern: uploadPattern, Target: downloadPattern, TargetProps: targetProps}
up.Archive = "zip"
up.Flat = true
_, err = testsUploadService.UploadFiles(up)
_, err = testsUploadService.UploadFiles(false, up)
assert.NoError(t, err)

// Download zip
Expand Down Expand Up @@ -424,7 +424,7 @@ func TestUploadFilesWithFailure(t *testing.T) {
service.SetServiceDetails(rtDetails)

// Upload files
summary, err := service.UploadFiles(params)
summary, err := service.UploadFiles(false, params)

// Check for expected results
assert.Error(t, err)
Expand Down
6 changes: 3 additions & 3 deletions tests/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ func uploadDummyFile(t *testing.T) {
}
up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "test/", TargetProps: targetProps}
up.Flat = true
summary, err := testsUploadService.UploadFiles(up)
summary, err := testsUploadService.UploadFiles(false, up)
if summary.TotalSucceeded != 1 {
t.Error("Expected to upload 1 file.")
}
Expand All @@ -643,7 +643,7 @@ func uploadDummyFile(t *testing.T) {
}
up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "b.in"}
up.Flat = true
summary, err = testsUploadService.UploadFiles(up)
summary, err = testsUploadService.UploadFiles(false, up)
assert.NoError(t, err)
if summary.TotalSucceeded != 1 {
t.Error("Expected to upload 1 file.")
Expand All @@ -658,7 +658,7 @@ func uploadDummyFile(t *testing.T) {
}
up.CommonParams = &utils.CommonParams{Pattern: archivePath, Recursive: true, Target: getRtTargetRepo()}
up.Flat = true
summary, err = testsUploadService.UploadFiles(up)
summary, err = testsUploadService.UploadFiles(false, up)
if summary.TotalSucceeded != 1 {
t.Error("Expected to upload 1 file.")
}
Expand Down
Loading