From 14073d13b38d534b4ddfdae082d47c0b7cb80dcb Mon Sep 17 00:00:00 2001 From: Omer Zidkoni <50792403+omerzi@users.noreply.github.com> Date: Sun, 15 Sep 2024 08:56:23 +0300 Subject: [PATCH 1/3] Add failFast flag to UploadFiles(), UploadFilesWithSummary() --- artifactory/emptymanager.go | 13 ++++--------- artifactory/manager.go | 17 ++++------------- artifactory/services/upload.go | 9 ++------- tests/artifactorymultipartupload_test.go | 2 +- tests/artifactoryupload_test.go | 18 +++++++++--------- tests/utils_test.go | 6 +++--- 6 files changed, 23 insertions(+), 42 deletions(-) diff --git a/artifactory/emptymanager.go b/artifactory/emptymanager.go index 2060c10f2..2351ad7ca 100644 --- a/artifactory/emptymanager.go +++ b/artifactory/emptymanager.go @@ -56,9 +56,8 @@ type ArtifactoryServicesManager interface { SetProps(params services.PropsParams) (int, error) DeleteProps(params services.PropsParams) (int, error) GetItemProps(relativePath string) (*utils.ItemProperties, error) - UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) - UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) - UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) + UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) + UploadFiles(failFast bool, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) Move(params ...services.MoveCopyParams) (successCount, failedCount int, err error) PublishGoProject(params _go.GoParams) (*utils.OperationSummary, error) @@ -263,15 +262,11 @@ func (esm *EmptyArtifactoryServicesManager) GetItemProps(string) (*utils.ItemPro panic("Failed: Method is not implemented") } -func (esm *EmptyArtifactoryServicesManager) UploadFiles(...services.UploadParams) (int, int, error) { +func (esm *EmptyArtifactoryServicesManager) UploadFiles(_ bool, _ ...services.UploadParams) (int, int, error) { panic("Failed: Method is not implemented") } -func (esm *EmptyArtifactoryServicesManager) UploadFilesWithFailFast(...services.UploadParams) (int, int, error) { - panic("Failed: Method is not implemented") -} - -func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(...services.UploadParams) (*utils.OperationSummary, error) { +func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(_ bool, _ ...services.UploadParams) (*utils.OperationSummary, error) { panic("Failed: Method is not implemented") } diff --git a/artifactory/manager.go b/artifactory/manager.go index 10a785bfe..8a4fcea9b 100644 --- a/artifactory/manager.go +++ b/artifactory/manager.go @@ -313,28 +313,19 @@ func (sm *ArtifactoryServicesManagerImp) initUploadService() *services.UploadSer return uploadService } -func (sm *ArtifactoryServicesManagerImp) UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { - return sm.uploadFiles(sm.initUploadService(), params...) -} - -func (sm *ArtifactoryServicesManagerImp) UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { +func (sm *ArtifactoryServicesManagerImp) UploadFiles(failFast bool, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { uploadService := sm.initUploadService() - uploadService.SetFailFast(true) - return sm.uploadFiles(uploadService, params...) -} - -func (sm *ArtifactoryServicesManagerImp) uploadFiles(uploadService *services.UploadService, uploadParams ...services.UploadParams) (totalUploaded, totalFailed int, err error) { - summary, err := uploadService.UploadFiles(uploadParams...) + summary, err := uploadService.UploadFiles(failFast, params...) if summary == nil { return 0, 0, err } return summary.TotalSucceeded, summary.TotalFailed, err } -func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) { +func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) { uploadService := sm.initUploadService() uploadService.SetSaveSummary(true) - return uploadService.UploadFiles(params...) + return uploadService.UploadFiles(failFast, params...) } func (sm *ArtifactoryServicesManagerImp) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) { diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 250b22678..864ede094 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -47,7 +47,6 @@ type UploadService struct { MultipartUpload *utils.MultipartUpload DryRun bool saveSummary bool - failFast bool Threads int resultsManager *resultsManager } @@ -78,10 +77,6 @@ func (us *UploadService) SetSaveSummary(saveSummary bool) { us.saveSummary = saveSummary } -func (us *UploadService) SetFailFast(failFast bool) { - us.failFast = failFast -} - func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *utils.OperationSummary { if !us.saveSummary { return &utils.OperationSummary{ @@ -92,10 +87,10 @@ func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *u return us.resultsManager.getOperationSummary(totalSucceeded, totalFailed) } -func (us *UploadService) UploadFiles(uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) { +func (us *UploadService) UploadFiles(failFast bool, uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) { // Uploading threads are using this struct to report upload results. uploadSummary := utils.NewResult(us.Threads) - producerConsumer := parallel.NewRunner(us.Threads, 20000, us.failFast) + producerConsumer := parallel.NewRunner(us.Threads, 20000, failFast) errorsQueue := clientutils.NewErrorsQueue(1) if us.saveSummary { us.resultsManager, err = newResultManager() diff --git a/tests/artifactorymultipartupload_test.go b/tests/artifactorymultipartupload_test.go index 6529d2424..359fad8ae 100644 --- a/tests/artifactorymultipartupload_test.go +++ b/tests/artifactorymultipartupload_test.go @@ -48,7 +48,7 @@ func multipartUpload(t *testing.T) { up.MinSplitSize = bigFileSize // Upload file and verify success - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) assert.NoError(t, err) assert.Equal(t, 1, summary.TotalSucceeded) assert.Zero(t, summary.TotalFailed) diff --git a/tests/artifactoryupload_test.go b/tests/artifactoryupload_test.go index 7f6dcf071..4eefd5f04 100644 --- a/tests/artifactoryupload_test.go +++ b/tests/artifactoryupload_test.go @@ -45,7 +45,7 @@ func flatUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo()} up.Flat = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if err != nil { t.Error(err) } @@ -85,7 +85,7 @@ func recursiveUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo()} up.Flat = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if err != nil { t.Error(err) } @@ -128,7 +128,7 @@ func placeholderUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "{1}"} up.Flat = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if err != nil { t.Error(err) } @@ -171,7 +171,7 @@ func includeDirsUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, IncludeDirs: true, Recursive: false, Target: getRtTargetRepo()} up.Flat = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if err != nil { t.Error(err) } @@ -227,7 +227,7 @@ func explodeUpload(t *testing.T) { up.CommonParams = &utils.CommonParams{Pattern: pattern, IncludeDirs: true, Recursive: false, Target: getRtTargetRepo()} up.Flat = true up.ExplodeArchive = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if err != nil { t.Error(err) } @@ -274,7 +274,7 @@ func propsUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Target: getRtTargetRepo(), TargetProps: targetProps} up.Flat = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) assert.NoError(t, err) assert.Equal(t, 1, summary.TotalSucceeded) assert.Equal(t, 0, summary.TotalFailed) @@ -311,7 +311,7 @@ func summaryUpload(t *testing.T) { up.Flat = true testsUploadService.SetSaveSummary(true) defer testsUploadService.SetSaveSummary(false) - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if err != nil { t.Error(err) } @@ -352,7 +352,7 @@ func archiveUpload(t *testing.T) { up.CommonParams = &utils.CommonParams{Pattern: uploadPattern, Target: downloadPattern, TargetProps: targetProps} up.Archive = "zip" up.Flat = true - _, err = testsUploadService.UploadFiles(up) + _, err = testsUploadService.UploadFiles(false, up) assert.NoError(t, err) // Download zip @@ -424,7 +424,7 @@ func TestUploadFilesWithFailure(t *testing.T) { service.SetServiceDetails(rtDetails) // Upload files - summary, err := service.UploadFiles(params) + summary, err := service.UploadFiles(false, params) // Check for expected results assert.Error(t, err) diff --git a/tests/utils_test.go b/tests/utils_test.go index 8c338017e..c7610fa28 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -631,7 +631,7 @@ func uploadDummyFile(t *testing.T) { } up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "test/", TargetProps: targetProps} up.Flat = true - summary, err := testsUploadService.UploadFiles(up) + summary, err := testsUploadService.UploadFiles(false, up) if summary.TotalSucceeded != 1 { t.Error("Expected to upload 1 file.") } @@ -643,7 +643,7 @@ func uploadDummyFile(t *testing.T) { } up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "b.in"} up.Flat = true - summary, err = testsUploadService.UploadFiles(up) + summary, err = testsUploadService.UploadFiles(false, up) assert.NoError(t, err) if summary.TotalSucceeded != 1 { t.Error("Expected to upload 1 file.") @@ -658,7 +658,7 @@ func uploadDummyFile(t *testing.T) { } up.CommonParams = &utils.CommonParams{Pattern: archivePath, Recursive: true, Target: getRtTargetRepo()} up.Flat = true - summary, err = testsUploadService.UploadFiles(up) + summary, err = testsUploadService.UploadFiles(false, up) if summary.TotalSucceeded != 1 { t.Error("Expected to upload 1 file.") } From c24c54aacfd748a3ef260c4bdcc9260e27ac22b3 Mon Sep 17 00:00:00 2001 From: Omer Zidkoni <50792403+omerzi@users.noreply.github.com> Date: Sun, 15 Sep 2024 08:57:47 +0300 Subject: [PATCH 2/3] README --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16f6cba66..dc62bf74b 100644 --- a/README.md +++ b/README.md @@ -470,7 +470,9 @@ params.TargetProps = targetProps TargetPathInArchive := "archive/path/" // Size limit for files to be uploaded. SizeLimit= &fspatterns.SizeThreshold{SizeInBytes: 10000, Condition: fspatterns.LessThan} -totalUploaded, totalFailed, err := rtManager.UploadFiles(params) +// Set to true to fail the upload operation if any of the files fail to upload +failFast := false +totalUploaded, totalFailed, err := rtManager.UploadFiles(failFast, params) ``` #### Downloading Files from Artifactory @@ -547,8 +549,10 @@ calling `Close()` on the OperationSummary struct. params := services.NewUploadParams() params.Pattern = "repo/*/*.zip" params.Target = "repo/path/" +// Set to true to fail the upload operation if any of the files fail to upload +failFast := false -summary, err := rtManager.UploadFilesWithSummary(params) +summary, err := rtManager.UploadFilesWithSummary(failFast, params) defer summary.Close() reader, totalDownloaded, totalExpected, err := rtManager.DownloadFilesWithResultReader(params) From 0155ae788a84fa29ce5d45d110da7d74d1461dbb Mon Sep 17 00:00:00 2001 From: Omer Zidkoni <50792403+omerzi@users.noreply.github.com> Date: Sun, 15 Sep 2024 12:05:01 +0300 Subject: [PATCH 3/3] CR --- artifactory/emptymanager.go | 2 +- artifactory/manager.go | 6 ++++-- artifactory/services/upload.go | 9 +++++++-- tests/artifactorymultipartupload_test.go | 2 +- tests/artifactoryupload_test.go | 18 +++++++++--------- tests/utils_test.go | 6 +++--- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/artifactory/emptymanager.go b/artifactory/emptymanager.go index 2351ad7ca..eac95735d 100644 --- a/artifactory/emptymanager.go +++ b/artifactory/emptymanager.go @@ -56,8 +56,8 @@ type ArtifactoryServicesManager interface { SetProps(params services.PropsParams) (int, error) DeleteProps(params services.PropsParams) (int, error) GetItemProps(relativePath string) (*utils.ItemProperties, error) - UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) UploadFiles(failFast bool, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) + UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) Move(params ...services.MoveCopyParams) (successCount, failedCount int, err error) PublishGoProject(params _go.GoParams) (*utils.OperationSummary, error) diff --git a/artifactory/manager.go b/artifactory/manager.go index 8a4fcea9b..ff48a5bf4 100644 --- a/artifactory/manager.go +++ b/artifactory/manager.go @@ -315,7 +315,8 @@ func (sm *ArtifactoryServicesManagerImp) initUploadService() *services.UploadSer func (sm *ArtifactoryServicesManagerImp) UploadFiles(failFast bool, params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { uploadService := sm.initUploadService() - summary, err := uploadService.UploadFiles(failFast, params...) + uploadService.SetFailFast(failFast) + summary, err := uploadService.UploadFiles(params...) if summary == nil { return 0, 0, err } @@ -325,7 +326,8 @@ func (sm *ArtifactoryServicesManagerImp) UploadFiles(failFast bool, params ...se func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(failFast bool, params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) { uploadService := sm.initUploadService() uploadService.SetSaveSummary(true) - return uploadService.UploadFiles(failFast, params...) + uploadService.SetFailFast(failFast) + return uploadService.UploadFiles(params...) } func (sm *ArtifactoryServicesManagerImp) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) { diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 864ede094..250b22678 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -47,6 +47,7 @@ type UploadService struct { MultipartUpload *utils.MultipartUpload DryRun bool saveSummary bool + failFast bool Threads int resultsManager *resultsManager } @@ -77,6 +78,10 @@ func (us *UploadService) SetSaveSummary(saveSummary bool) { us.saveSummary = saveSummary } +func (us *UploadService) SetFailFast(failFast bool) { + us.failFast = failFast +} + func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *utils.OperationSummary { if !us.saveSummary { return &utils.OperationSummary{ @@ -87,10 +92,10 @@ func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *u return us.resultsManager.getOperationSummary(totalSucceeded, totalFailed) } -func (us *UploadService) UploadFiles(failFast bool, uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) { +func (us *UploadService) UploadFiles(uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) { // Uploading threads are using this struct to report upload results. uploadSummary := utils.NewResult(us.Threads) - producerConsumer := parallel.NewRunner(us.Threads, 20000, failFast) + producerConsumer := parallel.NewRunner(us.Threads, 20000, us.failFast) errorsQueue := clientutils.NewErrorsQueue(1) if us.saveSummary { us.resultsManager, err = newResultManager() diff --git a/tests/artifactorymultipartupload_test.go b/tests/artifactorymultipartupload_test.go index 359fad8ae..6529d2424 100644 --- a/tests/artifactorymultipartupload_test.go +++ b/tests/artifactorymultipartupload_test.go @@ -48,7 +48,7 @@ func multipartUpload(t *testing.T) { up.MinSplitSize = bigFileSize // Upload file and verify success - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) assert.NoError(t, err) assert.Equal(t, 1, summary.TotalSucceeded) assert.Zero(t, summary.TotalFailed) diff --git a/tests/artifactoryupload_test.go b/tests/artifactoryupload_test.go index 4eefd5f04..7f6dcf071 100644 --- a/tests/artifactoryupload_test.go +++ b/tests/artifactoryupload_test.go @@ -45,7 +45,7 @@ func flatUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo()} up.Flat = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if err != nil { t.Error(err) } @@ -85,7 +85,7 @@ func recursiveUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo()} up.Flat = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if err != nil { t.Error(err) } @@ -128,7 +128,7 @@ func placeholderUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "{1}"} up.Flat = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if err != nil { t.Error(err) } @@ -171,7 +171,7 @@ func includeDirsUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, IncludeDirs: true, Recursive: false, Target: getRtTargetRepo()} up.Flat = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if err != nil { t.Error(err) } @@ -227,7 +227,7 @@ func explodeUpload(t *testing.T) { up.CommonParams = &utils.CommonParams{Pattern: pattern, IncludeDirs: true, Recursive: false, Target: getRtTargetRepo()} up.Flat = true up.ExplodeArchive = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if err != nil { t.Error(err) } @@ -274,7 +274,7 @@ func propsUpload(t *testing.T) { up := services.NewUploadParams() up.CommonParams = &utils.CommonParams{Pattern: pattern, Target: getRtTargetRepo(), TargetProps: targetProps} up.Flat = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) assert.NoError(t, err) assert.Equal(t, 1, summary.TotalSucceeded) assert.Equal(t, 0, summary.TotalFailed) @@ -311,7 +311,7 @@ func summaryUpload(t *testing.T) { up.Flat = true testsUploadService.SetSaveSummary(true) defer testsUploadService.SetSaveSummary(false) - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if err != nil { t.Error(err) } @@ -352,7 +352,7 @@ func archiveUpload(t *testing.T) { up.CommonParams = &utils.CommonParams{Pattern: uploadPattern, Target: downloadPattern, TargetProps: targetProps} up.Archive = "zip" up.Flat = true - _, err = testsUploadService.UploadFiles(false, up) + _, err = testsUploadService.UploadFiles(up) assert.NoError(t, err) // Download zip @@ -424,7 +424,7 @@ func TestUploadFilesWithFailure(t *testing.T) { service.SetServiceDetails(rtDetails) // Upload files - summary, err := service.UploadFiles(false, params) + summary, err := service.UploadFiles(params) // Check for expected results assert.Error(t, err) diff --git a/tests/utils_test.go b/tests/utils_test.go index c7610fa28..8c338017e 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -631,7 +631,7 @@ func uploadDummyFile(t *testing.T) { } up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "test/", TargetProps: targetProps} up.Flat = true - summary, err := testsUploadService.UploadFiles(false, up) + summary, err := testsUploadService.UploadFiles(up) if summary.TotalSucceeded != 1 { t.Error("Expected to upload 1 file.") } @@ -643,7 +643,7 @@ func uploadDummyFile(t *testing.T) { } up.CommonParams = &utils.CommonParams{Pattern: pattern, Recursive: true, Target: getRtTargetRepo() + "b.in"} up.Flat = true - summary, err = testsUploadService.UploadFiles(false, up) + summary, err = testsUploadService.UploadFiles(up) assert.NoError(t, err) if summary.TotalSucceeded != 1 { t.Error("Expected to upload 1 file.") @@ -658,7 +658,7 @@ func uploadDummyFile(t *testing.T) { } up.CommonParams = &utils.CommonParams{Pattern: archivePath, Recursive: true, Target: getRtTargetRepo()} up.Flat = true - summary, err = testsUploadService.UploadFiles(false, up) + summary, err = testsUploadService.UploadFiles(up) if summary.TotalSucceeded != 1 { t.Error("Expected to upload 1 file.") }