From 71e698867dedbd3a8fe0204bc63c5f7763ac4c62 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 15 Sep 2024 12:36:43 +0300 Subject: [PATCH] Handle archive upload general progress bar counter incrementation (#1019) --- artifactory/services/download.go | 2 +- artifactory/services/upload.go | 24 ++++++++++++++++++------ artifactory/usage/reportusage.go | 2 +- go.mod | 10 +++++----- go.sum | 16 ++++++++-------- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/artifactory/services/download.go b/artifactory/services/download.go index 64bdc39ef..a82375dd6 100644 --- a/artifactory/services/download.go +++ b/artifactory/services/download.go @@ -577,7 +577,7 @@ func (ds *DownloadService) createFileHandlerFunc(downloadParams DownloadParams, } log.Info(fmt.Sprintf("%sDownloading %q to %q", logMsgPrefix, downloadData.Dependency.GetItemRelativePath(), localFullPath)) if err = ds.downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix, downloadData, downloadParams); err != nil { - log.Error(logMsgPrefix, "Received an error: "+err.Error()) + log.Error(logMsgPrefix + "Received an error: " + err.Error()) return err } successCounters[threadId]++ diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 250b22678..75d9a0f2e 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -858,19 +858,22 @@ func (us *UploadService) CreateUploadAsZipFunc(uploadResult *utils.Result, targe } // Make sure all go routines in readFilesAsZip calls were done. var zipReadersWg sync.WaitGroup - checksumZipReader := us.readFilesAsZip(archiveDataReader, "Calculating size / checksums", + // We execute readFilesAsZip twice. The first execution is a dry-run. + // It calculates the size and checksums of the generated zip file, which is necessary before starting the upload task. + // The second execution is part of the zip upload process running in parallel to the archiving of them files to the zip. + checksumZipReader := us.readFilesAsZip(archiveDataReader, true, archiveData.uploadParams.Flat, archiveData.uploadParams.Symlink, saveFilesPathsFunc, errorsQueue, &zipReadersWg) details, err := fileutils.GetFileDetailsFromReader(checksumZipReader, archiveData.uploadParams.ChecksumsCalcEnabled) if err != nil { return } - log.Info(logMsgPrefix+"Uploading artifact:", targetPath) - getReaderFunc := func() (io.Reader, error) { archiveDataReader.Reset() - return us.readFilesAsZip(archiveDataReader, "Archiving", archiveData.uploadParams.Flat, + return us.readFilesAsZip(archiveDataReader, false, archiveData.uploadParams.Flat, archiveData.uploadParams.Symlink, nil, errorsQueue, &zipReadersWg), nil } + + log.Info(logMsgPrefix+"Uploading artifact:", targetPath) uploaded, err := us.uploadFileFromReader(getReaderFunc, targetUrlWithProps, archiveData.uploadParams, logMsgPrefix, details) if uploaded { @@ -886,12 +889,17 @@ func (us *UploadService) CreateUploadAsZipFunc(uploadResult *utils.Result, targe // Reads files and streams them as a ZIP to a Reader. // archiveDataReader is a ContentReader of UploadData items containing the details of the files to stream. +// zipDryRun - If true, this function is run as part of the dry-run process, to // saveFilesPathsFunc (optional) is a func that is called for each file that is written into the ZIP, and gets the file's local path as a parameter. -func (us *UploadService) readFilesAsZip(archiveDataReader *content.ContentReader, progressPrefix string, flat, symlink bool, +func (us *UploadService) readFilesAsZip(archiveDataReader *content.ContentReader, zipDryRun, flat, symlink bool, saveFilesPathsFunc func(sourcePath string) error, errorsQueue *clientutils.ErrorsQueue, zipReadersWg *sync.WaitGroup) io.Reader { pr, pw := io.Pipe() zipReadersWg.Add(1) + progressPrefix := "Archiving" + if zipDryRun { + progressPrefix = "Calculating size / checksums" + } go func() { defer zipReadersWg.Done() var e error @@ -912,6 +920,10 @@ func (us *UploadService) readFilesAsZip(archiveDataReader *content.ContentReader if e != nil { errorsQueue.AddError(e) } + if !zipDryRun && us.Progress != nil { + // Increment general progress by 1 for each file added to the zip if not in dry-run mode. + us.Progress.IncrementGeneralProgress() + } } if saveFilesPathsFunc != nil { e = saveFilesPathsFunc(uploadData.Artifact.LocalPath) @@ -979,7 +991,7 @@ func (us *UploadService) addFileToZip(artifact *clientutils.Artifact, progressPr err = errors.Join(err, errorutils.CheckError(file.Close())) } }() - // Show progress bar only for files larger than 250Kb to avoid polluting the terminal with endless progress bars. + // Show progress bar only for files larger than 500kb to avoid polluting the terminal with endless progress bars. if us.Progress != nil && info.Size() > minFileSizeForProgressInKb { progressReader := us.Progress.NewProgressReader(info.Size(), progressPrefix, localPath) reader = progressReader.ActionWithProgress(file) diff --git a/artifactory/usage/reportusage.go b/artifactory/usage/reportusage.go index 7ecb32a25..bda41b225 100644 --- a/artifactory/usage/reportusage.go +++ b/artifactory/usage/reportusage.go @@ -39,7 +39,7 @@ func validateAndGetUsageServerInfo(serviceManager artifactory.ArtifactoryService err = errors.New("Couldn't get Artifactory version. Error: " + err.Error()) return } - if e := clientutils.ValidateMinimumVersion(clientutils.Artifactory, artifactoryVersion, minArtifactoryVersion); e != nil { + if err = clientutils.ValidateMinimumVersion(clientutils.Artifactory, artifactoryVersion, minArtifactoryVersion); err != nil { return } url, err = clientutils.BuildUrl(rtDetails.GetUrl(), "api/system/usage", make(map[string]string)) diff --git a/go.mod b/go.mod index 7c62e00db..5ff444439 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jfrog/jfrog-client-go -go 1.22 +go 1.22.7 require ( github.com/ProtonMail/go-crypto v1.0.0 @@ -16,7 +16,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.27.0 - golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/term v0.24.0 ) @@ -49,11 +49,11 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect - golang.org/x/mod v0.20.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.25.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 63fbaf494..a135ec067 100644 --- a/go.sum +++ b/go.sum @@ -133,12 +133,12 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= -golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -146,8 +146,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -189,8 +189,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=