diff --git a/artifactory/commands/transferfiles/state/runstatus.go b/artifactory/commands/transferfiles/state/runstatus.go index dd03a88f6..5454aa8bb 100644 --- a/artifactory/commands/transferfiles/state/runstatus.go +++ b/artifactory/commands/transferfiles/state/runstatus.go @@ -24,6 +24,8 @@ type ActionOnStatusFunc func(transferRunStatus *TransferRunStatus) error // This state is used to allow showing the current run status by the 'jf rt transfer-files --status' command. // It is also used for the time estimation and more. type TransferRunStatus struct { + // Timestamp since the beginning of the current transfer execution + startTimestamp time.Time lastSaveTimestamp time.Time // This variable holds the total/transferred number of repositories (not their files). OverallTransfer ProgressState `json:"overall_transfer,omitempty"` diff --git a/artifactory/commands/transferfiles/state/statemanager.go b/artifactory/commands/transferfiles/state/statemanager.go index 303d5001f..273c9d28b 100644 --- a/artifactory/commands/transferfiles/state/statemanager.go +++ b/artifactory/commands/transferfiles/state/statemanager.go @@ -61,6 +61,14 @@ func (ts *TransferStateManager) UnlockTransferStateManager() error { return ts.unlockStateManager() } +func (ts *TransferStateManager) SetStartTimestamp(startTimestamp time.Time) { + ts.startTimestamp = startTimestamp +} + +func (ts *TransferStateManager) GetStartTimestamp() time.Time { + return ts.startTimestamp +} + // Set the repository state. // repoKey - Repository key // totalSizeBytes - Repository size in bytes @@ -394,21 +402,39 @@ func (ts *TransferStateManager) tryLockStateManager() error { return nil } -func getStartTimestamp() (int64, error) { +func (ts *TransferStateManager) Running() (running bool, err error) { lockDirPath, err := coreutils.GetJfrogTransferLockDir() if err != nil { - return 0, err + return false, err } - return lock.GetLastLockTimestamp(lockDirPath) + var startTimestamp int64 + startTimestamp, err = lock.GetLastLockTimestamp(lockDirPath) + return err == nil && startTimestamp != 0, err } -func GetRunningTime() (runningTime string, isRunning bool, err error) { - startTimestamp, err := getStartTimestamp() +func (ts *TransferStateManager) InitStartTimestamp() (running bool, err error) { + if !ts.startTimestamp.IsZero() { + return true, nil + } + lockDirPath, err := coreutils.GetJfrogTransferLockDir() + if err != nil { + return false, err + } + var startTimestamp int64 + startTimestamp, err = lock.GetLastLockTimestamp(lockDirPath) if err != nil || startTimestamp == 0 { - return + return false, err + } + ts.startTimestamp = time.Unix(0, startTimestamp) + return true, nil +} + +func (ts *TransferStateManager) GetRunningTimeString() (runningTime string) { + if ts.startTimestamp.IsZero() { + return "" } - runningSecs := int64(time.Since(time.Unix(0, startTimestamp)).Seconds()) - return SecondsToLiteralTime(runningSecs, ""), true, nil + runningSecs := int64(time.Since(ts.startTimestamp).Seconds()) + return SecondsToLiteralTime(runningSecs, "") } func UpdateChunkInState(stateManager *TransferStateManager, chunk *api.ChunkStatus) (err error) { diff --git a/artifactory/commands/transferfiles/state/statemanager_test.go b/artifactory/commands/transferfiles/state/statemanager_test.go index 292fad5dd..62db05c9e 100644 --- a/artifactory/commands/transferfiles/state/statemanager_test.go +++ b/artifactory/commands/transferfiles/state/statemanager_test.go @@ -235,3 +235,78 @@ func TestTryLockStateManager(t *testing.T) { assert.NoError(t, stateManager.tryLockStateManager()) assert.ErrorIs(t, new(AlreadyLockedError), stateManager.tryLockStateManager()) } + +func TestRunning(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + + // Assert no running=false + running, err := stateManager.Running() + assert.NoError(t, err) + assert.False(t, running) + + // Lock to simulate transfer + assert.NoError(t, stateManager.TryLockTransferStateManager()) + + // Assert running=true + running, err = stateManager.Running() + assert.NoError(t, err) + assert.True(t, running) +} + +func TestInitStartTimestamp(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + + // Init start timestamp and expect timestamp zero + running, err := stateManager.InitStartTimestamp() + assert.NoError(t, err) + assert.False(t, running) + assert.True(t, stateManager.startTimestamp.IsZero()) + + // Lock to simulate transfer + assert.NoError(t, stateManager.TryLockTransferStateManager()) + + // Init start timestamp and expect timestamp non-zero + running, err = stateManager.InitStartTimestamp() + assert.NoError(t, err) + assert.True(t, running) + assert.False(t, stateManager.startTimestamp.IsZero()) +} + +var getRunningTimeStringCases = []struct { + startTimestamp time.Time + expectedString string +}{ + {time.Now(), "Less than a minute"}, + {time.Now().Add(-time.Second), "Less than a minute"}, + {time.Now().Add(-time.Minute), "1 minute"}, + {time.Now().Add(-time.Hour), "1 hour"}, + {time.Now().Add(-time.Hour).Add(time.Minute), "59 minutes"}, + {time.Now().Add(-time.Hour).Add(time.Minute).Add(10 * time.Second), "58 minutes"}, +} + +func TestGetRunningTimeString(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + + runningTime := stateManager.GetRunningTimeString() + assert.Empty(t, runningTime) + + // Lock and init start timestamp to simulate transfer + assert.NoError(t, stateManager.TryLockTransferStateManager()) + running, err := stateManager.InitStartTimestamp() + assert.NoError(t, err) + assert.True(t, running) + + // Run test cases + for _, testCase := range getRunningTimeStringCases { + t.Run(testCase.startTimestamp.String(), func(t *testing.T) { + // Set start timestamp + stateManager.startTimestamp = testCase.startTimestamp + + // Assert running time string + assert.Equal(t, testCase.expectedString, stateManager.GetRunningTimeString()) + }) + } +} diff --git a/artifactory/commands/transferfiles/state/timeestimation.go b/artifactory/commands/transferfiles/state/timeestimation.go index 77dc88d7c..d317dcd53 100644 --- a/artifactory/commands/transferfiles/state/timeestimation.go +++ b/artifactory/commands/transferfiles/state/timeestimation.go @@ -2,21 +2,18 @@ package state import ( "fmt" + "time" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-client-go/utils/log" ) const ( - milliSecsInSecond = 1000 - bytesInMB = 1024 * 1024 - bytesPerMilliSecToMBPerSec = float64(milliSecsInSecond) / float64(bytesInMB) - // Precalculated average index time per build info, in seconds. This constant is used to estimate the processing time of all - // the build info files about to be transferred. Since the build info indexing time is not related to its file size, - // the estimation approach we use with data is irrelevant. - buildInfoAverageIndexTimeSec = 1.25 + milliSecsInSecond = 1000 + bytesInMB = 1024 * 1024 + bytesPerMilliSecToMBPerSec = float64(milliSecsInSecond) / float64(bytesInMB) + minTransferTimeToShowEstimation = time.Minute * 5 ) type timeTypeSingular string @@ -36,15 +33,14 @@ type TimeEstimationManager struct { LastSpeedsSum float64 `json:"last_speeds_sum,omitempty"` // The last calculated sum of speeds, in bytes/ms SpeedsAverage float64 `json:"speeds_average,omitempty"` - // Data estimated remaining time is saved so that it can be used when handling a build-info repository and speed cannot be calculated. - DataEstimatedRemainingTime int64 `json:"data_estimated_remaining_time,omitempty"` + // Total transferred bytes since the beginning of the current transfer execution + CurrentTotalTransferredBytes uint64 `json:"current_total_transferred_bytes,omitempty"` // The state manager stateManager *TransferStateManager } func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) { - // Build info repository requires no action here (transferred counter is updated in the state manager and no other calculation is needed). - if durationMillis == 0 || tem.stateManager.BuildInfoRepo { + if durationMillis == 0 { return } @@ -54,7 +50,10 @@ func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, du func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) { var chunkSizeBytes int64 for _, file := range chunkStatus.Files { - if file.Status == api.Success && !file.ChecksumDeployed { + if file.Status != api.Fail { + tem.CurrentTotalTransferredBytes += uint64(file.SizeBytes) + } + if (file.Status == api.Success || file.Status == api.SkippedLargeProps) && !file.ChecksumDeployed { chunkSizeBytes += file.SizeBytes } } @@ -96,103 +95,44 @@ func (tem *TimeEstimationManager) getSpeed() float64 { return tem.SpeedsAverage * bytesPerMilliSecToMBPerSec } -// GetSpeedString gets the transfer speed in an easy-to-read string. +// GetSpeedString gets the transfer speed as an easy-to-read string. func (tem *TimeEstimationManager) GetSpeedString() string { - if tem.stateManager.BuildInfoRepo { - return "Not available while transferring a build-info repository" - } if len(tem.LastSpeeds) == 0 { return "Not available yet" } return fmt.Sprintf("%.3f MB/s", tem.getSpeed()) } -// getEstimatedRemainingTime gets the estimated remaining time in seconds. -// The estimated remaining time is the sum of: -// 1. Data estimated remaining time, derived by the average speed and remaining data size. -// 2. Build info estimated remaining time, derived by a precalculated average time per build info. -func (tem *TimeEstimationManager) getEstimatedRemainingTime() (int64, error) { - err := tem.calculateDataEstimatedRemainingTime() - if err != nil { - return 0, err - } - return tem.DataEstimatedRemainingTime + tem.getBuildInfoEstimatedRemainingTime(), nil -} - -// calculateDataEstimatedRemainingTime calculates the data estimated remaining time in seconds, and sets it to the corresponding -// variable in the estimation manager. -func (tem *TimeEstimationManager) calculateDataEstimatedRemainingTime() error { - // If a build info repository is currently being handled, use the data estimated time previously calculated. - // Else, start calculating when the speeds average is set. - if tem.stateManager.BuildInfoRepo || tem.SpeedsAverage == 0 { - return nil - } - transferredSizeBytes, err := tem.stateManager.GetTransferredSizeBytes() - if err != nil { - return err - } - - // In case we reach a situation where we transfer more data than expected, we cannot estimate how long transferring the remaining data will take. - if tem.stateManager.OverallTransfer.TotalSizeBytes <= transferredSizeBytes { - tem.DataEstimatedRemainingTime = 0 - return nil +// GetEstimatedRemainingTimeString gets the estimated remaining time as an easy-to-read string. +// Return "Not available yet" in the following cases: +// 1. 5 minutes not passed since the beginning of the transfer +// 2. No files transferred +// 3. The transfer speed is less than 1 byte per second +func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() string { + remainingTimeSec := tem.getEstimatedRemainingSeconds() + if remainingTimeSec == 0 { + return "Not available yet" } - // We only convert to int64 at the end to avoid a scenario where the conversion of SpeedsAverage returns zero. - remainingTime := float64(tem.stateManager.OverallTransfer.TotalSizeBytes-transferredSizeBytes) / tem.SpeedsAverage - // Convert from milliseconds to seconds. - tem.DataEstimatedRemainingTime = int64(remainingTime) / milliSecsInSecond - return nil + return SecondsToLiteralTime(int64(remainingTimeSec), "About ") } -func (tem *TimeEstimationManager) getBuildInfoEstimatedRemainingTime() int64 { - if tem.stateManager.OverallBiFiles.TotalUnits <= tem.stateManager.OverallBiFiles.TransferredUnits { +func (tem *TimeEstimationManager) getEstimatedRemainingSeconds() uint64 { + if tem.CurrentTotalTransferredBytes == 0 { + // No files transferred return 0 } - - workingThreads, err := tem.getWorkingThreadsForBuildInfoEstimation() - if err != nil { - log.Error("Couldn't calculate time estimation:", err.Error()) + duration := time.Since(tem.stateManager.startTimestamp) + if duration < minTransferTimeToShowEstimation { + // 5 minutes not yet passed return 0 } - remainingBiFiles := float64(tem.stateManager.OverallBiFiles.TotalUnits - tem.stateManager.OverallBiFiles.TransferredUnits) - remainingTime := remainingBiFiles * buildInfoAverageIndexTimeSec / float64(workingThreads) - return int64(remainingTime) -} - -func (tem *TimeEstimationManager) getWorkingThreadsForBuildInfoEstimation() (int, error) { - workingThreads, err := tem.stateManager.GetWorkingThreads() - if err != nil { - return 0, err - } - // If the uploader didn't start working, temporarily display estimation according to one thread. - if workingThreads == 0 { - return 1, nil - } - // If currently handling a data repository and the number of threads is high, show build info estimation according to the build info threads limit. - if workingThreads > utils.MaxBuildInfoThreads { - return utils.MaxBuildInfoThreads, nil - } - return workingThreads, nil -} - -// GetEstimatedRemainingTimeString gets the estimated remaining time in an easy-to-read string. -func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() string { - if !tem.isTimeEstimationAvailable() { - return "Not available in this phase" - } - if !tem.stateManager.BuildInfoRepo && len(tem.LastSpeeds) == 0 { - return "Not available yet" - } - remainingTimeSec, err := tem.getEstimatedRemainingTime() - if err != nil { - return err.Error() + transferredBytesInSeconds := tem.CurrentTotalTransferredBytes / uint64(duration.Seconds()) + if transferredBytesInSeconds == 0 { + // Less than 1 byte per second + return 0 } - - return SecondsToLiteralTime(remainingTimeSec, "About ") -} - -func (tem *TimeEstimationManager) isTimeEstimationAvailable() bool { - return tem.stateManager.CurrentRepoPhase == api.Phase1 || tem.stateManager.CurrentRepoPhase == api.Phase3 + remainingBytes := tem.stateManager.OverallTransfer.TotalSizeBytes - tem.stateManager.OverallTransfer.TransferredSizeBytes + return uint64(remainingBytes) / transferredBytesInSeconds } diff --git a/artifactory/commands/transferfiles/state/timeestimation_test.go b/artifactory/commands/transferfiles/state/timeestimation_test.go index a199c0090..e4a5da1c4 100644 --- a/artifactory/commands/transferfiles/state/timeestimation_test.go +++ b/artifactory/commands/transferfiles/state/timeestimation_test.go @@ -1,9 +1,11 @@ package state import ( + "testing" + "time" + "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "testing" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/utils/tests" @@ -32,12 +34,7 @@ func initTimeEstimationDataTest(t *testing.T) (*TimeEstimationManager, func()) { return newDefaultTimeEstimationManager(t, false), cleanUpJfrogHome } -func initTimeEstimationBITest(t *testing.T) (*TimeEstimationManager, func()) { - cleanUpJfrogHome := initTimeEstimationTestSuite(t) - return newDefaultTimeEstimationManager(t, true), cleanUpJfrogHome -} - -func TestGetDataEstimatedRemainingTime(t *testing.T) { +func TestGetSpeed(t *testing.T) { timeEstMng, cleanUp := initTimeEstimationDataTest(t) defer cleanUp() @@ -52,8 +49,7 @@ func TestGetDataEstimatedRemainingTime(t *testing.T) { addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond) assert.Equal(t, 7.5, timeEstMng.getSpeed()) assert.Equal(t, "7.500 MB/s", timeEstMng.GetSpeedString()) - assertGetEstimatedRemainingTime(t, timeEstMng, int64(62)) - assert.Equal(t, "About 1 minute", timeEstMng.GetEstimatedRemainingTimeString()) + assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds()) // Chunk 2: the upload of one of the files failed and the files are not included in the repository's total size (includedInTotalSize == false) chunkStatus2 := api.ChunkStatus{ @@ -65,91 +61,21 @@ func TestGetDataEstimatedRemainingTime(t *testing.T) { addChunkStatus(t, timeEstMng, chunkStatus2, 2, false, 5*milliSecsInSecond) assert.Equal(t, float64(8), timeEstMng.getSpeed()) assert.Equal(t, "8.000 MB/s", timeEstMng.GetSpeedString()) - assertGetEstimatedRemainingTime(t, timeEstMng, int64(58)) - assert.Equal(t, "Less than a minute", timeEstMng.GetEstimatedRemainingTimeString()) -} - -func TestGetBuildInfoEstimatedRemainingTime(t *testing.T) { - timeEstMng, cleanUp := initTimeEstimationBITest(t) - defer cleanUp() - - totalBiFiles := 100.0 - timeEstMng.stateManager.OverallBiFiles.TotalUnits = int64(totalBiFiles) - assertGetEstimatedRemainingTime(t, timeEstMng, int64(totalBiFiles*buildInfoAverageIndexTimeSec)) - - chunkStatus1 := api.ChunkStatus{ - Files: []api.FileUploadStatusResponse{ - createFileUploadStatusResponse(repo1Key, 10*bytesInMB, false, api.Success), - createFileUploadStatusResponse(repo1Key, 15*bytesInMB, false, api.Success), - }, - } - err := UpdateChunkInState(timeEstMng.stateManager, &chunkStatus1) - assert.NoError(t, err) - assertGetEstimatedRemainingTime(t, timeEstMng, int64((totalBiFiles-2)*buildInfoAverageIndexTimeSec)) + assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds()) } -func TestGetCombinedEstimatedRemainingTime(t *testing.T) { +func TestGetEstimatedRemainingSeconds(t *testing.T) { timeEstMng, cleanUp := initTimeEstimationDataTest(t) defer cleanUp() - totalBiFiles := 100.0 - timeEstMng.stateManager.OverallBiFiles.TotalUnits = int64(totalBiFiles) + timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) + timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes + assert.Zero(t, timeEstMng.getEstimatedRemainingSeconds()) - // Start transferring a data repository, make sure the remaining time includes the estimation of both bi and data. - chunkStatus1 := api.ChunkStatus{ - Files: []api.FileUploadStatusResponse{ - createFileUploadStatusResponse(repo1Key, 10*bytesInMB, false, api.Success), - createFileUploadStatusResponse(repo1Key, 15*bytesInMB, false, api.Success), - createFileUploadStatusResponse(repo1Key, 8*bytesInMB, true, api.Success), - }, - } - addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond) - assert.Equal(t, 7.5, timeEstMng.getSpeed()) - assert.Equal(t, "7.500 MB/s", timeEstMng.GetSpeedString()) - assert.NoError(t, timeEstMng.calculateDataEstimatedRemainingTime()) - assert.Equal(t, int64(62), timeEstMng.DataEstimatedRemainingTime) - assert.Equal(t, int64(41), timeEstMng.getBuildInfoEstimatedRemainingTime()) - assertGetEstimatedRemainingTime(t, timeEstMng, int64(103)) - - // Change to transferring a bi repository. - assert.NoError(t, timeEstMng.stateManager.SetRepoState(repo3Key, 0, 0, true, true)) - chunkStatus2 := api.ChunkStatus{ - Files: []api.FileUploadStatusResponse{ - createFileUploadStatusResponse(repo3Key, 10*bytesInMB, false, api.Success), - createFileUploadStatusResponse(repo3Key, 15*bytesInMB, false, api.Success), - }, - } - addChunkStatus(t, timeEstMng, chunkStatus2, 3, true, 10*milliSecsInSecond) - assert.Equal(t, "Not available while transferring a build-info repository", timeEstMng.GetSpeedString()) - // Data estimated time should remain as it was before: - assert.NoError(t, timeEstMng.calculateDataEstimatedRemainingTime()) - assert.Equal(t, int64(62), timeEstMng.DataEstimatedRemainingTime) - // Build info estimation should be lowered because two build info files were transferred. - assert.Equal(t, int64(40), timeEstMng.getBuildInfoEstimatedRemainingTime()) - assertGetEstimatedRemainingTime(t, timeEstMng, int64(102)) -} - -func assertGetEstimatedRemainingTime(t *testing.T, timeEstMng *TimeEstimationManager, expected int64) { - estimatedRemainingTime, err := timeEstMng.getEstimatedRemainingTime() - assert.NoError(t, err) - assert.Equal(t, expected, estimatedRemainingTime) -} - -func TestGetWorkingThreadsForBuildInfoEstimation(t *testing.T) { - timeEstMng, cleanUp := initTimeEstimationBITest(t) - defer cleanUp() - - setWorkingThreadsAndAssertBiThreads(t, timeEstMng, 0, 1) - setWorkingThreadsAndAssertBiThreads(t, timeEstMng, 1, 1) - setWorkingThreadsAndAssertBiThreads(t, timeEstMng, 8, 8) - setWorkingThreadsAndAssertBiThreads(t, timeEstMng, 9, 8) -} - -func setWorkingThreadsAndAssertBiThreads(t *testing.T, timeEstMng *TimeEstimationManager, threads, expectedBiThreads int) { - timeEstMng.stateManager.WorkingThreads = threads - actualBiThreads, err := timeEstMng.getWorkingThreadsForBuildInfoEstimation() - assert.NoError(t, err) - assert.Equal(t, expectedBiThreads, actualBiThreads) + timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) / 2 + timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes / 2 + calculatedEstimatedSeconds := timeEstMng.getEstimatedRemainingSeconds() + assert.NotZero(t, calculatedEstimatedSeconds) } func TestGetEstimatedRemainingTimeStringNotAvailableYet(t *testing.T) { @@ -162,37 +88,33 @@ func TestGetEstimatedRemainingTimeStringNotAvailableYet(t *testing.T) { createFileUploadStatusResponse(repo1Key, 8*bytesInMB, true, api.Success), }, } + assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString()) addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond) assert.Equal(t, "Not available yet", timeEstMng.GetSpeedString()) - assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString()) } -func TestEstimationNotAvailable(t *testing.T) { +func TestGetEstimatedRemainingTimeString(t *testing.T) { timeEstMng, cleanUp := initTimeEstimationDataTest(t) defer cleanUp() - // Assert unavailable if on unsupported phase. - timeEstMng.stateManager.CurrentRepoPhase = api.Phase2 - assert.Equal(t, "Not available in this phase", timeEstMng.GetEstimatedRemainingTimeString()) - - // After made available, assert not available until LastSpeeds are set. - timeEstMng.stateManager.CurrentRepoPhase = api.Phase3 + // Test "Not available yet" by setting the TotalTransferredBytes to 0 + timeEstMng.CurrentTotalTransferredBytes = 0 assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString()) - timeEstMng.LastSpeeds = []float64{1.23} - assert.Equal(t, "Less than a minute", timeEstMng.GetEstimatedRemainingTimeString()) -} - -func TestSpeedUnavailableForBuildInfoRepo(t *testing.T) { - timeEstMng, cleanUp := initTimeEstimationDataTest(t) - defer cleanUp() + // Test "About 1 minute" by setting the transferred bytes to 80% + timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8) + timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8) + assert.Equal(t, "About 1 minute", timeEstMng.GetEstimatedRemainingTimeString()) - assert.NoError(t, timeEstMng.stateManager.SetRepoState(repo3Key, 0, 0, true, true)) - assert.Equal(t, "Not available while transferring a build-info repository", timeEstMng.GetSpeedString()) + // Test "Less than a minute" by setting the transferred bytes to 90% + timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9) + timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9) + assert.Equal(t, "Less than a minute", timeEstMng.GetEstimatedRemainingTimeString()) } func newDefaultTimeEstimationManager(t *testing.T, buildInfoRepos bool) *TimeEstimationManager { stateManager, err := NewTransferStateManager(true) + stateManager.startTimestamp = time.Now().Add(-minTransferTimeToShowEstimation) assert.NoError(t, err) assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, buildInfoRepos, true)) assert.NoError(t, stateManager.SetRepoState(repo2Key, 0, 0, buildInfoRepos, true)) diff --git a/artifactory/commands/transferfiles/status.go b/artifactory/commands/transferfiles/status.go index 4250542b4..b5880e32e 100644 --- a/artifactory/commands/transferfiles/status.go +++ b/artifactory/commands/transferfiles/status.go @@ -20,7 +20,12 @@ const sizeUnits = "KMGTPE" func ShowStatus() error { var output strings.Builder - runningTime, isRunning, err := state.GetRunningTime() + stateManager, err := state.NewTransferStateManager(true) + if err != nil { + return err + } + + isRunning, err := stateManager.InitStartTimestamp() if err != nil { return err } @@ -39,11 +44,7 @@ func ShowStatus() error { return nil } - stateManager, err := state.NewTransferStateManager(true) - if err != nil { - return err - } - addOverallStatus(stateManager, &output, runningTime) + addOverallStatus(stateManager, &output, stateManager.GetRunningTimeString()) if stateManager.CurrentRepoKey != "" { transferState, exists, err := state.LoadTransferState(stateManager.CurrentRepoKey, false) if err != nil { diff --git a/artifactory/commands/transferfiles/status_test.go b/artifactory/commands/transferfiles/status_test.go index 563d7490f..738d90e66 100644 --- a/artifactory/commands/transferfiles/status_test.go +++ b/artifactory/commands/transferfiles/status_test.go @@ -68,7 +68,7 @@ func TestShowStatus(t *testing.T) { assert.Contains(t, results, "Repositories: 15 / 1111 (1.4%)") assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: 0.011 MB/s") - assert.Contains(t, results, "Estimated time remaining: Less than a minute") + assert.Contains(t, results, "Estimated time remaining: Not available yet") assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)") // Check repository status @@ -100,7 +100,7 @@ func TestShowStatusDiffPhase(t *testing.T) { assert.Contains(t, results, "Repositories: 15 / 1111 (1.4%)") assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: 0.011 MB/s") - assert.Contains(t, results, "Estimated time remaining: Not available in this phase") + assert.Contains(t, results, "Estimated time remaining: Not available yet") assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)") // Check repository status @@ -131,8 +131,8 @@ func TestShowBuildInfoRepo(t *testing.T) { assert.Contains(t, results, "Storage: 4.9 KiB / 10.9 KiB (45.0%)") assert.Contains(t, results, "Repositories: 15 / 1111 (1.4%)") assert.Contains(t, results, "Working threads: 16") - assert.Contains(t, results, "Transfer speed: Not available while transferring a build-info repository") - assert.Contains(t, results, "Estimated time remaining: Less than a minute") + assert.Contains(t, results, "Transfer speed: 0.011 MB/s") + assert.Contains(t, results, "Estimated time remaining: Not available yet") assert.Contains(t, results, "Transfer failures: 223") // Check repository status @@ -179,6 +179,7 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks stateManager.OverallTransfer.TotalSizeBytes = 11111 stateManager.TotalRepositories.TotalUnits = 1111 stateManager.TotalRepositories.TransferredUnits = 15 + stateManager.CurrentTotalTransferredBytes = 15 stateManager.WorkingThreads = 16 stateManager.VisitedFolders = 15 stateManager.DelayedFiles = 20 diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index a3f168a42..03ecd3848 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "syscall" - "time" "github.com/jfrog/gofrog/version" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" @@ -48,7 +47,6 @@ type TransferFilesCommand struct { progressbar *TransferProgressMng includeReposPatterns []string excludeReposPatterns []string - timeStarted time.Time ignoreState bool proxyKey string status bool @@ -72,7 +70,6 @@ func NewTransferFilesCommand(sourceServer, targetServer *config.ServerDetails) ( cancelFunc: cancelFunc, sourceServerDetails: sourceServer, targetServerDetails: targetServer, - timeStarted: time.Now(), stateManager: stateManager, stopSignal: make(chan os.Signal, 1), }, nil @@ -121,7 +118,7 @@ func (tdc *TransferFilesCommand) Run() (err error) { if tdc.stop { return tdc.signalStop() } - if err := tdc.stateManager.TryLockTransferStateManager(); err != nil { + if err = tdc.stateManager.TryLockTransferStateManager(); err != nil { return err } defer func() { @@ -130,6 +127,9 @@ func (tdc *TransferFilesCommand) Run() (err error) { err = unlockErr } }() + if _, err = tdc.stateManager.InitStartTimestamp(); err != nil { + return err + } srcUpService, err := createSrcRtUserPluginServiceManager(tdc.context, tdc.sourceServerDetails) if err != nil { @@ -230,6 +230,7 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui tdc.stateManager.OverallTransfer.TotalUnits = totalFiles tdc.stateManager.TotalRepositories.TotalUnits = int64(len(allSourceLocalRepos)) tdc.stateManager.OverallBiFiles.TotalUnits = totalBiFiles + tdc.stateManager.TimeEstimationManager.CurrentTotalTransferredBytes = 0 if !tdc.ignoreState { numberInitialErrors, e := getRetryErrorCount(allSourceLocalRepos) if e != nil { @@ -648,7 +649,7 @@ func (tdc *TransferFilesCommand) cleanup(originalErr error, sourceRepos []string } } - csvErrorsFile, e := createErrorsCsvSummary(sourceRepos, tdc.timeStarted) + csvErrorsFile, e := createErrorsCsvSummary(sourceRepos, tdc.stateManager.GetStartTimestamp()) if e != nil { log.Error("Couldn't create the errors CSV file", e) if err == nil { @@ -703,11 +704,11 @@ func (tdc *TransferFilesCommand) handleMaxUniqueSnapshots(repoSummary *serviceUt // Create the '~/.jfrog/transfer/stop' file to mark the transfer-file process to stop func (tdc *TransferFilesCommand) signalStop() error { - _, isRunning, err := state.GetRunningTime() + running, err := tdc.stateManager.Running() if err != nil { return err } - if !isRunning { + if !running { return errorutils.CheckErrorf("There is no active file transfer process.") } diff --git a/go.mod b/go.mod index 7841d63fa..37e7ff3c9 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.20 require github.com/c-bata/go-prompt v0.2.5 // Should not be updated to 0.2.6 due to a bug (https://github.com/jfrog/jfrog-cli-core/pull/372) +require github.com/jedib0t/go-pretty/v6 v6.4.0 // Should not be updated to v6.4.1+ due to a bug (https://github.com/jfrog/jfrog-cli-core/pull/1045) + require ( github.com/buger/jsonparser v1.1.1 github.com/chzyer/readline v1.5.1 @@ -11,7 +13,6 @@ require ( github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d github.com/google/uuid v1.3.1 github.com/gookit/color v1.5.4 - github.com/jedib0t/go-pretty/v6 v6.4.8 github.com/jfrog/build-info-go v1.9.15 github.com/jfrog/gofrog v1.3.1 github.com/jfrog/jfrog-apps-config v1.0.1 diff --git a/go.sum b/go.sum index 280e81382..37055a6f1 100644 --- a/go.sum +++ b/go.sum @@ -194,8 +194,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jedib0t/go-pretty/v6 v6.4.8 h1:HiNzyMSEpsBaduKhmK+CwcpulEeBrTmxutz4oX/oWkg= -github.com/jedib0t/go-pretty/v6 v6.4.8/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= +github.com/jedib0t/go-pretty/v6 v6.4.0 h1:YlI/2zYDrweA4MThiYMKtGRfT+2qZOO65ulej8GTcVI= +github.com/jedib0t/go-pretty/v6 v6.4.0/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI= github.com/jfrog/build-info-go v1.8.9-0.20231119150101-5cfbe8fca39e h1:yhy4z08QtckwUfVs0W931wjYUif/Gfv46QazrgHqQrE= github.com/jfrog/build-info-go v1.8.9-0.20231119150101-5cfbe8fca39e/go.mod h1:XVFk2rCYhIdc7+hIGE8TC3le5PPM+xYHU22udoE2b7Q= github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk= @@ -518,7 +518,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/utils/progressbar/transferprogressbarmanager.go b/utils/progressbar/transferprogressbarmanager.go index dcf13ca17..ef8f2b535 100644 --- a/utils/progressbar/transferprogressbarmanager.go +++ b/utils/progressbar/transferprogressbarmanager.go @@ -281,8 +281,8 @@ func (tpm *TransferProgressMng) GetBarMng() *ProgressBarMng { func (tpm *TransferProgressMng) NewRunningTimeProgressBar() *TasksProgressBar { return tpm.barMng.NewStringProgressBar(tpm.transferLabels.RunningFor, func() string { - runningTime, isRunning, err := state.GetRunningTime() - if err != nil || !isRunning { + runningTime := tpm.stateMng.GetRunningTimeString() + if runningTime == "" { runningTime = "Running time not available" } return color.Green.Render(runningTime)