[WIP] Include total transfer time in DataUpload/DataDownload columns.

config/crd/v1/bases/velero.io_podvolumebackups.yaml

@@ -199,6 +199,9 @@ spec:
                   bytesDone:
                     format: int64
                     type: integer
+                  skippedBytes:
+                    format: int64
+                    type: integer
                   totalBytes:
                     format: int64
                     type: integer

config/crd/v1/bases/velero.io_podvolumerestores.yaml

@@ -191,6 +191,9 @@ spec:
                   bytesDone:
                     format: int64
                     type: integer
+                  skippedBytes:
+                    format: int64
+                    type: integer
                   totalBytes:
                     format: int64
                     type: integer

config/crd/v2alpha1/bases/velero.io_datadownloads.yaml

@@ -45,6 +45,15 @@ spec:
       jsonPath: .status.node
       name: Node
       type: string
+    - description: Elapsed time of the actual transfer, eventually completion time
+        - start time
+      jsonPath: .status.elapsedTransferTime
+      name: Elapsed Time
+      type: string
+    - description: Actual bytes/second moved onto the cluster
+      jsonPath: .status.throughput
+      name: Throughput bytes/sec
+      type: integer
     name: v2alpha1
     schema:
       openAPIV3Schema:
@@ -144,6 +153,9 @@ spec:
                 format: date-time
                 nullable: true
                 type: string
+              elapsedTransferTime:
+                description: ElapsedTransferTime
+                type: string
               message:
                 description: Message is a message about the DataDownload's status.
                 type: string
@@ -171,6 +183,9 @@ spec:
                   bytesDone:
                     format: int64
                     type: integer
+                  skippedBytes:
+                    format: int64
+                    type: integer
                   totalBytes:
                     format: int64
                     type: integer
@@ -182,6 +197,10 @@ spec:
                 format: date-time
                 nullable: true
                 type: string
+              throughput:
+                description: Throughput
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

config/crd/v2alpha1/bases/velero.io_datauploads.yaml

@@ -46,6 +46,17 @@ spec:
       jsonPath: .status.node
       name: Node
       type: string
+    - description: Elapsed time of actual transfer, eventually 'completion time' -
+        'start time'.
+      jsonPath: .status.elapsedTransferTime
+      name: Elapsed Time
+      type: string
+    - description: Actual bytes/second moved off cluster, ignoring skipped/cached
+        bytes due to incremental hashing
+      format: int64
+      jsonPath: .status.throughput
+      name: Throughput bytes/sec
+      type: integer
     name: v2alpha1
     schema:
       openAPIV3Schema:
@@ -159,6 +170,11 @@ spec:
                   as a result of the DataUpload.
                 nullable: true
                 type: object
+              elapsedTransferTime:
+                description: |-
+                  ElapsedTransferTime is the total amount of time it took to complete
+                  this DataUpload (completion time - start time).
+                type: string
               message:
                 description: Message is a message about the DataUpload's status.
                 type: string
@@ -183,13 +199,17 @@ spec:
                 type: string
               progress:
                 description: |-
-                  Progress holds the total number of bytes of the volume and the current
-                  number of backed up bytes. This can be used to display progress information
+                  Progress holds the total number of bytes of the volume, the current
+                  number of backed up bytes, and the number of bytes skipped due to
+                  any incremental caching. This can be used to display progress information
                   about the backup operation.
                 properties:
                   bytesDone:
                     format: int64
                     type: integer
+                  skippedBytes:
+                    format: int64
+                    type: integer
                   totalBytes:
                     format: int64
                     type: integer
@@ -207,6 +227,12 @@ spec:
                 format: date-time
                 nullable: true
                 type: string
+              throughput:
+                description: |-
+                  Throughput is the rate of actual bytes transferred off-cluster during
+                  this DataUpload. It is equivalent to: (doneBytes - skippedBytes) / elapsedTime
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

pkg/apis/velero/shared/data_move_operation_progress.go

@@ -26,4 +26,7 @@ type DataMoveOperationProgress struct {
 
 	// +optional
 	BytesDone int64 `json:"bytesDone,omitempty"`
+
+	// +optional
+	SkippedBytes int64 `json:"skippedBytes,omitempty"`
 }

pkg/apis/velero/v2alpha1/data_download_types.go

@@ -115,6 +115,14 @@ type DataDownloadStatus struct {
 	// Node is name of the node where the DataDownload is processed.
 	// +optional
 	Node string `json:"node,omitempty"`
+
+	// ElapsedTransferTime
+	// +optional
+	ElapsedTransferTime metav1.Duration `json:"elapsedTransferTime,omitempty"`
+
+	// Throughput
+	// +optional
+	Throughput int64 `json:"throughput,omitempty"`
 }
 
 // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed.
@@ -130,6 +138,8 @@ type DataDownloadStatus struct {
 // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where the backup data is stored"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataDownload was created"
 // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the DataDownload is processed"
+// +kubebuilder:printcolumn:name="Elapsed Time",type="string",JSONPath=".status.elapsedTransferTime",description="Elapsed time of the actual transfer, eventually completion time - start time"
+// +kubebuilder:printcolumn:name="Throughput bytes/sec",type="integer",JSONPath=".status.throughput",description="Actual bytes/second moved onto the cluster"
 
 // DataDownload acts as the protocol between data mover plugins and data mover controller for the datamover restore operation
 type DataDownload struct {

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -135,15 +135,26 @@ type DataUploadStatus struct {
 	// +nullable
 	CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"`
 
-	// Progress holds the total number of bytes of the volume and the current
-	// number of backed up bytes. This can be used to display progress information
+	// Progress holds the total number of bytes of the volume, the current
+	// number of backed up bytes, and the number of bytes skipped due to
+	// any incremental caching. This can be used to display progress information
 	// about the backup operation.
 	// +optional
 	Progress shared.DataMoveOperationProgress `json:"progress,omitempty"`
 
 	// Node is name of the node where the DataUpload is processed.
 	// +optional
 	Node string `json:"node,omitempty"`
+
+	// ElapsedTransferTime is the total amount of time it took to complete
+	// this DataUpload (completion time - start time).
+	// +optional
+	ElapsedTransferTime metav1.Duration `json:"elapsedTransferTime,omitempty"`
+
+	// Throughput is the rate of actual bytes transferred off-cluster during
+	// this DataUpload. It is equivalent to: (doneBytes - skippedBytes) / elapsedTime
+	// +optional
+	Throughput int64 `json:"throughput,omitempty"`
 }
 
 // TODO(2.0) After converting all resources to use the runttime-controller client,
@@ -160,6 +171,8 @@ type DataUploadStatus struct {
 // +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataUpload was created"
 // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the DataUpload is processed"
+// +kubebuilder:printcolumn:name="Elapsed Time",type="string",JSONPath=".status.elapsedTransferTime",description="Elapsed time of actual transfer, eventually 'completion time' - 'start time'."
+// +kubebuilder:printcolumn:name="Throughput bytes/sec",type="integer",format="int64",JSONPath=".status.throughput",description="Actual bytes/second moved off cluster, ignoring skipped/cached bytes due to incremental hashing"
 
 // DataUpload acts as the protocol between data mover plugins and data mover controller for the datamover backup operation
 type DataUpload struct {

pkg/controller/data_download_controller.go

@@ -486,6 +486,9 @@
 
 	original := dd.DeepCopy()
 	dd.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone}
+	dd.Status.ElapsedTransferTime = metav1.Duration{
+		time.Since(dd.Status.StartTimestamp.Time),
+	}
 
 	if err := r.client.Patch(ctx, &dd, client.MergeFrom(original)); err != nil {
 		log.WithError(err).Error("Failed to update restore snapshot progress")

pkg/controller/data_upload_controller.go

@@ -442,7 +442,12 @@
 		log.Info("Data upload completed")
 		r.metrics.RegisterDataUploadSuccess(r.nodeName)
 	}
+
+	du.Status.ElapsedTransferTime = metav1.Duration{
+		du.Status.CompletionTimestamp.Sub(du.Status.StartTimestamp.Time),
+	}
+
 }
 
 func (r *DataUploadReconciler) OnDataUploadFailed(ctx context.Context, namespace, duName string, err error) {
 	defer r.dataPathMgr.RemoveAsyncBR(duName)
@@ -539,7 +544,11 @@
 	}
 
 	original := du.DeepCopy()
-	du.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone}
+	du.Status.Progress = shared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone, SkippedBytes: progress.SkippedBytes}
+	du.Status.ElapsedTransferTime = metav1.Duration{
+		time.Since(du.Status.StartTimestamp.Time),
+	}
+	du.Status.Throughput = (progress.BytesDone - progress.SkippedBytes) / int64(du.Status.ElapsedTransferTime.Seconds())
 
 	if err := r.client.Patch(ctx, &du, client.MergeFrom(original)); err != nil {
 		log.WithError(err).Error("Failed to update progress")

pkg/controller/pod_volume_backup_controller.go

@@ -279,7 +279,7 @@ func (r *PodVolumeBackupReconciler) OnDataPathProgress(ctx context.Context, name
 	}
 
 	original := pvb.DeepCopy()
-	pvb.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone}
+	pvb.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone, SkippedBytes: progress.SkippedBytes}
 
 	if err := r.Client.Patch(ctx, &pvb, client.MergeFrom(original)); err != nil {
 		log.WithError(err).Error("Failed to update progress")

pkg/datapath/file_system.go

@@ -236,7 +236,7 @@ func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint, uplo
 // UpdateProgress which implement ProgressUpdater interface to update progress status
 func (fs *fileSystemBR) UpdateProgress(p *uploader.Progress) {
 	if fs.callbacks.OnProgress != nil {
-		fs.callbacks.OnProgress(context.Background(), fs.namespace, fs.jobName, &uploader.Progress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone})
+		fs.callbacks.OnProgress(context.Background(), fs.namespace, fs.jobName, &uploader.Progress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone, SkippedBytes: p.SkippedBytes})
 	}
 }
 

pkg/uploader/kopia/progress.go

@@ -98,7 +98,7 @@ func (p *Progress) EstimatedDataSize(fileCount int, totalBytes int64) {
 // UpdateProgress which calls Updater UpdateProgress interface, update progress by third-party implementation
 func (p *Progress) UpdateProgress() {
 	if p.outputThrottle.ShouldOutput() {
-		p.Updater.UpdateProgress(&uploader.Progress{TotalBytes: p.estimatedTotalBytes, BytesDone: p.processedBytes})
+		p.Updater.UpdateProgress(&uploader.Progress{TotalBytes: p.estimatedTotalBytes, BytesDone: p.processedBytes, SkippedBytes: p.cachedBytes})
 	}
 }
 

pkg/uploader/types.go

@@ -57,10 +57,11 @@ type SnapshotInfo struct {
 	Size int64  `json:"Size"`
 }
 
-// Progress which defined two variables to record progress
+// Progress which defined three variables to record progress
 type Progress struct {
-	TotalBytes int64 `json:"totalBytes,omitempty"`
-	BytesDone  int64 `json:"doneBytes,omitempty"`
+	TotalBytes   int64 `json:"totalBytes,omitempty"`
+	BytesDone    int64 `json:"doneBytes,omitempty"`
+	SkippedBytes int64 `json:"skippedBytes,omitempty"`
 }
 
 // UploaderProgress which defined generic interface to update progress