Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove backup slot number, 0 or 1, from Velero Backup request name #872

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Remove backup slot number, 0 or 1, from Velero Backup request name #872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a6d4c2
kubeobjects: Remove 0 1 from backup request name
hatfieldbrian May 5, 2023
8622815
shio-demo: cat s3 files instead of downloading
hatfieldbrian May 5, 2023
46547b8
recipe-e2e: Remove slot number from backup request name
hatfieldbrian May 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 22 additions & 21 deletions controllers/vrg_kubeobjects.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,25 +36,24 @@ func kubeObjectsCaptureInterval(kubeObjectProtectionSpec *ramen.KubeObjectProtec
return kubeObjectProtectionSpec.CaptureInterval.Duration
}

func kubeObjectsCapturePathNameAndNamePrefix(namespaceName, vrgName string, captureNumber int64) (string, string) {
func kubeObjectsCapturePathName(namespaceName, vrgName string, captureNumber int64) string {
const numberBase = 10
number := strconv.FormatInt(captureNumber, numberBase)

return s3PathNamePrefix(namespaceName, vrgName) + "kube-objects/" + number + "/",
// TODO fix: may exceed name capacity
namespaceName + "--" + vrgName + "--" + number
return s3PathNamePrefix(namespaceName, vrgName) + "kube-objects/" + number + "/"
}

func kubeObjectsCaptureName(prefix, groupName, s3ProfileName string) string {
return prefix + "--" + groupName + "--" + s3ProfileName
func kubeObjectsCaptureRequestName(vrgNamespaceName, vrgName, groupName, s3ProfileName string) string {
return kubeObjectsRequestName(vrgNamespaceName, vrgName, groupName+"--"+s3ProfileName)
}

func kubeObjectsRecoverNamePrefix(vrgNamespaceName, vrgName string) string {
return vrgNamespaceName + "--" + vrgName
func kubeObjectsRecoverRequestName(vrgNamespaceName, vrgName string, groupNumber int) string {
return kubeObjectsRequestName(vrgNamespaceName, vrgName, strconv.Itoa(groupNumber))
}

func kubeObjectsRecoverName(prefix string, groupNumber int) string {
return prefix + "--" + strconv.Itoa(groupNumber)
func kubeObjectsRequestName(vrgNamespaceName, vrgName, suffix string) string {
// TODO fix: may exceed name capacity
return vrgNamespaceName + "--" + vrgName + "--" + suffix
}

func (v *VRGInstance) kubeObjectsProtectPrimary(result *ctrl.Result, s3StoreAccessors []s3StoreAccessor) {
Expand Down Expand Up @@ -123,14 +122,14 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay(
interval := kubeObjectsCaptureInterval(vrg.Spec.KubeObjectProtection)
number := 1 - captureToRecoverFrom.Number
log := v.log.WithValues("number", number)
pathName, namePrefix := kubeObjectsCapturePathNameAndNamePrefix(vrg.Namespace, vrg.Name, number)
pathName := kubeObjectsCapturePathName(vrg.Namespace, vrg.Name, number)
labels := util.OwnerLabels(vrg.Namespace, vrg.Name)
captureStartOrResume := func(generation int64, startOrResume string) {
log.Info("Kube objects capture "+startOrResume, "generation", generation)
v.kubeObjectsCaptureStartOrResume(result, s3StoreAccessors,
captureStartConditionally,
captureInProgressStatusUpdate,
number, pathName, namePrefix, veleroNamespaceName, interval, labels, generation)
number, pathName, veleroNamespaceName, interval, labels, generation)
}

requests, err := v.reconciler.kubeObjects.ProtectRequestsGet(
Expand Down Expand Up @@ -227,7 +226,7 @@ const (
func (v *VRGInstance) kubeObjectsCaptureStartOrResume(
result *ctrl.Result, s3StoreAccessors []s3StoreAccessor,
captureStartConditionally captureStartConditionally, captureInProgressStatusUpdate captureInProgressStatusUpdate,
captureNumber int64, pathName, namePrefix, veleroNamespaceName string, interval time.Duration,
captureNumber int64, pathName, veleroNamespaceName string, interval time.Duration,
labels map[string]string, generation int64,
) {
vrg := v.instance
Expand All @@ -247,7 +246,8 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResume(
v.ctx, v.reconciler.Client, v.reconciler.APIReader, v.log,
s3StoreAccessor.url, s3StoreAccessor.bucketName, s3StoreAccessor.regionName,
pathName, s3StoreAccessor.veleroNamespaceSecretKeyRef, vrg.Namespace, captureGroup.Spec,
veleroNamespaceName, kubeObjectsCaptureName(namePrefix, captureGroup.Name, s3StoreAccessor.profileName),
veleroNamespaceName,
kubeObjectsCaptureRequestName(vrg.Namespace, vrg.Name, captureGroup.Name, s3StoreAccessor.profileName),
labels, annotations)
requests[requestsProcessedCount] = request
requestsProcessedCount++
Expand Down Expand Up @@ -482,9 +482,7 @@ func (v *VRGInstance) createRecoverOrProtectRequest(
annotations map[string]string,
) (kubeobjects.Request, error) {
vrg := v.instance
capturePathName, captureNamePrefix := kubeObjectsCapturePathNameAndNamePrefix(
sourceVrgNamespaceName, sourceVrgName, capture.Number)
recoverNamePrefix := kubeObjectsRecoverNamePrefix(vrg.Namespace, vrg.Name)
capturePathName := kubeObjectsCapturePathName(sourceVrgNamespaceName, sourceVrgName, capture.Number)

var request kubeobjects.Request

Expand All @@ -497,7 +495,7 @@ func (v *VRGInstance) createRecoverOrProtectRequest(
backupName := fmt.Sprintf("%s-restore-%d", recoverGroup.BackupName, groupNumber)
v.log.Info(fmt.Sprintf("backup: %s, captureToRecoverFrom: %d", backupName, captureToRecoverFrom.Number))

pathName, namePrefix := kubeObjectsCapturePathNameAndNamePrefix(vrg.Namespace, vrg.Name, backupSequenceNumber)
pathName := kubeObjectsCapturePathName(vrg.Namespace, vrg.Name, backupSequenceNumber)
request, err = v.reconciler.kubeObjects.ProtectRequestCreate(
v.ctx, v.reconciler.Client, v.reconciler.APIReader, v.log,
s3StoreAccessor.url,
Expand All @@ -507,7 +505,8 @@ func (v *VRGInstance) createRecoverOrProtectRequest(
s3StoreAccessor.veleroNamespaceSecretKeyRef,
vrg.Namespace,
recoverGroup.Spec,
veleroNamespaceName, kubeObjectsCaptureName(namePrefix, backupName, s3StoreAccessor.profileName),
veleroNamespaceName,
kubeObjectsCaptureRequestName(sourceVrgNamespaceName, sourceVrgName, backupName, s3StoreAccessor.profileName),
labels, annotations)
} else {
request, err = v.reconciler.kubeObjects.RecoverRequestCreate(
Expand All @@ -518,8 +517,10 @@ func (v *VRGInstance) createRecoverOrProtectRequest(
capturePathName,
s3StoreAccessor.veleroNamespaceSecretKeyRef,
sourceVrgNamespaceName, vrg.Namespace, recoverGroup, veleroNamespaceName,
kubeObjectsCaptureName(captureNamePrefix, recoverGroup.BackupName, s3StoreAccessor.profileName),
kubeObjectsRecoverName(recoverNamePrefix, groupNumber), labels, annotations)
kubeObjectsCaptureRequestName(
sourceVrgNamespaceName, sourceVrgName, recoverGroup.BackupName, s3StoreAccessor.profileName),
kubeObjectsRecoverRequestName(vrg.Namespace, vrg.Name, groupNumber),
labels, annotations)
}

return request, err
Expand Down
4 changes: 2 additions & 2 deletions hack/recipe_e2e/scripts/protect.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ mc du -r $MINIO_PROFILE

# wait for backups to be created according to Recipe spec
# backups should be successful (totalItems > 0); 5m replication period by default, starts at index 1
BACKUP_HOOK=$MINIO_PROFILE/$BUCKET_NAME/$NAMESPACE/$VRG_NAME/kube-objects/$BACKUP_START_INDEX/velero/backups/$NAMESPACE--$VRG_NAME--$BACKUP_START_INDEX--$BACKUP_HOOK_NAME--$MINIO_PROFILE/velero-backup.json
BACKUP_HOOK=$MINIO_PROFILE/$BUCKET_NAME/$NAMESPACE/$VRG_NAME/kube-objects/$BACKUP_START_INDEX/velero/backups/$NAMESPACE--$VRG_NAME--$BACKUP_HOOK_NAME--$MINIO_PROFILE/velero-backup.json
wait_for_and_check_backup_success $BACKUP_HOOK

BACKUP_RESOURCES=$MINIO_PROFILE/$BUCKET_NAME/$NAMESPACE/$VRG_NAME/kube-objects/$BACKUP_START_INDEX/velero/backups/$NAMESPACE--$VRG_NAME--$BACKUP_START_INDEX--$BACKUP_GROUP_NAME--$MINIO_PROFILE/velero-backup.json
BACKUP_RESOURCES=$MINIO_PROFILE/$BUCKET_NAME/$NAMESPACE/$VRG_NAME/kube-objects/$BACKUP_START_INDEX/velero/backups/$NAMESPACE--$VRG_NAME--$BACKUP_GROUP_NAME--$MINIO_PROFILE/velero-backup.json
wait_for_and_check_backup_success $BACKUP_RESOURCES

echo "protection successful"
Expand Down
14 changes: 7 additions & 7 deletions hack/shio-demo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ vrg_list() {
}; exit_stack_push unset -f vrg_list

vrg_get_s3() {
mc cp -q $(app_s3_object_name_prefix $1)v1alpha1.VolumeReplicationGroup/a /tmp/a.json.gz;gzip -df /tmp/a.json.gz;json_to_yaml </tmp/a.json
mc cat $(app_s3_object_name_prefix $1)v1alpha1.VolumeReplicationGroup/a|gzip -d|json_to_yaml
}; exit_stack_push unset -f vrg_get_s3

vr_get() {
Expand Down Expand Up @@ -451,7 +451,7 @@ app_s3_objects_delete() {

app_protection_info() {
for cluster_name in $s3_store_cluster_names; do
set -- "$1" $(app_s3_object_name_prefix_velero "$1" $cluster_name) $app_velero_kube_object_name$1----minio-on-$cluster_name
set -- "$1" $(app_s3_object_name_prefix_velero "$1" $cluster_name) $app_velero_kube_object_name--minio-on-$cluster_name
velero_backup_log $2 $3
velero_backup_backup_object $2 $3
velero_backup_resource_list $2 $3
Expand All @@ -467,23 +467,23 @@ app_recovery_info() {
}; exit_stack_push unset -f app_recovery_info

velero_backup_backup_object() {
mc cp -q $1backups/$2/velero-backup.json /tmp/$2-velero-backup.json;json_to_yaml </tmp/$2-velero-backup.json
mc cat $1backups/$2/velero-backup.json|jq
}; exit_stack_push unset -f velero_backup_backup_object

velero_backup_resource_list() {
mc cp -q $1backups/$2/$2-resource-list.json.gz /tmp;gzip -df /tmp/$2-resource-list.json.gz;json_to_yaml </tmp/$2-resource-list.json
mc cat $1backups/$2/$2-resource-list.json.gz|gzip -d|jq
}; exit_stack_push unset -f velero_backup_resource_list

velero_backup_log() {
mc cp -q $1backups/$2/$2-logs.gz /tmp;gzip -df /tmp/$2-logs.gz;cat /tmp/$2-logs
mc cat $1backups/$2/$2-logs.gz|gzip -d
}; exit_stack_push unset -f velero_backup_log

velero_restore_results() {
mc cp -q $1restores/$2/restore-$2-results.gz /tmp;gzip -df /tmp/restore-$2-results.gz;json_to_yaml </tmp/restore-$2-results
mc cat $1restores/$2/restore-$2-results.gz|gzip -d|jq
}; exit_stack_push unset -f velero_restore_results

velero_restore_log() {
mc cp -q $1restores/$2/restore-$2-logs.gz /tmp;gzip -df /tmp/restore-$2-logs.gz;cat /tmp/restore-$2-logs
mc cat $1restores/$2/restore-$2-logs.gz|gzip -d
}; exit_stack_push unset -f velero_restore_log

velero_kube_objects_list() {
Expand Down