feat(): Add support for s3 storage in Kubernetes etcd backup job

Dockerfile

@@ -10,6 +10,11 @@ RUN microdnf update -y && rm -rf /var/cache/yum
 # hadolint ignore=DL3041
 RUN microdnf install -y curl findutils gzip tar \
     && microdnf clean all
+
+RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc.rpm \
+    && rpm -ih mc.rpm \
+    && rm mc.rpm
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN /bin/bash -o pipefail -c "\
     curl -L https://github.com/etcd-io/etcd/releases/download/v3.5.15/etcd-v3.5.15-linux-amd64.tar.gz \
@@ -19,4 +24,8 @@ RUN /bin/bash -o pipefail -c "\
     && mv /tmp/etcd /usr/local/bin/ \
     "
 
+ENV MC_CONFIG_DIR=/opt/mc/config
+RUN mkdir -p $MC_CONFIG_DIR && \
+    chown 1000:1000 $MC_CONFIG_DIR
+
 CMD ["/usr/local/bin/backup.sh"]

README.md

@@ -82,6 +82,12 @@ kubectl edit -n etcd-backup cm/backup-config
 ```
 
 The following options are used:
+- `ETCD_BACKUP_S3`: Use S3 to store etcd-backup snapshots
+- `ETCD_BACKUP_S3_NAME`: MinIO client host alias name
+- `ETCD_BACKUP_S3_HOST`: S3 host endpoint (with scheme)
+- `ETCD_BACKUP_S3_BUCKET`: S3 bucket name
+- `ETCD_BACKUP_S3_ACCESS_KEY`: access key to access S3 bucket
+- `ETCD_BACKUP_S3_SECRET_KEY`: secret key to access S3 bucket
 - `ETCD_BACKUP_SUBDIR`: Sub directory on PVC that should be used to store the backup. If it does not exist it will be created.
 - `ETCD_BACKUP_DIRNAME`: Directory name for a single backup. This is a format string used by
 [`date`](https://man7.org/linux/man-pages/man1/date.1.html)
@@ -94,6 +100,8 @@ The following options are used:
 - `ETCD_BACKUP_UMASK`: Umask used inside the script to set restrictive permission on written files, as they contain sensitive information.
 - `ENDPOINT`: The IP address of the etcd endpoint, without scheme or port, e.g. `"192.168.39.86"`.
 
+Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. This can be done thanks to an objects expiration configuration as described in the object lifecycle management [documentation](https://min.io/docs/minio/linux/administration/object-management/object-lifecycle-management.html#object-expiration).
+
 Changing the schedule be done in the CronJob directly, with `spec.schedule`:
 ```
 kubectl edit -n etcd-backup cronjob/etcd-backup

backup-config.yaml

@@ -4,6 +4,12 @@ metadata:
   name: backup-config
   namespace: etcd-backup
 data:
+  ETCD_BACKUP_S3: "false"
+  ETCD_BACKUP_S3_NAME: "minio"
+  ETCD_BACKUP_S3_HOST: "http://minio.local:9000"
+  ETCD_BACKUP_S3_BUCKET: "etcd-backup"
+  ETCD_BACKUP_S3_ACCESS_KEY: "randomaccesskey"
+  ETCD_BACKUP_S3_SECRET_KEY: "secretkey"
   ETCD_BACKUP_SUBDIR: "/"
   ETCD_BACKUP_DIRNAME: "+etcd-backup-%FT%T%:z"
   ETCD_BACKUP_EXPIRE_TYPE: "days"

backup.sh

@@ -3,7 +3,7 @@
 # backup.sh General etcd backup script
 ################################################################################
 #
-# Copyright (C) 2023 Adfinis AG
+# Copyright (C) 2024 Adfinis AG
 #                    https://adfinis.com
 #                    [email protected]
 #
@@ -32,50 +32,77 @@
 
 set -xeuo pipefail
 
-# set proper umask
-umask "${ETCD_BACKUP_UMASK}"
+# check storage type
+if [ "${ETCD_BACKUP_S3}" = "true" ]; then
+    # prepare & push backup to S3
 
-# validate expire type
-case "${ETCD_BACKUP_EXPIRE_TYPE}" in
-    days|count|never) ;;
-    *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;;
-esac
+    # update CA trust
+    # update-ca-trust
 
-# validate  expire numbers
-if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
-  case "${ETCD_BACKUP_KEEP_DAYS}" in
-    ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;;
-    *) ;;
-  esac
-elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
-  case "${ETCD_BACKUP_KEEP_COUNT}" in
-    ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;;
-    *) ;;
+    # configure mcli assuming the bucket already exists
+    bash +o history
+    mcli alias set "${ETCD_BACKUP_S3_NAME}" "${ETCD_BACKUP_S3_HOST}" "${ETCD_BACKUP_S3_ACCESS_KEY}" "${ETCD_BACKUP_S3_SECRET_KEY}"
+    bash -o history
+
+    # make dirname
+    BACKUP_FOLDER="$( date "${ETCD_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
+
+    # make necessary directory
+    mkdir -p "/tmp/etcd-backup/${BACKUP_FOLDER}"
+
+    # create backup to temporary location
+    ETCDCTL_API=3 etcdctl --endpoints "${ENDPOINT}:2379" --cacert="/etc/kubernetes/pki/etcd-ca/ca.crt"  --cert="/etc/kubernetes/pki/etcd-peer/tls.crt" --key="/etc/kubernetes/pki/etcd-peer/tls.key" snapshot save "/tmp/etcd-backup/${BACKUP_FOLDER}/snapshot.db"
+    ETCDCTL_API=3 etcdutl --write-out=table snapshot status "/tmp/etcd-backup/${BACKUP_FOLDER}/snapshot.db"
+
+    # move files to S3 and delete temporary files
+    mcli mv $-r /tmp/etcd-backup/* "${ETCD_BACKUP_S3_NAME }"/"${ETCD_BACKUP_S3_BUCKET}"
+    rm -rv /tmp/etcd-backup
+else
+  # set proper umask
+  umask "${ETCD_BACKUP_UMASK}"
+
+  # validate expire type
+  case "${ETCD_BACKUP_EXPIRE_TYPE}" in
+      days|count|never) ;;
+      *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;;
   esac
-fi
 
-# make dirname and cleanup paths
-BACKUP_FOLDER="$( date "${ETCD_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
-BACKUP_PATH="$( realpath -m "${ETCD_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )"
-BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )"
-BACKUP_ROOTPATH="$( realpath -m "/backup/${ETCD_BACKUP_SUBDIR}" )"
-
-# make nescesary directorys
-mkdir -p "/tmp/etcd-backup"
-mkdir -p "${BACKUP_PATH_POD}"
-
-# create backup to temporary location
-ETCDCTL_API=3 etcdctl --endpoints "${ENDPOINT}:2379" --cacert='/etc/kubernetes/pki/etcd-ca/ca.crt'  --cert='/etc/kubernetes/pki/etcd-peer/tls.crt' --key='/etc/kubernetes/pki/etcd-peer/tls.key' snapshot save /tmp/etcd-backup/snapshot.db
-ETCDCTL_API=3 etcdutl --write-out=table snapshot status /tmp/etcd-backup/snapshot.db
-
-# move files to pvc and delete temporary files
-mv /tmp/etcd-backup/* "${BACKUP_PATH_POD}"
-rm -rv /tmp/etcd-backup
-
-# expire backup
-if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
-  find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1  -type d -mtime "+${ETCD_BACKUP_KEEP_DAYS}" -exec rm -rv {} +
-elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
-  # shellcheck disable=SC3040,SC2012
-  ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${ETCD_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}"
+  # validate  expire numbers
+  if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
+    case "${ETCD_BACKUP_KEEP_DAYS}" in
+      ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;;
+      *) ;;
+    esac
+  elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
+    case "${ETCD_BACKUP_KEEP_COUNT}" in
+      ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;;
+      *) ;;
+    esac
+  fi
+
+  # make dirname and cleanup paths
+  BACKUP_FOLDER="$( date "${ETCD_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
+  BACKUP_PATH="$( realpath -m "${ETCD_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )"
+  BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )"
+  BACKUP_ROOTPATH="$( realpath -m "/backup/${ETCD_BACKUP_SUBDIR}" )"
+
+  # make nescesary directorys
+  mkdir -p "/tmp/etcd-backup"
+  mkdir -p "${BACKUP_PATH_POD}"
+
+  # create backup to temporary location
+  ETCDCTL_API=3 etcdctl --endpoints "${ENDPOINT}:2379" --cacert='/etc/kubernetes/pki/etcd-ca/ca.crt'  --cert='/etc/kubernetes/pki/etcd-peer/tls.crt' --key='/etc/kubernetes/pki/etcd-peer/tls.key' snapshot save /tmp/etcd-backup/snapshot.db
+  ETCDCTL_API=3 etcdutl --write-out=table snapshot status /tmp/etcd-backup/snapshot.db
+
+  # move files to pvc and delete temporary files
+  mv /tmp/etcd-backup/* "${BACKUP_PATH_POD}"
+  rm -rv /tmp/etcd-backup
+
+  # expire backup
+  if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
+    find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1  -type d -mtime "+${ETCD_BACKUP_KEEP_DAYS}" -exec rm -rv {} +
+  elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
+    # shellcheck disable=SC3040,SC2012
+    ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${ETCD_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}"
+  fi
 fi