add Storj DCS support for keys & snapshots (fixes #368)

akash-network · Jun 27, 2023 · b31f29f · b31f29f
1 parent 80177c7
commit b31f29f
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 21 deletions.
diff --git a/.env.sample b/.env.sample
@@ -1,3 +1,4 @@
 S3_KEY=<key>
 S3_SECRET=<secret>
 KEY_PASSWORD=<password>
+STORJ_ACCESS_GRANT=<DCS Storj Access Grant token>
diff --git a/Dockerfile b/Dockerfile
@@ -189,6 +189,12 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
   && unzip awscliv2.zip -d /usr/src && rm -f awscliv2.zip \
   && /usr/src/aws/install --bin-dir /usr/bin
 
+# Install Storj DCS uplink client
+RUN curl -L https://github.com/storj/storj/releases/latest/download/uplink_linux_amd64.zip -o uplink_linux_amd64.zip && \
+  unzip -o uplink_linux_amd64.zip && \
+  install uplink /usr/bin/uplink && \
+  rm -f uplink uplink_linux_amd64.zip
+
 # Copy scripts
 COPY run.sh snapshot.sh /usr/bin/
 RUN chmod +x /usr/bin/run.sh /usr/bin/snapshot.sh

diff --git a/README.md b/README.md
@@ -193,6 +193,7 @@ The `node_key.json` and `priv_validator_key.json` are both backed up, and can be
 |`S3_KEY`|S3 access key| | |
 |`S3_SECRET`|S3 secret key| | |
 |`S3_HOST`|The S3 API host|`https://s3.filebase.com`|`https://s3.us-east-1.amazonaws.com`|
+|`STORJ_ACCESS_GRANT`|DCS Storj Access Grant token (replaces `S3_KEY`, `S3_SECRET`, `S3_HOST`| | |
 |`KEY_PATH`|Bucket and directory to backup/restore to| |`bucket/nodes/node_1`|
 |`KEY_PASSWORD`|An optional password to encrypt your private keys. Shouldn't be optional| | |
 
@@ -254,6 +255,8 @@ Snapshots older than a specified time can also be deleted. Finally a JSON metada
 |`S3_KEY`|S3 access key| | |
 |`S3_SECRET`|S3 secret key| | |
 |`S3_HOST`|The S3 API host|`https://s3.filebase.com`|`s3.us-east-1.amazonaws.com`|
+|`STORJ_ACCESS_GRANT`|DCS Storj Access Grant token (replaces `S3_KEY`, `S3_SECRET`, `S3_HOST`| | |
+|`STORJ_UPLINK_ARGS`|DCS Storj Uplink arguments|`-p 4 -t 4 --progress=false`|`-p 8 -t 8 --parallelism-chunk-size 128M --progress=false`|
 |`SNAPSHOT_PATH`|The S3 path to upload snapshots to, including the bucket| |`cosmos-snapshots/akash`|
 |`SNAPSHOT_PREFIX`|The prefix for the snapshot filename|`$CHAIN_ID`|`snapshot`|
 |`SNAPSHOT_TIME`|The time the snapshot will run|`00:00:00`|`09:00:00`|

diff --git a/run.sh b/run.sh
@@ -50,9 +50,17 @@ fi
 export AWS_ACCESS_KEY_ID=$S3_KEY
 export AWS_SECRET_ACCESS_KEY=$S3_SECRET
 export S3_HOST="${S3_HOST:-https://s3.filebase.com}"
+export STORJ_ACCESS_GRANT=$STORJ_ACCESS_GRANT
+
+storj_args="${STORJ_UPLINK_ARGS:--p 4 -t 4 --progress=false}"
+
+if [ -n "$STORJ_ACCESS_GRANT" ]; then
+  uplink access import --force --interactive=false default "$STORJ_ACCESS_GRANT"
+fi
 
 if [ -n "$KEY_PATH" ]; then
-  s3_uri_base="s3://${KEY_PATH}"
+  s3_uri_base="s3://${KEY_PATH%/}"
+  storj_uri_base="sj://${KEY_PATH%/}"
   aws_args="--endpoint-url ${S3_HOST}"
   if [ -n "$KEY_PASSWORD" ]; then
     file_suffix=".gpg"
@@ -62,12 +70,20 @@ if [ -n "$KEY_PATH" ]; then
 fi
 
 restore_key () {
-  existing=$(aws $aws_args s3 ls "${s3_uri_base}/$1" | head -n 1)
+  if [ -n "$STORJ_ACCESS_GRANT" ]; then
+    existing=$(uplink ls "${storj_uri_base}/$1" | head -n 1)
+  else
+    existing=$(aws $aws_args s3 ls "${s3_uri_base}/$1" | head -n 1)
+  fi
   if [[ -z $existing ]]; then
     echo "$1 backup not found"
   else
     echo "Restoring $1"
-    aws $aws_args s3 cp "${s3_uri_base}/$1" $CONFIG_PATH/$1$file_suffix
+    if [ -n "$STORJ_ACCESS_GRANT" ]; then
+      uplink cp "${storj_uri_base}/$1" $CONFIG_PATH/$1$file_suffix
+    else
+      aws $aws_args s3 cp "${s3_uri_base}/$1" $CONFIG_PATH/$1$file_suffix
+    fi
 
     if [ -n "$KEY_PASSWORD" ]; then
       echo "Decrypting"
@@ -78,14 +94,22 @@ restore_key () {
 }
 
 backup_key () {
-  existing=$(aws $aws_args s3 ls "${s3_uri_base}/$1" | head -n 1)
+  if [ -n "$STORJ_ACCESS_GRANT" ]; then
+    existing=$(uplink ls "${storj_uri_base}/$1" | head -n 1)
+  else
+    existing=$(aws $aws_args s3 ls "${s3_uri_base}/$1" | head -n 1)
+  fi
   if [[ -z $existing ]]; then
     echo "Backing up $1"
     if [ -n "$KEY_PASSWORD" ]; then
       echo "Encrypting backup..."
       gpg --symmetric --batch --passphrase "$KEY_PASSWORD" $CONFIG_PATH/$1
     fi
-    aws $aws_args s3 cp $CONFIG_PATH/$1$file_suffix "${s3_uri_base}/$1"
+    if [ -n "$STORJ_ACCESS_GRANT" ]; then
+      uplink cp $CONFIG_PATH/$1$file_suffix "${storj_uri_base}/$1"
+    else
+      aws $aws_args s3 cp $CONFIG_PATH/$1$file_suffix "${s3_uri_base}/$1"
+    fi
     [ -n "$KEY_PASSWORD" ] && rm $CONFIG_PATH/$1.gpg
   fi
 }
@@ -240,7 +264,9 @@ if [ "$DOWNLOAD_SNAPSHOT" == "1" ]; then
 
   # SNAPSHOT_FORMAT default value generation via SNAPSHOT_URL
   SNAPSHOT_FORMAT_DEFAULT="tar.gz"
-  case "${SNAPSHOT_URL,,}" in
+  # DCS Storj backups adding ?download=1 part which needs to be stripped before determining the extension
+  SNAPSHOT_URL_TRIM="${SNAPSHOT_URL%?download=1}"
+  case "${SNAPSHOT_URL_TRIM,,}" in
     *.tar.gz)   SNAPSHOT_FORMAT_DEFAULT="tar.gz";;
     *.tar.lz4)  SNAPSHOT_FORMAT_DEFAULT="tar.lz4";;
     *.tar.zst)  SNAPSHOT_FORMAT_DEFAULT="tar.zst";;
@@ -269,7 +295,16 @@ if [ "$DOWNLOAD_SNAPSHOT" == "1" ]; then
       # Value cannot be started with `0`, and must be integer
       [1-9]*[0-9]) pv_extra_args="-s $snapshot_size_in_bytes";;
     esac
-    (wget -nv -O - $SNAPSHOT_URL | pv -petrafb -i 5 $pv_extra_args | eval $tar_cmd) 2>&1 | stdbuf -o0 tr '\r' '\n'
+
+    # use DCS Storj uplink for the Storj backups (much faster)
+    if [[ "${SNAPSHOT_URL}" == *"link.storjshare.io"* ]] && [ -n "$STORJ_ACCESS_GRANT" ]; then
+      STORJ_SNAPSHOT_URL=${SNAPSHOT_URL#*link.storjshare.io/s/}
+      STORJ_SNAPSHOT_URL=${STORJ_SNAPSHOT_URL#*/}
+      STORJ_SNAPSHOT_URL=${STORJ_SNAPSHOT_URL%%\?*}
+      (uplink cp $storj_args sj://${STORJ_SNAPSHOT_URL} - | pv -petrafb -i 5 $pv_extra_args | eval $tar_cmd) 2>&1 | stdbuf -o0 tr '\r' '\n'
+    else
+      (wget -nv -O - $SNAPSHOT_URL | pv -petrafb -i 5 $pv_extra_args | eval $tar_cmd) 2>&1 | stdbuf -o0 tr '\r' '\n'
+    fi
 
     [ -n "${SNAPSHOT_DATA_PATH}" ] && mv ./${SNAPSHOT_DATA_PATH}/* ./ && rm -rf ./${SNAPSHOT_DATA_PATH}
     if [ -n "${SNAPSHOT_WASM_PATH}" ]; then

diff --git a/snapshot.sh b/snapshot.sh
@@ -8,7 +8,7 @@ SNAPSHOT_TIME="${SNAPSHOT_TIME:-00:00:00}"
 SNAPSHOT_DAY="${SNAPSHOT_DAY:-*}"
 SNAPSHOT_DIR="${SNAPSHOT_DIR:-$PROJECT_ROOT/data}"
 SNAPSHOT_CMD="${SNAPSHOT_CMD:-$@}"
-SNAPSHOT_PATH="${SNAPSHOT_PATH}"
+SNAPSHOT_PATH="${SNAPSHOT_PATH%/}"
 SNAPSHOT_PREFIX="${SNAPSHOT_PREFIX:-$CHAIN_ID}"
 SNAPSHOT_RETAIN="${SNAPSHOT_RETAIN:-2 days}"
 SNAPSHOT_METADATA="${SNAPSHOT_METADATA:-1}"
@@ -53,23 +53,42 @@ while true; do
 
         echo "$TIME: Running snapshot"
         aws_args="--endpoint-url ${S3_HOST}"
+        storj_args="${STORJ_UPLINK_ARGS:--p 4 -t 4 --progress=false}"
         s3_uri_base="s3://${SNAPSHOT_PATH}"
+        storj_uri_base="sj://${SNAPSHOT_PATH}"
         timestamp=$(date +"%Y-%m-%dT%H:%M:%S")
         s3_uri="${s3_uri_base}/${SNAPSHOT_PREFIX}_${timestamp}.${SNAPSHOT_SAVE_FORMAT}"
+        storj_uri="${storj_uri_base}/${SNAPSHOT_PREFIX}_${timestamp}.${SNAPSHOT_SAVE_FORMAT}"
 
         SNAPSHOT_SIZE=$(du -sb $SNAPSHOT_DIR | cut -f1)
 
-        case "${SNAPSHOT_SAVE_FORMAT,,}" in
-          tar.gz)   (tar c -C $SNAPSHOT_DIR . | gzip -1 | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | aws $aws_args s3 cp - "$s3_uri" --expected-size $SNAPSHOT_SIZE) 2>&1 | stdbuf -o0 tr '\r' '\n';;
-          # Compress level can be set via `ZSTD_CLEVEL`, default `3`
-          # No. of threads can be set via `ZSTD_NBTHREADS`, default `1`, `0` = detected no. of cpu cores
-          tar.zst)  (tar c -C $SNAPSHOT_DIR . | zstd -c $zstd_extra_arg | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | aws $aws_args s3 cp - "$s3_uri" --expected-size $SNAPSHOT_SIZE) 2>&1 | stdbuf -o0 tr '\r' '\n';;
-          # Catchall, assume to be tar
-          *)        (tar c -C $SNAPSHOT_DIR . | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | aws $aws_args s3 cp - "$s3_uri" --expected-size $SNAPSHOT_SIZE) 2>&1 | stdbuf -o0 tr '\r' '\n';;
-        esac
+        if [ -n "$STORJ_ACCESS_GRANT" ]; then
+          case "${SNAPSHOT_SAVE_FORMAT,,}" in
+            tar.gz)   (tar c -C $SNAPSHOT_DIR . | gzip -1 | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | uplink cp $storj_args - "$storj_uri") 2>&1 | stdbuf -o0 tr '\r' '\n';;
+            # Compress level can be set via `ZSTD_CLEVEL`, default `3`
+            # No. of threads can be set via `ZSTD_NBTHREADS`, default `1`, `0` = detected no. of cpu cores
+            tar.zst)  (tar c -C $SNAPSHOT_DIR . | zstd -c $zstd_extra_arg | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | uplink cp $storj_args - "$storj_uri") 2>&1 | stdbuf -o0 tr '\r' '\n';;
+            # Catchall, assume to be tar
+            *)        (tar c -C $SNAPSHOT_DIR . | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | uplink cp $storj_args - "$storj_uri") 2>&1 | stdbuf -o0 tr '\r' '\n';;
+          esac
+	else
+          case "${SNAPSHOT_SAVE_FORMAT,,}" in
+            tar.gz)   (tar c -C $SNAPSHOT_DIR . | gzip -1 | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | aws $aws_args s3 cp - "$s3_uri" --expected-size $SNAPSHOT_SIZE) 2>&1 | stdbuf -o0 tr '\r' '\n';;
+            # Compress level can be set via `ZSTD_CLEVEL`, default `3`
+            # No. of threads can be set via `ZSTD_NBTHREADS`, default `1`, `0` = detected no. of cpu cores
+            tar.zst)  (tar c -C $SNAPSHOT_DIR . | zstd -c $zstd_extra_arg | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | aws $aws_args s3 cp - "$s3_uri" --expected-size $SNAPSHOT_SIZE) 2>&1 | stdbuf -o0 tr '\r' '\n';;
+            # Catchall, assume to be tar
+            *)        (tar c -C $SNAPSHOT_DIR . | pv -petrafb -i 5 -s $SNAPSHOT_SIZE | aws $aws_args s3 cp - "$s3_uri" --expected-size $SNAPSHOT_SIZE) 2>&1 | stdbuf -o0 tr '\r' '\n';;
+          esac
+	fi
 
         if [[ $SNAPSHOT_RETAIN != "0" || $SNAPSHOT_METADATA != "0" ]]; then
-            readarray -t s3Files < <(aws $aws_args s3 ls "${s3_uri_base}/${SNAPSHOT_PREFIX}_")
+            if [ -n "$STORJ_ACCESS_GRANT" ]; then
+              SNAPSHOT_METADATA_URL=$(uplink share --url --not-after=none ${storj_uri_base}/ | grep ^URL | awk '{print $NF}')
+	      readarray -t s3Files < <(uplink ls ${storj_uri_base}/ | grep "${SNAPSHOT_PREFIX}_" | awk '{print $2,$3,$4,$5}' | sort -d -k4,4)
+            else
+              readarray -t s3Files < <(aws $aws_args s3 ls "${s3_uri_base}/${SNAPSHOT_PREFIX}_")
+            fi
             snapshots=()
             for line in "${s3Files[@]}"; do
                 createDate=`echo $line|awk {'print $1" "$2'}`
@@ -80,12 +99,19 @@ while true; do
                 else
                     fileUrl="${SNAPSHOT_METADATA_URL}${fileName}"
                 fi
+                if [ -n "$STORJ_ACCESS_GRANT" ]; then
+		    fileUrl="${fileUrl}?download=1"
+                fi
                 if [ "$SNAPSHOT_RETAIN" != "0" ]; then
                     olderThan=`date -d"-$SNAPSHOT_RETAIN" +%s`
                     if [[ $createDate -lt $olderThan ]]; then
                         if [[ $fileName != "" ]]; then
                             echo "$TIME: Deleting snapshot $fileName"
-                            aws $aws_args s3 rm "${s3_uri_base}/$fileName"
+                            if [ -n "$STORJ_ACCESS_GRANT" ]; then
+                              uplink rm "${storj_uri_base}/$fileName"
+                            else
+                              aws $aws_args s3 rm "${s3_uri_base}/$fileName"
+                            fi
                         fi
                     else
                         snapshots+=("$fileUrl")
@@ -101,9 +127,19 @@ while true; do
                 for url in ${snapshots[@]}; do
                     snapshotJson="$(echo $snapshotJson | jq ".+[\"$url\"]")"
                 done
-                echo $snapshotJson | jq '{chain_id: $c, snapshots: ., latest: $l}' \
-                   --arg c "$CHAIN_ID" --arg l "${snapshots[-1]}" | \
-                   aws $aws_args s3 cp - "${s3_uri_base}/snapshot.json"
+                if [ -n "$STORJ_ACCESS_GRANT" ]; then
+                  echo $snapshotJson | jq '{chain_id: $c, snapshots: ., latest: $l}' \
+                     --arg c "$CHAIN_ID" --arg l "${snapshots[-1]}" | \
+                     uplink cp - "${storj_uri_base}/snapshot.json"
+		  echo "=== Use the following as SNAPSHOT_JSON to restore the DCS Storj backup ==="
+                  ##uplink share --url --not-after=none "${storj_uri_base}/snapshot.json" | grep ^URL | awk '{print $NF"?download=1"}'
+		  echo "${SNAPSHOT_METADATA_URL%/}/snapshot.json?download=1"
+		  echo "=== === ==="
+                else
+                  echo $snapshotJson | jq '{chain_id: $c, snapshots: ., latest: $l}' \
+                     --arg c "$CHAIN_ID" --arg l "${snapshots[-1]}" | \
+                     aws $aws_args s3 cp - "${s3_uri_base}/snapshot.json"
+                fi
             fi
         fi