From a03b832864297635df48477ef9acb766550f26b8 Mon Sep 17 00:00:00 2001
From: Jean Khawand <22157081+jeankhawand@users.noreply.github.com>
Date: Thu, 29 Aug 2024 19:33:14 +0200
Subject: [PATCH] Introduce `ENABLE_JSON_LOGGING`

This change adds the `ENABLE_JSON_LOGGING` option to export restic metadata in JSON format. The goal is to improve accessibility by allowing users to utilize tools like `jq` to process the metadata. This enables users to create external scripts for extracting metadata and sending notifications using `run_commands`.

update doc and include notify script as sample to send backup metadata via apprise

Export `LOG_FILE` and create docker-compose for contributing

- export `LOG_FILE` so it will be available for subprocess executed via `run_commands`
- Add docker-compose.contribute.yml facilitate contributing process
- make `notify` executable and copy it to /usr/local/bin
- notify: update content to only include summary and errors for now. jq was complaining on payload too large when using verbose mode.
- update doc
---
 Dockerfile                      |  4 +-
 README.md                       |  5 +-
 backup                          | 11 ++++-
 docker-compose.contributing.yml | 72 ++++++++++++++++++++++++++++
 notify                          | 83 +++++++++++++++++++++++++++++++++
 5 files changed, 170 insertions(+), 5 deletions(-)
 create mode 100644 docker-compose.contributing.yml
 create mode 100755 notify

diff --git a/Dockerfile b/Dockerfile
index e602c69..bcd4795 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -79,12 +79,12 @@ RUN curl -sL -o go-cron.tar.gz https://github.com/djmaze/go-cron/archive/v${GO_C
 #
 FROM alpine:3.20
 
-RUN apk add --update --no-cache ca-certificates fuse nfs-utils openssh tzdata bash curl docker-cli gzip tini
+RUN apk add --update --no-cache ca-certificates jq fuse nfs-utils openssh tzdata bash curl docker-cli gzip tini
 
 ENV RESTIC_REPOSITORY /mnt/restic
 
 COPY --from=builder /usr/local/bin/* /usr/local/bin/
-COPY backup prune check /usr/local/bin/
+COPY backup prune check notify /usr/local/bin/
 COPY entrypoint /
 
 ENTRYPOINT ["/sbin/tini", "--", "/entrypoint"]
diff --git a/README.md b/README.md
index 7dec4c6..e23bcdf 100644
--- a/README.md
+++ b/README.md
@@ -99,7 +99,10 @@ _Note: `BACKUP_CRON`, `PRUNE_CRON` and `CHECK_CRON` are mutually exclusive._
 - `TZ` - Optional. Set your timezone for the correct cron execution time.
 - `SKIP_INIT` - Skip initialization of the restic repository, even if it can not be accessed.
 - `SKIP_INIT_CHECK`- Do not fail, if initialization of the restic repository fails for whatever reason.
-
+- `ENABLE_JSON_LOGGING` = Converts Restic's standard output to JSON format. This allows users to leverage tools like `jq` to extract specific fields from the metadata. Additionally, it supports calling external scripts for further processing, such as [executing commands after a backup](#execute-commands-after-backup). For example:
+```sh
+notify "discord://,telegram://,slack://" "http://apprise:8000/notify"
+```
 ### Using the `rclone` repository type
 
 In order to use the `rclone` repository type, you need to prepare an `rclone.conf` file and mount it inside the container at `/run/secrets/rclone.conf`.
diff --git a/backup b/backup
index c547302..ed2b87f 100755
--- a/backup
+++ b/backup
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -eo pipefail
-
+LOG_FILE=""
 function run_commands {
 	COMMANDS=$1
 	while IFS= read -r cmd; do echo "$cmd" && eval "$cmd" ; done < <(printf '%s\n' "$COMMANDS")
@@ -62,7 +62,14 @@ start=$(date +%s)
 echo Starting Backup at $(date +"%Y-%m-%d %H:%M:%S")
 
 set +e
-restic --repo="${RESTIC_REPOSITORY}" backup "${backup_args[@]}" "${tag_options[@]}" "${backup_sources[@]}"
+if [ "$(check_bool "${ENABLE_JSON_LOGGING:-}")" = "true" ]; then
+  # Set LOG_FILE
+  LOG_FILE="/var/log/restic_backup_$(date +%Y%m%d_%H%M%S).json"
+  export LOG_FILE
+  restic --repo="${RESTIC_REPOSITORY}" backup "${backup_args[@]}" "${tag_options[@]}" "${backup_sources[@]}" --json | tee "$LOG_FILE"
+else 
+  restic --repo="${RESTIC_REPOSITORY}" backup "${backup_args[@]}" "${tag_options[@]}" "${backup_sources[@]}"
+fi
 rc=$?
 set -e
 
diff --git a/docker-compose.contributing.yml b/docker-compose.contributing.yml
new file mode 100644
index 0000000..40e57d0
--- /dev/null
+++ b/docker-compose.contributing.yml
@@ -0,0 +1,72 @@
+version: "3.3"
+
+services:
+  backup:
+    build:
+      context: .
+    hostname: docker
+    restart: unless-stopped
+    environment:
+      RUN_ON_STARTUP: "true"
+      BACKUP_CRON: "0 30 3 * * *"
+      RESTIC_PASSWORD: supersecret
+      RESTIC_BACKUP_SOURCES: /mnt/volumes
+      ENABLE_JSON_LOGGING: "true"
+      POST_COMMANDS_EXIT: |-
+        notify "discord://" "http://apprise:8080/notify"
+      RESTIC_BACKUP_ARGS: >-
+        --tag docker-volumes
+        --exclude some-folder/cache
+        --exclude another-folder/with\ space
+        --exclude *.tmp
+      RESTIC_FORGET_ARGS: >-
+        --keep-last 10
+        --keep-daily 7
+        --keep-weekly 5
+        --keep-monthly 12
+      B2_ACCOUNT_ID: xxxxxxx
+      B2_ACCOUNT_KEY: yyyyyyyy
+      TZ: Europe/Berlin
+    volumes:
+      - /var/lib/docker/volumes:/mnt/volumes:ro
+
+  prune:
+    build:
+      context: .
+    hostname: docker
+    restart: unless-stopped
+    environment:
+      SKIP_INIT: "true"
+      RUN_ON_STARTUP: "true"
+      PRUNE_CRON: "0 0 4 * * *"
+      RESTIC_REPOSITORY: b2:my-repo:/restic
+      RESTIC_PASSWORD: supersecret
+      B2_ACCOUNT_ID: xxxxxxx
+      B2_ACCOUNT_KEY: yyyyyyyy
+      TZ: Europe/Berlin
+
+  check:
+    build:
+      context: .
+    hostname: docker
+    restart: unless-stopped
+    environment:
+      SKIP_INIT: "true"
+      RUN_ON_STARTUP: "false"
+      CHECK_CRON: "0 15 5 * * *"
+      RESTIC_CHECK_ARGS: >-
+        --read-data-subset=10%
+      RESTIC_REPOSITORY: b2:my-repo:/restic
+      RESTIC_PASSWORD: supersecret
+      B2_ACCOUNT_ID: xxxxxxx
+      B2_ACCOUNT_KEY: yyyyyyyy
+      TZ: Europe/Berlin
+
+  apprise: 
+    image: caronc/apprise:latest
+    hostname: docker
+    restart: unless-stopped
+    environment:
+      TZ: Europe/Berlin
+    ports:
+      - 8080:8080
diff --git a/notify b/notify
new file mode 100755
index 0000000..43b8e16
--- /dev/null
+++ b/notify
@@ -0,0 +1,83 @@
+#!/bin/bash
+set -eo pipefail
+# Define the log file variable (make sure to replace this with the actual log file path)
+APPRISE_SERVICES_URL=$1
+APPRISE_ENDPOINT=$2
+# Check if LOG_FILE is set
+if [ -z "$LOG_FILE" ]; then
+    echo "LOG_FILE is not set"
+    exit 1
+fi
+
+# Perform the notification using LOG_FILE
+echo "Notifying about the log file: $LOG_FILE"
+
+# Extract information from the log file
+extract_field() {
+    jq -r "select(.message_type == \"$1\") | .$2" < "$LOG_FILE"
+}
+
+# Extract summary information
+files_new=$(extract_field "summary" "files_new")
+files_changed=$(extract_field "summary" "files_changed")
+files_unmodified=$(extract_field "summary" "files_unmodified")
+dirs_new=$(extract_field "summary" "dirs_new")
+dirs_changed=$(extract_field "summary" "dirs_changed")
+dirs_unmodified=$(extract_field "summary" "dirs_unmodified")
+data_blobs=$(extract_field "summary" "data_blobs")
+tree_blobs=$(extract_field "summary" "tree_blobs")
+data_added=$(extract_field "summary" "data_added")
+total_files_processed=$(extract_field "summary" "total_files_processed")
+total_bytes_processed=$(extract_field "summary" "total_bytes_processed")
+total_duration=$(extract_field "summary" "total_duration")
+snapshot_id=$(extract_field "summary" "snapshot_id")
+
+# Extract error information
+error_message=$(extract_field "error" "error")
+error_during=$(extract_field "error" "during")
+error_item=$(extract_field "error" "item")
+
+# Handle cases where error and verbose status might be empty
+if [[ -z "$error_message" ]]; then
+    error_section="No errors reported."
+else
+    error_section="Error encountered:
+- Message: $error_message
+- During: $error_during
+- Item: $error_item"
+fi
+
+# Create a message with status, summary, error, and verbose details
+MESSAGE="Restic Backup Report
+Summary:
+- Files New: $files_new
+- Files Changed: $files_changed
+- Files Unmodified: $files_unmodified
+- Dirs New: $dirs_new
+- Dirs Changed: $dirs_changed
+- Dirs Unmodified: $dirs_unmodified
+- Data Blobs: $data_blobs
+- Tree Blobs: $tree_blobs
+- Data Added: $data_added bytes
+- Total Files Processed: $total_files_processed
+- Total Bytes Processed: $total_bytes_processed
+- Total Duration: $total_duration seconds
+- Snapshot ID: $snapshot_id
+Errors:
+$error_section"
+
+# Print the message to the console (for debugging purposes)
+echo "$MESSAGE"
+
+# Create the JSON payload for Apprise
+PAYLOAD=$(jq -n \
+    --arg urls "$APPRISE_SERVICES_URL" \
+    --arg body "$MESSAGE" \
+    '{
+        "urls": $urls,
+        "body": $body
+    }'
+)
+
+# Send the payload using curl
+curl -X POST "$APPRISE_ENDPOINT" -H 'Content-Type: application/json' -d "$PAYLOAD"
\ No newline at end of file