From 5f86aaf22b9edc9c93a4af5aae5014b769ecdc09 Mon Sep 17 00:00:00 2001 From: realAP <44711679+realAP@users.noreply.github.com> Date: Sun, 8 Dec 2024 17:58:55 +0100 Subject: [PATCH] 5 add pgdump as provisioning for backup data (#8) * a lot of work1 * 5: add backup scripts * 5: add comments * 5: fix for long messages * 5: restart option was buggy, fixxed * 5: add provisioning mode for manager * 5: cleaning * 5: adjust template for env and build in Variable checker * 5: add additional way for debugging * 5: create folders * 5: first step of new images * 5: first step of new images II * 5: update images * 5: add version bump file * 5: init repo when not existing * 5: not needed * 5: cleaning * 5: fix bug #10 * 5: adjustments * 5: done * 5: update * 5: update --- .env | 48 ++- .gitignore | 1 + README.md | 184 +++++---- VERSION | 2 +- backup_overview.drawio.svg | 4 - docker/Dockerfile | 13 +- docker/backup.sh | 54 +-- docker/entry.sh | 66 ++-- docker/manager.sh | 56 +-- docker/postgres_backup.sh | 9 + docker/postgres_restore.sh | 9 + docker/prepare.sh | 57 +++ .../{create_ssh_config.sh => prepare_ssh.sh} | 39 +- docker/telegram.sh | 90 ++--- resources/backup_overview.drawio | 195 ++++++++++ resources/backup_overview.svg | 4 + .../backup_overview_nextcloud.drawio | 351 +++++++++--------- resources/backup_overview_nextcloud.svg | 4 + resources/backup_overview_postgres.drawio | 177 +++++++++ resources/backup_overview_postgres.svg | 4 + run_backup.sh | 100 ++++- source/.gitkeep | 0 testing/preset_data.sql | 7 + testing/test.compose.yaml | 13 + 24 files changed, 1050 insertions(+), 437 deletions(-) delete mode 100644 backup_overview.drawio.svg create mode 100644 docker/postgres_backup.sh create mode 100644 docker/postgres_restore.sh create mode 100644 docker/prepare.sh rename docker/{create_ssh_config.sh => prepare_ssh.sh} (57%) create mode 100644 resources/backup_overview.drawio create mode 100644 resources/backup_overview.svg rename backup_overview.drawio => resources/backup_overview_nextcloud.drawio (84%) create mode 100644 resources/backup_overview_nextcloud.svg create mode 100644 resources/backup_overview_postgres.drawio create mode 100644 resources/backup_overview_postgres.svg create mode 100644 source/.gitkeep create mode 100644 testing/preset_data.sql create mode 100644 testing/test.compose.yaml diff --git a/.env b/.env index 5532029..f1b4a85 100644 --- a/.env +++ b/.env @@ -1,20 +1,44 @@ -### data in which nextcloud is synched and used from restic as backup source -DATA_TO_BACKUP=/path/to/data/to/backup +### for run_backup.sh itself not for the image +# data in which provisioing places the data to backup, needs to be fullpath +# uncomment the line when you don't want to bind any data to the container (optional) +# when not explicitly set, defaults to creating a folder near the script and mount it into it +# options: 'fullpath' or 'none' +SCRIPT_DATA_TO_BACKUP=/path/to/data/to/backup -### storagebox settings +# data in which restic places the restore of the backup, needs to be fullpath +# when not set the run script will create a folder near the script and mount it into it +SCRIPT_RESTORE_DATA_TO=/path/to/restore/data + +# to access log files outside of the container, default not needed (optional) +SCRIPT_LOG_PATH=/path/to/logfiles + +### storagebox settings (needed) ENV_TARGET_DOMAIN=name_of_your_storagebox_domain ENV_TARGET_DOMAIN_USER=login_name_of_your_storagebox -ENV_PATH_OF_PRIVATE_KEY=/path/to/your/private/key +ENV_SSH_PRIVATE_KEY='your private key between single quotes' -### nextcloud -ENV_NC_URL=url_of_your_nextcloud -ENV_NC_USER=your_nextcloud_user -ENV_NC_PASS='password to your nextcloud between single quotes' - -### restic +### restic (needed) ENV_RESTIC_REPOSITORY_NAME=name_of_your_repository ENV_RESTIC_PASSWORD='password of your restic repository between single quotes' -### telegram +### telegram (needed) ENV_TELEGRAM_TOKEN=your_telegram_token -ENV_TELEGRAM_CHAT_ID=your_telegram_chat_id \ No newline at end of file +ENV_TELEGRAM_CHAT_ID=your_telegram_chat_id + +### CRON, defaults to 1am (needed) +ENV_CRON='0 1 * * *' + +### provision mode (needed) +### possible values: "postgres", "nextcloud", "none" +ENV_PROVISION_MODE="nextcloud" + +### nextcloud when used +ENV_NC_URL=url_of_your_nextcloud +ENV_NC_USER=your_nextcloud_user +ENV_NC_PASS='password to your nextcloud between single quotes' + +### postgres when used +ENV_POSTGRES_USER="user of your postgres database" +ENV_POSTGRES_PASSWORD="password of your postgres database" +ENV_POSTGRES_DATABASE="name of your postgres database" +ENV_POSTGRES_HOST="host of your postgres database" diff --git a/.gitignore b/.gitignore index 746e09a..33e236f 100644 --- a/.gitignore +++ b/.gitignore @@ -234,3 +234,4 @@ fabric.properties !.idea/runConfigurations # End of https://www.toptal.com/developers/gitignore/api/intellij+all +/local.env diff --git a/README.md b/README.md index 5ee5315..31196ac 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,108 @@ -[![Pipeline](https://github.com/realAP/backup/actions/workflows/pipeline.yml/badge.svg?branch=main)](https://github.com/realAP/backup/actions/workflows/pipeline.yml) -# backup -Use this image to sync your nextcloud to a local directory. This will be mounted in your container and used as input for restic. Restic creates a backup and upload it to your sftp host. -All of it in a container based manner. - -# Overview -![backup_overview.drawio.svg](backup_overview.drawio.svg) - - -# General -There are two modes. -1. Default mode is running the script without any arguments. - * e.g. `./run_backup.sh` -1. Argument mode. Use the script as you would use restic. The script will run the container in which restic is started and places every argument behind it. -You have access to all the environment variables set in the `.env` file. - * `./run_backup.sh snapshots` - * `./run_backup.sh init` - * and more... - -# How to use it -### Prerequisite -These things are needed: -* Nextcloud Server -* SFTP Server -* Docker -* a device where the backup is running - -Place your public key at the sftp server and use the private key to log into it. -> Currently, the private key should not have a password, it is not supported yet. -> This is not ideal, in further versions private key with passwords are supported and recommended to use. - -### Build image -1. clone the repository `https://github.com/realAP/backup.git` -1. `cd backup/docker` -1. build the docker image `docker build . -t restic` - -### Set .env file -Fill all variables in the `.env` file, it is provided with example values. - -### Create Repository -In first place a repository has to be created on the remote (sftp). - -1. `./run_backup.sh init` this will initialize a repository -1. exit the container - -### Run Backup -Just run the script `./run_backup.sh` it will immediately sync the nextcloud and creates a backup, the status report will be sent via telegram. -This repeats every day at 1am (default), feel free to adjust the `cron` variable in the run_backup.sh - -# Restore data -Just work as you would do it with restic. -When cloned the repo there is an empty folder called `restore` which is mounted into the container under /restore. -This folder can be used as target for your restored data. -Example: to get the latest snapshot from your data -`./run_backup.sh restore latest --target /restore` - ---- -# Debugging -The `run_backup.sh` has a flag -* `"DEBUG"=0` which disables -* `"DEBUG"=1` which enables debugging - * the container will start and opens a bash terminal - -# Logging -The `log` folder is mounted to `/var/log` which enables to have access to different kinds of log files. - -# ToDo -TBD - -# Why | Motivation -I have used https://duplicati.com/ which i can recommend. -My problem is duplicati is not supported for my hardware anymore. -This is the reason for this project. - -I have nextcloud hosted by hetzner (https://www.hetzner.com/storage/storage-share/) and storagebox (https://www.hetzner.com/storage/storage-box/) a sftp hosted platform. -The backup should work without hetzner it just needs a nextcloud server and sftp access. +[![Pipeline](https://github.com/realAP/backup/actions/workflows/pipeline.yml/badge.svg?branch=main)](https://github.com/realAP/backup/actions/workflows/pipeline.yml) +[![Docker Image Version](https://img.shields.io/docker/v/devp1337/backup?sort=semver)](https://hub.docker.com/r/devp1337/backup) +# backup +Use this image to load your data from a **provider** into your **binded** folder. This folder will be used as input for restic. Restic creates a backup and upload it to your sftp host. +You can configure a cron job to run the backup every day at a specific time and telegram will be used to send status reports. +All of it in a container based manner. + +## Overview +![backup_overview.drawio.svg](resources/backup_overview.svg) + +## Prerequisite +These things are needed: +* Provider (Nextcloud, PostgreSQL, None) +* SFTP Server to store (e.g. Hetzner Storagebox) +* Docker +* a device where backup is running + +## Configuration +**1. Bind** +Restic looks for a folder `/source` to back up data. The behavior of this folder depends on the mode you select at the beginning of the script: + +- **Bind Mounting**: + - You can mount a folder from the host system into `/source`. + - This approach is recommended because the data is persistent even if the container is deleted. + +- **Internal Folder**: + - If no folder is mounted, the `/source` directory exists within the container. + - **Warning**: By default the container deletes itself after it stops. Any data stored in the container's `/source` directory will be lost. +--- +**2. Provider** +The provider mode dynamically writes data into the `/source` directory before restic creates a backup. This mode is useful when the data is generated by another service or application. + +Providers currently supported: +- **Nextcloud**: Synchronizes files into `/source`. +- **PostgreSQL**: Dumps database content into `/source`. +- **None**: No additional data is written to `/source`. + +--- +**3. Environment Variables** + +There is a `.env` file where you can set all needed variables. +It describes every variable and provides an example value. + + +## How to backup +### 1. Place your public key at the sftp server and use the private key to log into it. +When used with Hetzner Storagebox, follow this [guide](https://docs.hetzner.com/storage/storage-box/backup-space-ssh-keys) +> Currently, the private key should not have a password, it is not supported yet. + +### 2. Download the script and .env file +* `curl -O https://raw.githubusercontent.com/realAP/backup/main/run_backup.sh` +* `curl -0 https://raw.githubusercontent.com/realAP/backup/main/.env` +* make the script executable `chmod +x run_backup.sh` + +### 3. Set .env file +Fill all needed variables in the `.env` file, it is provided with example values. + +### 4. Run Backup +Just run the script `./run_backup.sh` the backup will immediately start and repeat every day at 1am (default). +For the first run, the script will initialize the restic repository. + +## Restore data +Just work as you would do it with restic. + +Example: to get the latest snapshot from your data +* `./run_backup.sh restore latest --target /restore` + * this will restore the latest snapshot to the `/restore` folder in the container which is binded to the host +> as target use `/restore` to restore the data to the binded folder. For more information read the .env file example + + +## Operations +You can use the script in two ways: +1. Default is running the script without any arguments. As shown in the example above. + * e.g. `./run_backup.sh` +1. With arguments. Use the script as you would use restic. The script will run the container in which restic is started and places every argument behind it. + You have access to all the environment variables set in the `.env` file. Remember `/restore` is **always** binded to the host. And `/source` folder is binded to the host when not set to `none`. + * `./run_backup.sh snapshots` + * `./run_backup.sh init` + * and more... + > Attention: the script will run the container which will be deleted after the command is executed. The data in the container is lost. + +--- +# Debugging +When you want to debug the container you can run the container in interactive mode. +Just run the script as follows `./run_backup.sh DEBUG` and the container will start in interactive mode. +This is just used in the development phase. + +# Logging +You can bind the `/var/log` folder to your host to get the logs. For this set the `SCRIPT_LOG_PATH` variable in the `.env` file. +When the variable is not set then log files will be stored in the container. +This is just for debugging purposes. + +# Why | Motivation +I have used https://duplicati.com/ which i can recommend. +My problem is duplicati is not supported for my hardware anymore. +This is the reason for this project. + +I have nextcloud hosted by hetzner (https://www.hetzner.com/storage/storage-share/) and storagebox (https://www.hetzner.com/storage/storage-box/) a sftp hosted platform. +The backup should work without hetzner it just needs a nextcloud server and sftp access. + +## Example for Provider Nextcloud +![backup_overview_nextcloud.svg](resources/backup_overview_nextcloud.svg) +* TBD + +## Example for Provider PostgreSQL +![backup_overview_postgres.svg](resources/backup_overview_postgres.svg) +* TBD + diff --git a/VERSION b/VERSION index cbaf3b3..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/backup_overview.drawio.svg b/backup_overview.drawio.svg deleted file mode 100644 index 69aa09a..0000000 --- a/backup_overview.drawio.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
data
cron job
localhost
data
data
backup
triggers
docker
storageshare
storagebox
user
status
status
run_backup.sh
.env
start
uses
start run_backup.sh
 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 9cc7417..9bf7143 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,19 +1,26 @@ FROM ubuntu:24.04 -RUN apt update && apt install cron restic nextcloud-desktop-cmd ssh curl -y +RUN apt update && apt install cron restic nextcloud-desktop-cmd ssh curl postgresql-client -y # user permissions? ADD backup.sh /bin ADD entry.sh /bin ADD manager.sh /bin ADD nextcloud.sh /bin +ADD postgres_backup.sh /bin +ADD postgres_restore.sh /bin +ADD prepare.sh /bin +ADD prepare_ssh.sh /bin ADD telegram.sh /bin -ADD create_ssh_config.sh /bin + RUN chmod 0744 /bin/backup.sh RUN chmod 0744 /bin/entry.sh RUN chmod 0744 /bin/manager.sh RUN chmod 0744 /bin/nextcloud.sh +RUN chmod 0744 /bin/postgres_backup.sh +RUN chmod 0744 /bin/postgres_restore.sh +RUN chmod 0744 /bin/prepare.sh +RUN chmod 0744 /bin/prepare_ssh.sh RUN chmod 0744 /bin/telegram.sh -RUN chmod 0744 /bin/create_ssh_config.sh ENTRYPOINT ["entry.sh"] diff --git a/docker/backup.sh b/docker/backup.sh index 5c41138..301f2f4 100644 --- a/docker/backup.sh +++ b/docker/backup.sh @@ -1,27 +1,27 @@ -#!/bin/bash -l -set -e -echo "=========LET'S DO THIS, BACKUP THE SHIT!!!=========" -date -hasRepo="restic cat config" -$hasRepo || exit 1 - -echo "+------------------------------------------+" -echo "|::::::::::::::::call backup:::::::::::::::|" -echo "+------------------------------------------+" -restic backup /source - -echo "+------------------------------------------+" -echo "|::::::::::::::::call check::::::::::::::::|" -echo "+------------------------------------------+" -restic check - -echo "+------------------------------------------+" -echo "|::::::::::::::::call forget:::::::::::::::|" -echo "+------------------------------------------+" -restic forget --keep-weekly 52 --keep-monthly 12 --keep-yearly 100 - -echo "+------------------------------------------+" -echo "|::::::::::::::::call check::::::::::::::::|" -echo "+------------------------------------------+" -restic check -echo "==================DONE=====================" +#!/usr/bin/env bash +set -e +echo "=========LET'S DO THIS, BACKUP THE SHIT!!!=========" +date +hasRepo="restic cat config" +$hasRepo || restic init || exit 1 + +echo "+------------------------------------------+" +echo "|::::::::::::::::call backup:::::::::::::::|" +echo "+------------------------------------------+" +restic backup /source + +echo "+------------------------------------------+" +echo "|::::::::::::::::call check::::::::::::::::|" +echo "+------------------------------------------+" +restic check + +echo "+------------------------------------------+" +echo "|::::::::::::::::call forget:::::::::::::::|" +echo "+------------------------------------------+" +restic forget --keep-weekly 52 --keep-monthly 12 --keep-yearly 100 + +echo "+------------------------------------------+" +echo "|::::::::::::::::call check::::::::::::::::|" +echo "+------------------------------------------+" +restic check +echo "==================DONE=====================" diff --git a/docker/entry.sh b/docker/entry.sh index be656c2..006883f 100644 --- a/docker/entry.sh +++ b/docker/entry.sh @@ -1,35 +1,31 @@ -#!/bin/bash -l - -##### CONECTION SPECIFIC FOR REMOTE HOST -# ssh specific -# makes an entry into known hosts so it will not stop at prompt -ssh-keyscan -t rsa $TARGET_DOMAIN > /etc/ssh/ssh_known_hosts - -# https://stackoverflow.com/a/48651061 -# is needed to save the container environment variables which for later usage from script for cron jobs -export RESTIC_REPOSITORY=sftp:storagebox:${RESTIC_REPOSITORY_NAME} -declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env - -# create ssh config for easier access in backup.sh -create_ssh_config.sh - -#create crontab -# https://stackoverflow.com/a/47960145 comment section recommended the proc for cron output is usable for docker logs -(echo " -SHELL=/bin/bash -BASH_ENV=/container.env -$CRON manager.sh > /proc/1/fd/1 2> /proc/1/fd/2") | crontab - - -if [[ "$DEBUG" == "1" ]];then - echo "Debug mode enabled. Starting /bin/bash..." - exec bin/bash -fi - -# argument mode -if [[ "$#" > 0 ]]; then - restic $@ -else -# default mode - manager.sh - cron -f -fi +#!/bin/bash -l + +# Build repository variable which is later used for restic to determine repo +export RESTIC_REPOSITORY=sftp:storagebox:${RESTIC_REPOSITORY_NAME} + +prepare.sh + +# https://stackoverflow.com/a/48651061 +# is needed to save the container environment variables which for later usage from script for cron jobs +declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env + +#create crontab +# https://stackoverflow.com/a/47960145 comment section recommended the proc for cron output is usable for docker logs +(echo " +SHELL=/bin/bash +BASH_ENV=/container.env +$CRON manager.sh > /proc/1/fd/1 2> /proc/1/fd/2") | crontab - + +if [[ "$DEBUG" == "1" ]];then + echo "Debug mode enabled. Starting /bin/bash..." + exec bin/bash +fi + +# argument mode +if [[ $# -gt 0 ]]; then + restic "${@}" +else +# default mode for provisioning + manager.sh + cron -f +fi diff --git a/docker/manager.sh b/docker/manager.sh index 8e719f2..e1e1cf9 100644 --- a/docker/manager.sh +++ b/docker/manager.sh @@ -1,24 +1,32 @@ -#!/usr/bin/bash - -nextcloudLastLogfile="/var/log/nextcloud-last.log" -backupLastLogfile="/var/log/backup-last.log" - -nextcloud.sh &> ${nextcloudLastLogfile} -status_nextcloud=$? -backup.sh &> ${backupLastLogfile} -status_backup=$? - -telegram.sh ${nextcloudLastLogfile} -telegram.sh ${backupLastLogfile} - -if [[ $status_nextcloud == 0 ]]; then - telegram.sh "Nextcloud: Successfull" -else - telegram.sh "Nextcloud: Failure" -fi - -if [[ $status_backup == 0 ]]; then - telegram.sh "Backup Successfull" -else - telegram.sh "Backup: Failure" -fi +#!/usr/bin/bash + +provisionLastLogfile="/var/log/provision-last.log" +backupLastLogfile="/var/log/backup-last.log" + +# check which provision mode to execute +if [[ "$PROVISION_MODE" == "nextcloud" ]]; then + nextcloud.sh &> ${provisionLastLogfile} + status_provision=$? +fi +if [[ "$PROVISION_MODE" == "postgres" ]]; then + postgres_backup.sh &> ${provisionLastLogfile} + status_provision=$? +fi + +backup.sh &> ${backupLastLogfile} +status_backup=$? + +telegram.sh ${provisionLastLogfile} +telegram.sh ${backupLastLogfile} + +if [[ $status_provision == 0 ]]; then + telegram.sh "$PROVISION_MODE: Successful" +else + telegram.sh "$PROVISION_MODE: Failure" +fi + +if [[ $status_backup == 0 ]]; then + telegram.sh "Backup: Successfull" +else + telegram.sh "Backup: Failure" +fi diff --git a/docker/postgres_backup.sh b/docker/postgres_backup.sh new file mode 100644 index 0000000..14170a7 --- /dev/null +++ b/docker/postgres_backup.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +date +echo "+------------------------------------------------------+" +echo "|::::::::::::::::execute postgres_backup:::::::::::::::|" +echo "+------------------------------------------------------+" +PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DATABASE} -F c -f /source/backup.dump +echo "==========================DONE==========================" diff --git a/docker/postgres_restore.sh b/docker/postgres_restore.sh new file mode 100644 index 0000000..4e6faf6 --- /dev/null +++ b/docker/postgres_restore.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +date +echo "+------------------------------------------------------+" +echo "|::::::::::::::::execute postgres_restore:::::::::::::::|" +echo "+------------------------------------------------------+" +PGPASSWORD="${POSTGRES_PASSWORD}" pg_restore --clean -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DATABASE} -v /restore/source/backup.dump +echo "==========================DONE==========================" diff --git a/docker/prepare.sh b/docker/prepare.sh new file mode 100644 index 0000000..c3503ed --- /dev/null +++ b/docker/prepare.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -e + +### check if all environment variables are set +# Function to check if a variable is set +check_var() { + local var_name="$1" + local var_value="${!var_name}" + if [ -z "$var_value" ]; then + echo "Error: Environment variable '$var_name' is not set." + exit 1 + fi +} + +# Common environment variables +check_var "TARGET_DOMAIN" +check_var "TARGET_DOMAIN_USER" +check_var "SSH_PRIVATE_KEY" +check_var "RESTIC_REPOSITORY_NAME" +check_var "RESTIC_PASSWORD" +check_var "TELEGRAM_TOKEN" +check_var "TELEGRAM_CHAT_ID" +check_var "CRON" +check_var "PROVISION_MODE" + +# Check environment variables based on provision mode +case "$PROVISION_MODE" in + postgres) + echo "Provision mode: postgres" + check_var "POSTGRES_USER" + check_var "POSTGRES_PASSWORD" + check_var "POSTGRES_DATABASE" + check_var "POSTGRES_HOST" + ;; + nextcloud) + echo "Provision mode: nextcloud" + check_var "NC_URL" + check_var "NC_USER" + check_var "NC_PASS" + ;; + *) + echo "Error: Invalid provision mode '$PROVISION_MODE'. Expected 'postgres' or 'nextcloud'." + exit 1 + ;; +esac + +echo "All required environment variables are set." + +### create folder which are over written when mounted +mkdir -p /source +mkdir -p /restore + +## Backup specific parts not plugins +prepare_ssh.sh + +## Feature specific parts +# currently nothing here diff --git a/docker/create_ssh_config.sh b/docker/prepare_ssh.sh similarity index 57% rename from docker/create_ssh_config.sh rename to docker/prepare_ssh.sh index 88f617b..f4cbf92 100644 --- a/docker/create_ssh_config.sh +++ b/docker/prepare_ssh.sh @@ -1,16 +1,23 @@ -#!/usr/bin/bash -set -e - -# Check if all arguments are provided -if [ -z "$TARGET_DOMAIN" ] || [ -z "$TARGET_DOMAIN_USER" ] || [ -z "$PATH_OF_PRIVATE_KEY" ]; then - echo "Error: Variable: ${TARGET_DOMAIN} ${TARGET_DOMAIN_USER} ${PATH_OF_PRIVATE_KEY} not defined" - exit 1 -fi - -# Create the ssh_config file -cat < /etc/ssh/ssh_config -Host storagebox - Hostname $TARGET_DOMAIN - User $TARGET_DOMAIN_USER - IdentityFile /private_key -EOL \ No newline at end of file +#!/usr/bin/env bash +set -e + +# Check if all arguments are provided +if [ -z "$TARGET_DOMAIN" ] || [ -z "$TARGET_DOMAIN_USER" ] || [ -z "$SSH_PRIVATE_KEY" ]; then + echo "Error: Variable: ${TARGET_DOMAIN} ${TARGET_DOMAIN_USER} ${SSH_PRIVATE_KEY} not defined" + exit 1 +fi + +# make the host known +ssh-keyscan -t rsa $TARGET_DOMAIN > /etc/ssh/ssh_known_hosts + +# Create the private key file +echo "$SSH_PRIVATE_KEY" > /private_key +chmod 600 /private_key + +# Create the ssh_config file +cat < /etc/ssh/ssh_config +Host storagebox + Hostname $TARGET_DOMAIN + User $TARGET_DOMAIN_USER + IdentityFile /private_key +EOL diff --git a/docker/telegram.sh b/docker/telegram.sh index 1387727..4d6bcba 100755 --- a/docker/telegram.sh +++ b/docker/telegram.sh @@ -1,45 +1,45 @@ -#!/usr/bin/bash -# Replace with your bot token and chat ID - -BOT_TOKEN=${TELEGRAM_TOKEN} -CHAT_ID=${TELEGRAM_CHAT_ID} - -# Check if the file path is provided as the first argument -if [ -z "$1" ]; then - echo "Usage: $0 path/to/your/logfile.log | or message to send" - exit 1 -fi - -IS_MESSAGE_OR_FILE_PATH="$1" - -# Check if the log file exists -if [ -f "$IS_MESSAGE_OR_FILE_PATH" ]; then - LOG_CONTENT=$(cat "$IS_MESSAGE_OR_FILE_PATH") -else - LOG_CONTENT="${IS_MESSAGE_OR_FILE_PATH}" -fi - -# Define the endpoint -URL="https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" - -# Telegram has a limit of 4096 characters per message, so split the message if necessary -MAX_LENGTH=4096 - -send_message() { - local message=$1 - curl -o /dev/null -s -X POST $URL -d chat_id=$CHAT_ID -d text="$message" -} - -# Check the length of the log content and send it in chunks if necessary -if [ ${#LOG_CONTENT} -le $MAX_LENGTH ]; then - send_message "${LOG_CONTENT}" -else - echo "Log content is too large, splitting into multiple messages" - start=1 - while [ $start -le ${#LOG_CONTENT} ]; do - chunk=$(echo "$LOG_CONTENT" | cut -c $start-$(($start+$MAX_LENGTH-1))) - send_message "${chunk}" - start=$(($start+$MAX_LENGTH)) - sleep 1 # To avoid hitting Telegram's rate limits - done -fi +#!/usr/bin/env bash + +# Replace with your bot token and chat ID +BOT_TOKEN=${TELEGRAM_TOKEN} +CHAT_ID=${TELEGRAM_CHAT_ID} + +# Check if the file path or message is provided as the first argument +if [ -z "$1" ]; then + echo "Usage: $0 path/to/your/logfile.log | message to send" + exit 1 +fi + +IS_MESSAGE_OR_FILE_PATH="$1" + +# Check if the input is a file or a message +if [ -f "$IS_MESSAGE_OR_FILE_PATH" ]; then + LOG_CONTENT=$(cat "$IS_MESSAGE_OR_FILE_PATH") +else + LOG_CONTENT="${IS_MESSAGE_OR_FILE_PATH}" +fi + +# Define the endpoint +URL="https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" + +# Telegram has a limit of 4096 characters per message +MAX_LENGTH=4096 + +send_message() { + local message=$1 + curl -o /dev/null -s -X POST $URL -d chat_id=$CHAT_ID -d text="$message" +} + +# Send the message in chunks if it exceeds MAX_LENGTH +if [ ${#LOG_CONTENT} -le $MAX_LENGTH ]; then + send_message "${LOG_CONTENT}" +else + echo "Log content is too large, splitting into multiple messages" + start=0 + while [ $start -lt ${#LOG_CONTENT} ]; do + chunk="${LOG_CONTENT:start:MAX_LENGTH}" + send_message "${chunk}" + start=$(($start + $MAX_LENGTH)) + sleep 1 # To avoid hitting Telegram's rate limits + done +fi diff --git a/resources/backup_overview.drawio b/resources/backup_overview.drawio new file mode 100644 index 0000000..27a029f --- /dev/null +++ b/resources/backup_overview.drawio @@ -0,0 +1,195 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/backup_overview.svg b/resources/backup_overview.svg new file mode 100644 index 0000000..ebf949a --- /dev/null +++ b/resources/backup_overview.svg @@ -0,0 +1,4 @@ + + + +
data
cron job
localhost
data
data
backup
triggers
docker
storagebox
user
status
status
run_backup.sh
.env
start
uses
start run_backup.sh
/source
provider
data
Binds
Provider
 \ No newline at end of file diff --git a/backup_overview.drawio b/resources/backup_overview_nextcloud.drawio similarity index 84% rename from backup_overview.drawio rename to resources/backup_overview_nextcloud.drawio index 9f7c113..7dfdb9d 100644 --- a/backup_overview.drawio +++ b/resources/backup_overview_nextcloud.drawio @@ -1,174 +1,177 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/backup_overview_nextcloud.svg b/resources/backup_overview_nextcloud.svg new file mode 100644 index 0000000..2c49256 --- /dev/null +++ b/resources/backup_overview_nextcloud.svg @@ -0,0 +1,4 @@ + + + +
data
cron job
localhost
data
data
backup
triggers
docker
storageshare
storagebox
user
status
status
run_backup.sh
.env
start
uses
start run_backup.sh
/source
 \ No newline at end of file diff --git a/resources/backup_overview_postgres.drawio b/resources/backup_overview_postgres.drawio new file mode 100644 index 0000000..7c189e0 --- /dev/null +++ b/resources/backup_overview_postgres.drawio @@ -0,0 +1,177 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/backup_overview_postgres.svg b/resources/backup_overview_postgres.svg new file mode 100644 index 0000000..12b9240 --- /dev/null +++ b/resources/backup_overview_postgres.svg @@ -0,0 +1,4 @@ + + + +
data
cron job
localhost
data
data
backup
triggers
docker
database
storagebox
user
status
status
run_backup.sh
.env
start
uses
start run_backup.sh
pg_dump
/source
 \ No newline at end of file diff --git a/run_backup.sh b/run_backup.sh index 6d2374e..ad6895b 100755 --- a/run_backup.sh +++ b/run_backup.sh @@ -1,20 +1,80 @@ -#!/bin/env bash -source .env -docker run --restart=always -it --hostname restic \ - -v ./restore:/restore \ - -v ./log/:/var/log \ - -v "${DATA_TO_BACKUP}":/source \ - -v "${ENV_PATH_OF_PRIVATE_KEY}":/private_key \ - -e "DEBUG"=0 \ - -e "TARGET_DOMAIN"="${ENV_TARGET_DOMAIN}" \ - -e "TARGET_DOMAIN_USER"="${ENV_TARGET_DOMAIN_USER}" \ - -e "PATH_OF_PRIVATE_KEY"="${ENV_PATH_OF_PRIVATE_KEY}" \ - -e "NC_URL"="${ENV_NC_URL}" \ - -e "NC_USER"="${ENV_NC_USER}" \ - -e "NC_PASS"="${ENV_NC_PASS}" \ - -e "RESTIC_REPOSITORY_NAME"="${ENV_RESTIC_REPOSITORY_NAME}" \ - -e "RESTIC_PASSWORD"="${ENV_RESTIC_PASSWORD}" \ - -e "TELEGRAM_TOKEN"="${ENV_TELEGRAM_TOKEN}" \ - -e "TELEGRAM_CHAT_ID"="${ENV_TELEGRAM_CHAT_ID}" \ - -e "CRON"='0 1 * * *' \ - restic $@ \ No newline at end of file +#!/bin/env bash + +### JUST FOR DEBUGGING +# Default values +RESTART_OPTION="--restart=always" +_DEBUG=0 +# Check the first argument +if [[ "$#" -gt 0 ]]; then + case $1 in + DEBUG) + RESTART_OPTION="-it --rm" + _DEBUG=1 + shift + ;; + *) + RESTART_OPTION="--rm" + ;; + esac +fi +### +source local.env +### + +### create restore folder at the same level as the script +# Default paths based on the location of run_backup.sh +SCRIPT_DIR=$(dirname "$(realpath "$0")") + +# Handle SCRIPT_RESTORE_DATA_TO +if [[ -z "$SCRIPT_RESTORE_DATA_TO" ]]; then + SCRIPT_RESTORE_DATA_TO="$SCRIPT_DIR/restore" + mkdir -p "$SCRIPT_RESTORE_DATA_TO" + echo "Defaulting SCRIPT_RESTORE_DATA_TO to $SCRIPT_RESTORE_DATA_TO" +fi + +# Handle SCRIPT_LOG_DIR +if [[ -n "$SCRIPT_LOG_DIR" ]]; then + LOG_MOUNT="-v ${SCRIPT_LOG_DIR}:/var/log" + echo "Mounting log directory: $SCRIPT_LOG_DIR" +else + LOG_MOUNT="" + echo "Skipping log directory mount" +fi + +# Handle SCRIPT_DATA_TO_BACKUP +if [[ "$SCRIPT_DATA_TO_BACKUP" == "none" ]]; then + SOURCE_MOUNT="" + echo "Skipping bind a folder to container as source, SCRIPT_DATA_TO_BACKUP is set to 'none'" +elif [[ -z "$SCRIPT_DATA_TO_BACKUP" ]]; then + SCRIPT_DATA_TO_BACKUP="$SCRIPT_DIR/source" + mkdir -p "$SCRIPT_DATA_TO_BACKUP" + SOURCE_MOUNT="-v ${SCRIPT_DATA_TO_BACKUP}:/source" + echo "Defaulting SCRIPT_DATA_TO_BACKUP to $SCRIPT_DATA_TO_BACKUP" +else + SOURCE_MOUNT="-v ${SCRIPT_DATA_TO_BACKUP}:/source" + echo "Mounting data to backup directory: $SCRIPT_DATA_TO_BACKUP" +fi + + +docker run ${RESTART_OPTION} --network=backup_default --hostname backup \ + -v "${SCRIPT_RESTORE_DATA_TO}":/restore \ + $LOG_MOUNT \ + $SOURCE_MOUNT \ + -e "DEBUG"="${_DEBUG}" \ + -e "TARGET_DOMAIN"="${ENV_TARGET_DOMAIN}" \ + -e "TARGET_DOMAIN_USER"="${ENV_TARGET_DOMAIN_USER}" \ + -e "SSH_PRIVATE_KEY"="${ENV_SSH_PRIVATE_KEY}" \ + -e "NC_URL"="${ENV_NC_URL}" \ + -e "NC_USER"="${ENV_NC_USER}" \ + -e "NC_PASS"="${ENV_NC_PASS}" \ + -e "RESTIC_REPOSITORY_NAME"="${ENV_RESTIC_REPOSITORY_NAME}" \ + -e "RESTIC_PASSWORD"="${ENV_RESTIC_PASSWORD}" \ + -e "TELEGRAM_TOKEN"="${ENV_TELEGRAM_TOKEN}" \ + -e "TELEGRAM_CHAT_ID"="${ENV_TELEGRAM_CHAT_ID}" \ + -e POSTGRES_USER="${ENV_POSTGRES_USER}" \ + -e POSTGRES_PASSWORD="${ENV_POSTGRES_PASSWORD}" \ + -e POSTGRES_DATABASE="${ENV_POSTGRES_DATABASE}" \ + -e POSTGRES_HOST="${ENV_POSTGRES_HOST}" \ + -e PROVISION_MODE="${ENV_PROVISION_MODE}" \ + -e "CRON"="${ENV_CRON}" \ + devp1337/backup:latest "${@}" diff --git a/source/.gitkeep b/source/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/testing/preset_data.sql b/testing/preset_data.sql new file mode 100644 index 0000000..2f2d6cc --- /dev/null +++ b/testing/preset_data.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS example_table ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO example_table (name) VALUES ('Sample Data'); diff --git a/testing/test.compose.yaml b/testing/test.compose.yaml new file mode 100644 index 0000000..1098597 --- /dev/null +++ b/testing/test.compose.yaml @@ -0,0 +1,13 @@ +version: '3.8' + +services: + db: + image: postgres:16.4 + container_name: postgres + restart: always + ports: + - "5444:5432" + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: database