From 3abd7c1786724292323fbd79b47d7b73269c80c6 Mon Sep 17 00:00:00 2001 From: John Heenan Date: Thu, 12 Sep 2024 02:26:11 +1000 Subject: [PATCH 1/8] One line fix for failing to backup two lnd database files The deleted line removes the failure to backup lnd database files channel.db and sphinxreplay.db. --- btcpay-backup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index a313cb93d..74b0e472c 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -110,7 +110,6 @@ echo "ℹ️ Archiving files in $(pwd)…" --exclude="volumes/generated_mariadb_datadir" \ --exclude="volumes/generated_postgres_datadir" \ --exclude="volumes/generated_electrumx_datadir" \ - --exclude="volumes/generated_lnd_bitcoin_datadir/_data/data/graph" \ --exclude="volumes/generated_clightning_bitcoin_datadir/_data/lightning-rpc" \ --exclude="**/logs/*" \ -cvzf $backup_path $postgres_dump_name $mariadb_dump_name volumes/generated_* From 5f626286ca3c205b4d4cb9c37a14948a270148eb Mon Sep 17 00:00:00 2001 From: John Heenan Date: Fri, 13 Sep 2024 10:41:46 +1000 Subject: [PATCH 2/8] Add in option for btcpay_backup to backup full lnd state for migration --- btcpay-backup.sh | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index 74b0e472c..00422e1e7 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -16,6 +16,51 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi + +function display_help () { +cat <<-END +Usage: +------ + +Backup postgres database and docker volumes without chain states + --lnd : For migration, also backup full lnd channel state and + leave lnd disabled. When this option is used, do not + reuse this backup when lnd is enabled again. Otherwise + the lnd state may become toxic with loss of some or all funds. + +END +} + +EXCLUDE_LND_GRAPH=' --exclude="volumes/generated_lnd_bitcoin_datadir/_data/data/graph"' + +while (( "$#" )); do + case "$1" in + -h|--help) + display_help + exit + ;; + --lnd) + EXCLUDE_LND_GRAPH="" + shift 1 + ;; + --) # end argument parsing + shift + break + ;; + -*|--*=) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + display_help + return + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + + + # preparation if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OS @@ -42,6 +87,7 @@ fi cd $btcpay_dir . helpers.sh + # Postgres database postgres_container=$(docker ps -a -q -f "name=postgres_1") if [ -z "$postgres_container" ]; then @@ -85,6 +131,13 @@ if [ ! -z "$mariadb_container" ]; then } fi +# If doing full lnd backup and lnd is enabled then disable lnd from restarting +if [ -z "$EXCLUDE_LND_GRAPH" ] && [[ "$BTCPAYGEN_LIGHTNING" == "lnd" ]]; then + export BTCPAYGEN_LIGHTNING="none" + btcpay_update_docker_env + fi +fi + # BTCPay Server backup printf "\nℹ️ Stopping BTCPay Server …\n\n" btcpay_down @@ -110,6 +163,7 @@ echo "ℹ️ Archiving files in $(pwd)…" --exclude="volumes/generated_mariadb_datadir" \ --exclude="volumes/generated_postgres_datadir" \ --exclude="volumes/generated_electrumx_datadir" \ + "$EXCLUDE_LND_GRAPH" \ --exclude="volumes/generated_clightning_bitcoin_datadir/_data/lightning-rpc" \ --exclude="**/logs/*" \ -cvzf $backup_path $postgres_dump_name $mariadb_dump_name volumes/generated_* From 7880a53e17c039ae8d316df8b801e1577a1b8c7b Mon Sep 17 00:00:00 2001 From: John Heenan Date: Fri, 13 Sep 2024 18:18:36 +1000 Subject: [PATCH 3/8] Tested version of previous commit that ignores exclusion. There may be container start/stop async issues. --- btcpay-backup.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index 00422e1e7..2552534b6 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -31,7 +31,7 @@ Backup postgres database and docker volumes without chain states END } -EXCLUDE_LND_GRAPH=' --exclude="volumes/generated_lnd_bitcoin_datadir/_data/data/graph"' +EXCLUDE_LND_GRAPH="volumes/generated_lnd_bitcoin_datadir/_data/data/graph" while (( "$#" )); do case "$1" in @@ -40,8 +40,8 @@ while (( "$#" )); do exit ;; --lnd) - EXCLUDE_LND_GRAPH="" - shift 1 + EXCLUDE_LND_GRAPH="$EXCLUDE_LND_GRAPH/false" # now does not exclude + shift ;; --) # end argument parsing shift @@ -87,7 +87,6 @@ fi cd $btcpay_dir . helpers.sh - # Postgres database postgres_container=$(docker ps -a -q -f "name=postgres_1") if [ -z "$postgres_container" ]; then @@ -132,10 +131,12 @@ if [ ! -z "$mariadb_container" ]; then fi # If doing full lnd backup and lnd is enabled then disable lnd from restarting -if [ -z "$EXCLUDE_LND_GRAPH" ] && [[ "$BTCPAYGEN_LIGHTNING" == "lnd" ]]; then - export BTCPAYGEN_LIGHTNING="none" - btcpay_update_docker_env - fi +if [[ "$EXCLUDE_LND_GRAPH" == *false ]] && [[ "$BTCPAYGEN_LIGHTNING" == lnd ]]; then + echo "Disabling lnd from starting up." + export BTCPAYGEN_LIGHTNING="none" + # btcpay_update_docker_env # does not work to dsiable knd container + # source ./btcpay-setup.sh --install-only # using docker stats, lnd container not stopped before backup taken + source ./btcpay-setup.sh -i # using docker stats, lnd container not stopped before backup taken fi # BTCPay Server backup @@ -163,7 +164,7 @@ echo "ℹ️ Archiving files in $(pwd)…" --exclude="volumes/generated_mariadb_datadir" \ --exclude="volumes/generated_postgres_datadir" \ --exclude="volumes/generated_electrumx_datadir" \ - "$EXCLUDE_LND_GRAPH" \ + --exclude="$EXCLUDE_LND_GRAPH" \ --exclude="volumes/generated_clightning_bitcoin_datadir/_data/lightning-rpc" \ --exclude="**/logs/*" \ -cvzf $backup_path $postgres_dump_name $mariadb_dump_name volumes/generated_* From 6bc7a0f1c433e568cc3351f341b10507cb3a54d4 Mon Sep 17 00:00:00 2001 From: John Heenan Date: Fri, 13 Sep 2024 19:22:23 +1000 Subject: [PATCH 4/8] Fix for lack of synch stopping container by reordering. --- btcpay-backup.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index 2552534b6..fac11a7d1 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -130,21 +130,22 @@ if [ ! -z "$mariadb_container" ]; then } fi +# BTCPay Server backup +printf "\nℹ️ Stopping BTCPay Server …\n\n" +btcpay_down + # If doing full lnd backup and lnd is enabled then disable lnd from restarting if [[ "$EXCLUDE_LND_GRAPH" == *false ]] && [[ "$BTCPAYGEN_LIGHTNING" == lnd ]]; then echo "Disabling lnd from starting up." + echo export BTCPAYGEN_LIGHTNING="none" - # btcpay_update_docker_env # does not work to dsiable knd container - # source ./btcpay-setup.sh --install-only # using docker stats, lnd container not stopped before backup taken - source ./btcpay-setup.sh -i # using docker stats, lnd container not stopped before backup taken + # btcpay_update_docker_env # does not work to disable lnd container + source ./btcpay-setup.sh --install-only # is there a better way to disable lnd using high level script commands? fi -# BTCPay Server backup -printf "\nℹ️ Stopping BTCPay Server …\n\n" -btcpay_down - printf "\n" cd $docker_dir + echo "ℹ️ Archiving files in $(pwd)…" { From ae5ead9656d2f5545cc7a0def32d46b2d922e315 Mon Sep 17 00:00:00 2001 From: John Heenan Date: Sat, 14 Sep 2024 19:08:16 +1000 Subject: [PATCH 5/8] change option name and add in another as per pull request discussion --- btcpay-backup.sh | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index fac11a7d1..6fb78e3cf 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -23,14 +23,23 @@ Usage: ------ Backup postgres database and docker volumes without chain states - --lnd : For migration, also backup full lnd channel state and - leave lnd disabled. When this option is used, do not + +For backup and restart with LND Static Channel Backup (SCB), if used, then run without options +./btcpay-backup.sh + +For migration purposes with full LND state backup and no restart (if backup succeeds) run as +./btcpay-backup.sh --include-lnd-graph --no-restart + + --include-lnd-graph For lnd migration purposes, backup full lnd channel state + and leave lnd disabled. When this option is used, do not reuse this backup when lnd is enabled again. Otherwise the lnd state may become toxic with loss of some or all funds. + --no-restart Do not restart btcpay if the backup succeeds END } +RESTART=true EXCLUDE_LND_GRAPH="volumes/generated_lnd_bitcoin_datadir/_data/data/graph" while (( "$#" )); do @@ -39,10 +48,14 @@ while (( "$#" )); do display_help exit ;; - --lnd) + --include-lnd-graph) EXCLUDE_LND_GRAPH="$EXCLUDE_LND_GRAPH/false" # now does not exclude shift ;; + --no-restart) + RESTART=false + shift + ;; --) # end argument parsing shift break @@ -50,7 +63,7 @@ while (( "$#" )); do -*|--*=) # unsupported flags echo "Error: Unsupported flag $1" >&2 display_help - return + exit 1 ;; *) # preserve positional arguments PARAMS="$PARAMS $1" @@ -139,13 +152,11 @@ if [[ "$EXCLUDE_LND_GRAPH" == *false ]] && [[ "$BTCPAYGEN_LIGHTNING" == lnd ]]; echo "Disabling lnd from starting up." echo export BTCPAYGEN_LIGHTNING="none" - # btcpay_update_docker_env # does not work to disable lnd container - source ./btcpay-setup.sh --install-only # is there a better way to disable lnd using high level script commands? + source ./btcpay-setup.sh --install-only fi printf "\n" cd $docker_dir - echo "ℹ️ Archiving files in $(pwd)…" { @@ -180,26 +191,35 @@ echo "ℹ️ Archiving files in $(pwd)…" backup_path="$backup_path.gpg" echo "✅ Encryption done." } || { - echo "🚨 Encrypting failed. Please check the error message above." printf "\nℹ️ Restarting BTCPay Server …\n\n" + echo "🚨 Encrypting failed. Please check the error message above." cd $btcpay_dir btcpay_up exit 1 } fi } || { + printf "\nℹ️ Restarting BTCPay Server …\n\n" echo "🚨 Archiving failed. Please check the error message above." - printf "\nℹ️ Restarting BTCPay Server …\n\n" cd $btcpay_dir btcpay_up exit 1 } -printf "\nℹ️ Restarting BTCPay Server …\n\n" cd $btcpay_dir -btcpay_up +if $START; then + printf "\nℹ️ Restarting BTCPay Server …\n\n" + btcpay_up +else + printf "\nℹ️ Not restarting BTCPay Server …\n\n" +fi printf "\nℹ️ Cleaning up …\n\n" rm $postgres_dump_path -printf "✅ Backup done => $backup_path\n\n" +printf "\n✅ Backup done => $backup_path\n\n" + +if [[ "$EXCLUDE_LND_GRAPH" == *false ]]; then + printf "\n✅ Full lnd state, if available, has been fully backed up\n" + printf "\nℹ️ This backup should only be restored once and only onto to another server\n\n" +fi From 263a30ec88e21ec68c9492a978ae284ad565c4f0 Mon Sep 17 00:00:00 2001 From: John Heenan Date: Sun, 15 Sep 2024 09:52:15 +1000 Subject: [PATCH 6/8] Fix variable name --- btcpay-backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index 6fb78e3cf..0c486d010 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -207,7 +207,7 @@ echo "ℹ️ Archiving files in $(pwd)…" } cd $btcpay_dir -if $START; then +if $RESTART; then printf "\nℹ️ Restarting BTCPay Server …\n\n" btcpay_up else From 90e85427c3af740c4f55ea58227985b8b2939674 Mon Sep 17 00:00:00 2001 From: John Heenan Date: Wed, 18 Sep 2024 13:43:16 +1000 Subject: [PATCH 7/8] Remove disabling LND and insert dire warnings for a condition that can cause fund loss with a restore --- btcpay-backup.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index 0c486d010..29dc793e4 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -31,9 +31,8 @@ For migration purposes with full LND state backup and no restart (if backup succ ./btcpay-backup.sh --include-lnd-graph --no-restart --include-lnd-graph For lnd migration purposes, backup full lnd channel state - and leave lnd disabled. When this option is used, do not - reuse this backup when lnd is enabled again. Otherwise - the lnd state may become toxic with loss of some or all funds. + When this option is used, do not reuse this backup when lnd is enabled again. + Otherwise the lnd state may become toxic with loss of some or all funds. --no-restart Do not restart btcpay if the backup succeeds END @@ -41,6 +40,9 @@ END RESTART=true EXCLUDE_LND_GRAPH="volumes/generated_lnd_bitcoin_datadir/_data/data/graph" +WARNING_LND_DIRE1A="🚨🚨🚨 LND is currently enabled and will be restarting 🚨🚨🚨" +WARNING_LND_DIRE1B="🚨🚨🚨 LND is currently enabled and has been resarted 🚨🚨🚨" +WARNING_LND_DIRE2="🚨🚨🚨 You cannot restore from this backup anywhere as is!!! 🚨🚨🚨" while (( "$#" )); do case "$1" in @@ -143,18 +145,15 @@ if [ ! -z "$mariadb_container" ]; then } fi +# If will be restarting, doing full lnd backup and lnd is enabled then give loud warning +if $RESTART && [[ "$EXCLUDE_LND_GRAPH" == *false ]] && [[ "$BTCPAYGEN_LIGHTNING" == lnd ]]; then + printf '\n%s\n%s\n\n' "$WARNING_LND_DIRE1A" "$WARNING_LND_DIRE2" +fi + # BTCPay Server backup printf "\nℹ️ Stopping BTCPay Server …\n\n" btcpay_down -# If doing full lnd backup and lnd is enabled then disable lnd from restarting -if [[ "$EXCLUDE_LND_GRAPH" == *false ]] && [[ "$BTCPAYGEN_LIGHTNING" == lnd ]]; then - echo "Disabling lnd from starting up." - echo - export BTCPAYGEN_LIGHTNING="none" - source ./btcpay-setup.sh --install-only -fi - printf "\n" cd $docker_dir echo "ℹ️ Archiving files in $(pwd)…" @@ -221,5 +220,9 @@ printf "\n✅ Backup done => $backup_path\n\n" if [[ "$EXCLUDE_LND_GRAPH" == *false ]]; then printf "\n✅ Full lnd state, if available, has been fully backed up\n" - printf "\nℹ️ This backup should only be restored once and only onto to another server\n\n" + if $RESTART && [[ "$BTCPAYGEN_LIGHTNING" == lnd ]]; then + printf '\n%s\n%s\n\n' "$WARNING_LND_DIRE1B" "$WARNING_LND_DIRE2" + else + printf "\nℹ️ This backup should only be restored once and only onto to another server\n\n" + fi fi From e0e2ec51b120e8fe5e09a2e6905bfd98d02794fb Mon Sep 17 00:00:00 2001 From: John Heenan Date: Wed, 18 Sep 2024 13:48:31 +1000 Subject: [PATCH 8/8] Correct word spelling in warning --- btcpay-backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btcpay-backup.sh b/btcpay-backup.sh index 29dc793e4..c4c0f5eea 100755 --- a/btcpay-backup.sh +++ b/btcpay-backup.sh @@ -41,7 +41,7 @@ END RESTART=true EXCLUDE_LND_GRAPH="volumes/generated_lnd_bitcoin_datadir/_data/data/graph" WARNING_LND_DIRE1A="🚨🚨🚨 LND is currently enabled and will be restarting 🚨🚨🚨" -WARNING_LND_DIRE1B="🚨🚨🚨 LND is currently enabled and has been resarted 🚨🚨🚨" +WARNING_LND_DIRE1B="🚨🚨🚨 LND is currently enabled and has been restarted 🚨🚨🚨" WARNING_LND_DIRE2="🚨🚨🚨 You cannot restore from this backup anywhere as is!!! 🚨🚨🚨" while (( "$#" )); do