From b26cad2ebfabe9e73c2b80e3a7821df2893e9119 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Fri, 22 Mar 2024 15:40:22 -0700
Subject: [PATCH 01/14] add preliminary daily sync config

---
 py/desitransfer/data/desi_utah_daily.txt | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 py/desitransfer/data/desi_utah_daily.txt

diff --git a/py/desitransfer/data/desi_utah_daily.txt b/py/desitransfer/data/desi_utah_daily.txt
new file mode 100644
index 0000000..1278bd6
--- /dev/null
+++ b/py/desitransfer/data/desi_utah_daily.txt
@@ -0,0 +1,8 @@
+dashboard
+exposure_tables
+processing_tables
+tiles/archive
+exposures-daily.*
+tiles.*
+tiles-daily.*
+tsnr-exposures.*

From 692b52aa4ec06087528f0d9c4dc304777ae644c6 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <baweaver@lbl.gov>
Date: Mon, 25 Mar 2024 15:13:18 -0600
Subject: [PATCH 02/14] update top-level includes

---
 py/desitransfer/data/desi_utah_daily.txt | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/py/desitransfer/data/desi_utah_daily.txt b/py/desitransfer/data/desi_utah_daily.txt
index 1278bd6..d847b95 100644
--- a/py/desitransfer/data/desi_utah_daily.txt
+++ b/py/desitransfer/data/desi_utah_daily.txt
@@ -1,6 +1,13 @@
+# dashboard/dashboard.html
 dashboard
+dashboard.html
+# exposure_tables/??????
 exposure_tables
+exposure_tables/??????
+exposure_table_*.csv
 processing_tables
+processing_table_daily-*.csv
+unprocessed_table_daily-*.csv
 tiles/archive
 exposures-daily.*
 tiles.*

From 777200b68656e83517cc0059ed10eecd98cd4a5c Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Mon, 25 Mar 2024 16:52:52 -0700
Subject: [PATCH 03/14] add notes

---
 py/desitransfer/data/desi_utah_daily.txt | 43 +++++++++++++++++++-----
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/py/desitransfer/data/desi_utah_daily.txt b/py/desitransfer/data/desi_utah_daily.txt
index d847b95..f3eceac 100644
--- a/py/desitransfer/data/desi_utah_daily.txt
+++ b/py/desitransfer/data/desi_utah_daily.txt
@@ -1,14 +1,39 @@
-# dashboard/dashboard.html
+#
+# cd desi/spectro/redux/daily/
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file ${HOME}/.desi --include-from ${DESITRANSFER}/py/desitransfer/data/desi_utah_daily.txt \
+#     --exclude \* rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/ ./
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/tiles/archive/ tiles/archive/
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/exposure_tables/ exposure_tables/
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/processing_tables/ processing_tables/
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file $HOME/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
+#     --include fibermap-\*.fits --include preproc-\*.fits.gz --include tilepix-\*.json --exclude \*
+#     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/preproc/ preproc/
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file $HOME/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
+#     --include \*.fits --include \*.fits.gz --include \*.csv --exclude \*
+#     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/exposures/ exposures/
+#
+# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
+#     --password-file $HOME/.desi --include 202403\?\? --include \*.fits --include \*.fits.gz --include \*.csv --include tmp --include old --exclude \* \
+#     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/calibnight/ calibnight/
+#
+# tiles/cumulative has a lot of symlinks into tiles/archive.
+# * Save all symlinks
+# * Save only last 30 days of non-symlinks.
+#
 dashboard
 dashboard.html
-# exposure_tables/??????
-exposure_tables
-exposure_tables/??????
-exposure_table_*.csv
-processing_tables
-processing_table_daily-*.csv
-unprocessed_table_daily-*.csv
-tiles/archive
 exposures-daily.*
 tiles.*
 tiles-daily.*

From ac2c5ae43f2b2df4f74e7f23f40691d7d425af17 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 26 Mar 2024 11:45:41 -0700
Subject: [PATCH 04/14] testing rsync commands

---
 py/desitransfer/data/desi_utah_daily.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/py/desitransfer/data/desi_utah_daily.txt b/py/desitransfer/data/desi_utah_daily.txt
index f3eceac..fca96c5 100644
--- a/py/desitransfer/data/desi_utah_daily.txt
+++ b/py/desitransfer/data/desi_utah_daily.txt
@@ -15,17 +15,17 @@
 #     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/processing_tables/ processing_tables/
 #
 # rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file $HOME/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
-#     --include fibermap-\*.fits --include preproc-\*.fits.gz --include tilepix-\*.json --exclude \*
+#     --password-file ${HOME}/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
+#     --include fibermap-\*.fits --include preproc-\*.fits.gz --include tilepix-\*.json --exclude \* \
 #     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/preproc/ preproc/
 #
 # rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file $HOME/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
+#     --password-file ${HOME}/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
 #     --include \*.fits --include \*.fits.gz --include \*.csv --exclude \*
 #     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/exposures/ exposures/
 #
 # rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file $HOME/.desi --include 202403\?\? --include \*.fits --include \*.fits.gz --include \*.csv --include tmp --include old --exclude \* \
+#     --password-file ${HOME}/.desi --include 202403\?\? --include \*.fits --include \*.fits.gz --include \*.csv --include tmp --include old --exclude \* \
 #     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/calibnight/ calibnight/
 #
 # tiles/cumulative has a lot of symlinks into tiles/archive.

From 15cf57b1516dd763473c182f186b8041713854e8 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 26 Mar 2024 12:12:30 -0700
Subject: [PATCH 05/14] add transfer script

---
 bin/desi_utah_transfer.sh | 72 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100755 bin/desi_utah_transfer.sh

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
new file mode 100755
index 0000000..557daef
--- /dev/null
+++ b/bin/desi_utah_transfer.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Parallel copy DESI mirror data to Utah.
+#
+#
+# Help!
+#
+function usage() {
+    local execName=$(basename $0)
+    (
+    echo "${execName} [-h] [-t] [-v]"
+    echo ""
+    echo "Parallel copy DESI mirror data to Utah."
+    echo ""
+    echo "     -h = Print this message and exit."
+    echo "     -t = Test mode.  Do not make any changes. Implies -v."
+    echo "     -v = Verbose mode. Print extra information."
+    echo ""
+    ) >&2
+}
+#
+# Environment variables.
+#
+if [[ -z "${DESITRANSFER}" ]]; then
+    echo "ERROR: DESITRANSFER is undefined!"
+    exit 1
+fi
+if [[ -z "${DESISYNC_HOSTNAME}" ]]; then
+    echo "ERROR: DESISYNC_HOSTNAME is undefined!"
+    exit 1
+fi
+if [[ -z "${DESI_ROOT}" ]]; then
+    echo "ERROR: DESI_ROOT is undefined!"
+    exit 1
+fi
+#
+# Configuration.
+#
+syn="/usr/bin/rsync --archive --verbose --delete --delete-after --no-motd --password-file ${HOME}/.desi"
+src=rsync://${DESISYNC_HOSTNAME}/desi
+dst=${DESI_ROOT}
+log_root=${HOME}/Documents/Logfiles
+verbose=/usr/bin/false
+test=/usr/bin/false
+while getopts htv argname; do
+    case ${argname} in
+        h) usage; exit 0 ;;
+        t) test=/usr/bin/true; verbose=/usr/bin/true ;;
+        v) verbose=/usr/bin/true ;;
+        *) usage; exit 1 ;;
+    esac
+done
+shift $((OPTIND - 1))
+#
+# Execute rsync commands.
+#
+for d in spectro/redux/daily spectro/redux/daily/calibnight \
+    spectro/redux/daily/exposure_tables spectro/redux/daily/exposures \
+    spectro/redux/daily/preproc spectro/redux/daily/processing_tables \
+    spectro/redux/daily/tiles/archive spectro/redux/daily/tiles/cumulative; do
+    case ${d} in
+        spectro/redux/daily) priority='nice'; exclude="--include-from ${DESITRANSFER}/py/desitransfer/data/desi_utah_daily.txt --exclude *" ;;
+        spectro/redux/daily/calibnight) priority='nice'; exclude='--include 202403?? --include *.fits --include *.fits.gz --include *.csv --include tmp --include old --exclude *' ;;
+        spectro/redux/daily/exposures) priority='nice'; exclude='--include 202403?? --include 202403??/???????? --include *.fits --include *.fits.gz --include *.csv --exclude *' ;;
+        spectro/redux/daily/preproc) priority='nice'; exclude='--include 202403?? --include 202403??/???????? --include fibermap-*.fits --include preproc-*.fits.gz --include tilepix-*.json --exclude *' ;;
+        *) priority=''; exclude='' ;;
+    esac
+    log=${log_root}/utah_$(tr '/' '_' <<<${d}).log
+    [[ -f ${log} ]] || touch ${log}
+    ${verbose} && echo "${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &"
+    ${test}    || ${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &
+done

From 6aee3bdda265eed89995b8f2f2dd6207778dbe19 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 26 Mar 2024 13:51:40 -0700
Subject: [PATCH 06/14] add GFA sync

---
 bin/desi_utah_transfer.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index 557daef..d2b522b 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -57,7 +57,9 @@ shift $((OPTIND - 1))
 for d in spectro/redux/daily spectro/redux/daily/calibnight \
     spectro/redux/daily/exposure_tables spectro/redux/daily/exposures \
     spectro/redux/daily/preproc spectro/redux/daily/processing_tables \
-    spectro/redux/daily/tiles/archive spectro/redux/daily/tiles/cumulative; do
+    spectro/redux/daily/tiles/archive \
+    # spectro/redux/daily/tiles/cumulative \
+    survey/GFA; do
     case ${d} in
         spectro/redux/daily) priority='nice'; exclude="--include-from ${DESITRANSFER}/py/desitransfer/data/desi_utah_daily.txt --exclude *" ;;
         spectro/redux/daily/calibnight) priority='nice'; exclude='--include 202403?? --include *.fits --include *.fits.gz --include *.csv --include tmp --include old --exclude *' ;;

From 10b6317cc8ce5a9f39a76c78e38aa5dc6fa77f41 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 26 Mar 2024 13:52:41 -0700
Subject: [PATCH 07/14] temporarily remove tiles/cumulative

---
 bin/desi_utah_transfer.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index d2b522b..7a0813e 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -58,7 +58,6 @@ for d in spectro/redux/daily spectro/redux/daily/calibnight \
     spectro/redux/daily/exposure_tables spectro/redux/daily/exposures \
     spectro/redux/daily/preproc spectro/redux/daily/processing_tables \
     spectro/redux/daily/tiles/archive \
-    # spectro/redux/daily/tiles/cumulative \
     survey/GFA; do
     case ${d} in
         spectro/redux/daily) priority='nice'; exclude="--include-from ${DESITRANSFER}/py/desitransfer/data/desi_utah_daily.txt --exclude *" ;;

From 8961460ff02513e2bf13db5ea4b7433f9cdf6db3 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 26 Mar 2024 15:19:30 -0700
Subject: [PATCH 08/14] add notes on tiles/cumulative

---
 py/desitransfer/data/desi_utah_daily.txt | 33 +++---------------------
 1 file changed, 4 insertions(+), 29 deletions(-)

diff --git a/py/desitransfer/data/desi_utah_daily.txt b/py/desitransfer/data/desi_utah_daily.txt
index fca96c5..f16008f 100644
--- a/py/desitransfer/data/desi_utah_daily.txt
+++ b/py/desitransfer/data/desi_utah_daily.txt
@@ -1,37 +1,12 @@
 #
-# cd desi/spectro/redux/daily/
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi --include-from ${DESITRANSFER}/py/desitransfer/data/desi_utah_daily.txt \
-#     --exclude \* rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/ ./
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/tiles/archive/ tiles/archive/
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/exposure_tables/ exposure_tables/
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/processing_tables/ processing_tables/
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
-#     --include fibermap-\*.fits --include preproc-\*.fits.gz --include tilepix-\*.json --exclude \* \
-#     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/preproc/ preproc/
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi --include 202403\?\? --include 202403\?\?/\?\?\?\?\?\?\?\? \
-#     --include \*.fits --include \*.fits.gz --include \*.csv --exclude \*
-#     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/exposures/ exposures/
-#
-# rsync --verbose --recursive --links --times --omit-dir-times --no-motd \
-#     --password-file ${HOME}/.desi --include 202403\?\? --include \*.fits --include \*.fits.gz --include \*.csv --include tmp --include old --exclude \* \
-#     rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/calibnight/ calibnight/
-#
 # tiles/cumulative has a lot of symlinks into tiles/archive.
 # * Save all symlinks
 # * Save only last 30 days of non-symlinks.
 #
+# find . -type l > ~/daily_tiles_cumulative.txt
+# find . -type f | grep -E '^\./[0-9]+/202403../' >> ~/daily_tiles_cumulative.txt
+# rsync -rlvtO --files-from $HOME/daily_tiles_cumulative.txt rsync://${DESISYNC_HOSTNAME}/desi/spectro/redux/daily/tiles/cumulative/ tiles/cumulative/
+#
 dashboard
 dashboard.html
 exposures-daily.*

From 1c2851a349241af7037082e1a7ae1a57e2fe674c Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Wed, 27 Mar 2024 13:48:08 -0700
Subject: [PATCH 09/14] set noglob

---
 bin/desi_utah_transfer.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index 7a0813e..44160dd 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -19,6 +19,10 @@ function usage() {
     ) >&2
 }
 #
+# Do not expand globs, pass them on to rsync.
+#
+set -o noglob
+#
 # Environment variables.
 #
 if [[ -z "${DESITRANSFER}" ]]; then

From c85b2d4738de5a590e2f2433ce514be0f7c503a2 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Wed, 27 Mar 2024 13:57:16 -0700
Subject: [PATCH 10/14] add tiles/cumulative

---
 bin/desi_utah_transfer.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index 44160dd..c61b7d7 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -61,13 +61,14 @@ shift $((OPTIND - 1))
 for d in spectro/redux/daily spectro/redux/daily/calibnight \
     spectro/redux/daily/exposure_tables spectro/redux/daily/exposures \
     spectro/redux/daily/preproc spectro/redux/daily/processing_tables \
-    spectro/redux/daily/tiles/archive \
+    spectro/redux/daily/tiles/archive spectro/redux/daily/tiles/cumulative \
     survey/GFA; do
     case ${d} in
         spectro/redux/daily) priority='nice'; exclude="--include-from ${DESITRANSFER}/py/desitransfer/data/desi_utah_daily.txt --exclude *" ;;
         spectro/redux/daily/calibnight) priority='nice'; exclude='--include 202403?? --include *.fits --include *.fits.gz --include *.csv --include tmp --include old --exclude *' ;;
         spectro/redux/daily/exposures) priority='nice'; exclude='--include 202403?? --include 202403??/???????? --include *.fits --include *.fits.gz --include *.csv --exclude *' ;;
         spectro/redux/daily/preproc) priority='nice'; exclude='--include 202403?? --include 202403??/???????? --include fibermap-*.fits --include preproc-*.fits.gz --include tilepix-*.json --exclude *' ;;
+        spectro/redux/daily/tiles/cumulative) priority='nice'; exclude="--files-from ${DESI_ROOT}/spectro/redux/daily_tiles_cumulative.txt" ;;
         *) priority=''; exclude='' ;;
     esac
     log=${log_root}/utah_$(tr '/' '_' <<<${d}).log

From b537f5a86dbb5bb0012da73829c2d8918dc9694d Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 2 Apr 2024 15:22:49 -0700
Subject: [PATCH 11/14] Download tiles/cumulative file

---
 bin/desi_utah_transfer.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index c61b7d7..6cb0bf4 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -37,6 +37,10 @@ if [[ -z "${DESI_ROOT}" ]]; then
     echo "ERROR: DESI_ROOT is undefined!"
     exit 1
 fi
+if [[ -z "${DAILY_TILES_CUMULATIVE_OUTPUT}" ]]; then
+    echo "ERROR: DAILY_TILES_CUMULATIVE_OUTPUT is undefined!"
+    exit 1
+fi
 #
 # Configuration.
 #
@@ -56,6 +60,10 @@ while getopts htv argname; do
 done
 shift $((OPTIND - 1))
 #
+# Copy the daily/tiles/cumulative description file from NERSC.
+#
+wget --quiet --unlink --output-document=${DESI_ROOT}/spectro/redux/daily_tiles_cumulative.txt ${DAILY_TILES_CUMULATIVE_OUTPUT}
+#
 # Execute rsync commands.
 #
 for d in spectro/redux/daily spectro/redux/daily/calibnight \

From 47681daad3c121c57bee16115dd11f26a226bfc5 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Wed, 24 Apr 2024 09:40:25 -0700
Subject: [PATCH 12/14] try to fix permission error

---
 bin/desi_tucson_transfer_catchup.sh | 2 +-
 bin/desi_utah_transfer.sh           | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh
index e88e067..90f40cb 100755
--- a/bin/desi_tucson_transfer_catchup.sh
+++ b/bin/desi_tucson_transfer_catchup.sh
@@ -22,7 +22,7 @@ for d in engineering/focalplane engineering/focalplane/hwtables \
     case ${d} in
         engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;;
         engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;
-        spectro/data) priority=''; exclude='--exclude 2018* --exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022*' ;;
+        spectro/data) priority=''; exclude='--exclude 2018* --exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022* --exclude 2023*' ;;
         spectro/redux/daily) priority=''; exclude='--exclude *.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles' ;;
         spectro/redux/daily/exposures) priority=''; exclude='--exclude *.tmp' ;;
         spectro/redux/daily/preproc) priority=''; exclude='--exclude *.tmp --exclude preproc-*.fits --exclude preproc-*.fits.gz' ;;
diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index 6cb0bf4..e8eccd8 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -84,3 +84,8 @@ for d in spectro/redux/daily spectro/redux/daily/calibnight \
     ${verbose} && echo "${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &"
     ${test}    || ${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &
 done
+#
+# Set user-write on some files.
+#
+${verbose} && echo "chmod -R u+w ${dst}/spectro/redux/daily/tiles/cumulative"
+${test}    || chmod -R u+w ${dst}/spectro/redux/daily/tiles/cumulative

From 96b8ba4047132102a9a886fcabae2f9814ce3f8a Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Thu, 25 Apr 2024 09:37:45 -0700
Subject: [PATCH 13/14] fix possible glob error

---
 bin/desi_tucson_transfer_catchup.sh | 56 ++++++-----------------------
 1 file changed, 10 insertions(+), 46 deletions(-)

diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh
index 90f40cb..28ef2ef 100755
--- a/bin/desi_tucson_transfer_catchup.sh
+++ b/bin/desi_tucson_transfer_catchup.sh
@@ -10,11 +10,20 @@ if [[ -z "${DESI_ROOT}" ]]; then
     echo "ERROR: DESI_ROOT is undefined!"
     exit 1
 fi
+#
+# Do not expand globs, pass them on to rsync.
+#
+set -o noglob
+#
+# Configuration.
+#
 syn="/usr/bin/rsync --archive --verbose --delete --delete-after --no-motd --password-file ${HOME}/.desi"
 src=rsync://${DESISYNC_HOSTNAME}/desi
 dst=${DESI_ROOT}
 log_root=${HOME}/Documents/Logfiles
-
+#
+# Execute rsync commands.
+#
 for d in engineering/focalplane engineering/focalplane/hwtables \
     spectro/data \
     spectro/redux/daily spectro/redux/daily/exposures spectro/redux/daily/preproc spectro/redux/daily/tiles \
@@ -34,48 +43,3 @@ for d in engineering/focalplane engineering/focalplane/hwtables \
     echo "${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &"
     ${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &
 done
-
-# log=${log_root}/catchup_engineering_focalplane.log
-# [[ -f ${log} ]] || touch ${log}
-# nice ${syn} --exclude archive --exclude hwtables --exclude \*.ipynb --exclude .ipynb_checkpoints \
-#     ${src}/engineering/focalplane/ ${dst}/engineering/focalplane/ &>> ${log} &
-
-# log=${log_root}/catchup_engineering_focalplane_hwtables.log
-# [[ -f ${log} ]] || touch ${log}
-# nice ${syn} --include \*.csv --exclude \* \
-#     ${src}/engineering/focalplane/hwtables/ ${dst}/engineering/focalplane/hwtables/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_data.log
-# [[ -f ${log} ]] || touch ${log}
-# ${syn} --exclude 2018\* --exclude 2019\* --exclude 2020\* --exclude 2021\* --exclude 2022\* \
-#     ${src}/spectro/data/ ${dst}/spectro/data/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_redux_daily.log
-# [[ -f ${log} ]] || touch ${log}
-# ${syn} --exclude \*.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles \
-#     ${src}/spectro/redux/daily/ ${dst}/spectro/redux/daily/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_redux_daily_exposures.log
-# [[ -f ${log} ]] || touch ${log}
-# ${syn} --exclude \*.tmp \
-#     ${src}/spectro/redux/daily/exposures/ ${dst}/spectro/redux/daily/exposures/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_redux_daily_preproc.log
-# [[ -f ${log} ]] || touch ${log}
-# ${syn} --exclude \*.tmp --exclude preproc-\*.fits --exclude preproc-\*.fits.gz \
-#     ${src}/spectro/redux/daily/preproc/ ${dst}/spectro/redux/daily/preproc/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_redux_daily_tiles.log
-# [[ -f ${log} ]] || touch ${log}
-# ${syn} --exclude \*.tmp --exclude temp \
-#     ${src}/spectro/redux/daily/tiles/ ${dst}/spectro/redux/daily/tiles/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_nightwatch_kpno.log
-# [[ -f ${log} ]] || touch ${log}
-# nice ${syn} \
-#     ${src}/spectro/nightwatch/kpno/ ${dst}/spectro/nightwatch/kpno/ &>> ${log} &
-
-# log=${log_root}/catchup_spectro_staging_lost+found.log
-# [[ -f ${log} ]] || touch ${log}
-# nice ${syn} \
-#     ${src}/spectro/staging/lost+found/ ${dst}/spectro/staging/lost+found/ &>> ${log} &

From 99da3fe7c06b1c4229b4a2cbb9c7626a5c493661 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 30 Apr 2024 14:01:04 -0700
Subject: [PATCH 14/14] run chmod first

---
 bin/desi_utah_transfer.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/desi_utah_transfer.sh b/bin/desi_utah_transfer.sh
index e8eccd8..09e0ecb 100755
--- a/bin/desi_utah_transfer.sh
+++ b/bin/desi_utah_transfer.sh
@@ -60,6 +60,11 @@ while getopts htv argname; do
 done
 shift $((OPTIND - 1))
 #
+# Set user-write on some files.
+#
+${verbose} && echo "chmod -R u+w ${dst}/spectro/redux/daily/tiles/cumulative"
+${test}    || chmod -R u+w ${dst}/spectro/redux/daily/tiles/cumulative
+#
 # Copy the daily/tiles/cumulative description file from NERSC.
 #
 wget --quiet --unlink --output-document=${DESI_ROOT}/spectro/redux/daily_tiles_cumulative.txt ${DAILY_TILES_CUMULATIVE_OUTPUT}
@@ -84,8 +89,3 @@ for d in spectro/redux/daily spectro/redux/daily/calibnight \
     ${verbose} && echo "${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &"
     ${test}    || ${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &
 done
-#
-# Set user-write on some files.
-#
-${verbose} && echo "chmod -R u+w ${dst}/spectro/redux/daily/tiles/cumulative"
-${test}    || chmod -R u+w ${dst}/spectro/redux/daily/tiles/cumulative