Spacewatch downloads and other updates

.github/workflows/python-package.yml

@@ -17,15 +17,15 @@ jobs:
             fail-fast: true
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.8', '3.9', '3.10']
+                python-version: ['3.9', '3.10', '3.11']
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v4
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
@@ -43,15 +43,15 @@ jobs:
             fail-fast: true
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v4
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
@@ -74,15 +74,15 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v4
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
@@ -97,15 +97,15 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v4
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
@@ -124,15 +124,15 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v4
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies

.readthedocs.yml

@@ -5,21 +5,20 @@
 # Required
 version: 2
 
-# Set the version of Python and other tools you might need
 build:
-    os: ubuntu-22.04
+    os: "ubuntu-22.04"
     tools:
         python: "3.10"
 
 # Build documentation in the doc/ directory with Sphinx
 sphinx:
     configuration: doc/conf.py
-    fail_on_warning: true
 
 # Optionally build your docs in additional formats such as PDF and ePub
-formats: all
+# formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
     install:
-      - requirements: doc/rtd-requirements.txt
+        - requirements: doc/rtd-requirements.txt
+

MANIFEST.in

@@ -1,12 +1,2 @@
-include LICENSE.rst
-include README.rst
-include requirements.txt
-
-graft bin
-graft doc
-graft etc
-
-prune build
-prune dist
-prune htmlcov
-prune doc/_build
+prune .github
+global-exclude .gitignore .readthedocs.yml

bin/desi_nightlog_transfer → bin/desi_spacewatch_transfer

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 from sys import exit
-from desitransfer.nightlog import main
+from desitransfer.spacewatch import main
 exit(main())

bin/desi_transfer_statistics.sh

@@ -49,6 +49,36 @@ if [[ "${year}" == "2021" ]]; then
     echo "Total data during fire recovery = ${total_fire_data} KB."
 fi
 #
+# Statistics during security recovery period. 20230804 - 20230924
+#
+if [[ "${year}" == "2022" ]]; then
+    number_of_fire_nights=0
+    number_of_fire_exposures=0
+    total_fire_data=0
+    declare -a nights
+    for d in $(seq 4 31); do
+        nights+=( ${next_year}08$(printf "%02d" ${d}) )
+    done
+    for d in $(seq 1 24); do
+        nights+=( ${next_year}09$(printf "%02d" ${d}) )
+    done
+    for night in "${nights[@]}"; do
+        if [[ -d ${DESI_SPECTRO_DATA}/${night} ]]; then
+            number_of_fire_nights=$(( number_of_fire_nights + 1 ))
+            for e in ${DESI_SPECTRO_DATA}/${night}/*; do
+                echo ${e}
+                expid=$(basename ${e})
+                expid_size=$(du -k -s ${e} | awk '{print $1}')
+                number_of_fire_exposures=$(( number_of_fire_exposures + 1 ))
+                total_fire_data=$(( total_fire_data + expid_size ))
+            done
+        fi
+    done
+    echo "Number of nights during security recovery = ${number_of_fire_nights}."
+    echo "Number of exposures during security recovery = ${number_of_fire_exposures}."
+    echo "Total data during security recovery = ${total_fire_data} KB."
+fi
+#
 # Nightwatch data.
 #
 NIGHTWATCH=${DESI_ROOT}/spectro/nightwatch/kpno
@@ -87,4 +117,4 @@ echo "Total nightwatch data = ${total_nightwatch_data} KB."
 # - We are transferring a small amount of engineering files ~ 10 MB/night.
 # - We are *not* transferring nightwatch data.
 # - We are streaming to the database replica at NERSC.  This is the biggest share.
-#
+#

bin/desi_tucson_transfer_catchup.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Parallel copy DESI mirror data, to catch up after outages.
+#
+if [[ -z "${DESISYNC_HOSTNAME}" ]]; then
+    echo "ERROR: DESISYNC_HOSTNAME is undefined!"
+    exit 1
+fi
+if [[ -z "${DESI_ROOT}" ]]; then
+    echo "ERROR: DESI_ROOT is undefined!"
+    exit 1
+fi
+syn="/usr/bin/rsync --archive --verbose --delete --delete-after --no-motd --password-file ${HOME}/.desi"
+src=rsync://${DESISYNC_HOSTNAME}/desi
+dst=${DESI_ROOT}
+log_root=${HOME}/Documents/Logfiles
+
+for d in engineering/focalplane engineering/focalplane/hwtables \
+    spectro/data \
+    spectro/redux/daily spectro/redux/daily/exposures spectro/redux/daily/preproc spectro/redux/daily/tiles \
+    spectro/nightwatch/kpno spectro/staging/lost+found; do
+    case ${d} in
+        engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;;
+        engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;
+        spectro/data) priority=''; exclude='--exclude 2018* --exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022*' ;;
+        spectro/redux/daily) priority=''; exclude='--exclude *.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles' ;;
+        spectro/redux/daily/exposures) priority=''; exclude='--exclude *.tmp' ;;
+        spectro/redux/daily/preproc) priority=''; exclude='--exclude *.tmp --exclude preproc-*.fits --exclude preproc-*.fits.gz' ;;
+        spectro/redux/daily/tiles) priority=''; exclude='--exclude *.tmp --exclude temp' ;;
+        *) priority='nice'; exclude='' ;;
+    esac
+    log=${log_root}/catchup_$(tr '/' '_' <<<${d}).log
+    [[ -f ${log} ]] || touch ${log}
+    echo "${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &"
+    ${priority} ${syn} ${exclude} ${src}/${d}/ ${dst}/${d}/ &>> ${log} &
+done
+
+# log=${log_root}/catchup_engineering_focalplane.log
+# [[ -f ${log} ]] || touch ${log}
+# nice ${syn} --exclude archive --exclude hwtables --exclude \*.ipynb --exclude .ipynb_checkpoints \
+#     ${src}/engineering/focalplane/ ${dst}/engineering/focalplane/ &>> ${log} &
+
+# log=${log_root}/catchup_engineering_focalplane_hwtables.log
+# [[ -f ${log} ]] || touch ${log}
+# nice ${syn} --include \*.csv --exclude \* \
+#     ${src}/engineering/focalplane/hwtables/ ${dst}/engineering/focalplane/hwtables/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_data.log
+# [[ -f ${log} ]] || touch ${log}
+# ${syn} --exclude 2018\* --exclude 2019\* --exclude 2020\* --exclude 2021\* --exclude 2022\* \
+#     ${src}/spectro/data/ ${dst}/spectro/data/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_redux_daily.log
+# [[ -f ${log} ]] || touch ${log}
+# ${syn} --exclude \*.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles \
+#     ${src}/spectro/redux/daily/ ${dst}/spectro/redux/daily/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_redux_daily_exposures.log
+# [[ -f ${log} ]] || touch ${log}
+# ${syn} --exclude \*.tmp \
+#     ${src}/spectro/redux/daily/exposures/ ${dst}/spectro/redux/daily/exposures/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_redux_daily_preproc.log
+# [[ -f ${log} ]] || touch ${log}
+# ${syn} --exclude \*.tmp --exclude preproc-\*.fits --exclude preproc-\*.fits.gz \
+#     ${src}/spectro/redux/daily/preproc/ ${dst}/spectro/redux/daily/preproc/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_redux_daily_tiles.log
+# [[ -f ${log} ]] || touch ${log}
+# ${syn} --exclude \*.tmp --exclude temp \
+#     ${src}/spectro/redux/daily/tiles/ ${dst}/spectro/redux/daily/tiles/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_nightwatch_kpno.log
+# [[ -f ${log} ]] || touch ${log}
+# nice ${syn} \
+#     ${src}/spectro/nightwatch/kpno/ ${dst}/spectro/nightwatch/kpno/ &>> ${log} &
+
+# log=${log_root}/catchup_spectro_staging_lost+found.log
+# [[ -f ${log} ]] || touch ${log}
+# nice ${syn} \
+#     ${src}/spectro/staging/lost+found/ ${dst}/spectro/staging/lost+found/ &>> ${log} &

doc/api.rst

@@ -14,10 +14,10 @@ desitransfer API
 .. automodule:: desitransfer.daily
     :members:
 
-.. automodule:: desitransfer.nightlog
+.. automodule:: desitransfer.nightwatch
     :members:
 
-.. automodule:: desitransfer.nightwatch
+.. automodule:: desitransfer.spacewatch
     :members:
 
 .. automodule:: desitransfer.status

doc/changes.rst

@@ -5,13 +5,14 @@ Change Log
 1.0.0 (unreleased)
 ------------------
 
-* *Planned*: refactor package to deprecate ``setup.py``.
-* *Planned*: remove deprecated code such as ``nightlog.py``.
-
-0.9.3 (unreleased)
-------------------
-
-* No changes yet.
+* Refactor package to deprecate ``setup.py`` (PR `#58`_).
+* Remove deprecated code such as ``nightlog.py`` (PR `#58`_).
+* Add Spacewatch image download (PR `#58`_).
+* Download nightlog data earlier in the day (PR `#58`_).
+* Update engineering data transferred to Tucson (PR `#58`_).
+* Better logging of and resilience to nightwatch transfer errors (PR `#58`_).
+
+.. _`#58`: https://github.com/desihub/desitransfer/pull/58
 
 0.9.2 (2023-05-31)
 ------------------