From 595213c076d14537182c4e9db8d9ce0f9d0c36d6 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 20 Aug 2024 10:49:45 -0700
Subject: [PATCH 01/12] parallelize Tucson transfer

---
 py/desitransfer/test/test_tucson.py | 41 +++++++++++++--
 py/desitransfer/tucson.py           | 77 +++++++++++++++++++++++------
 2 files changed, 100 insertions(+), 18 deletions(-)

diff --git a/py/desitransfer/test/test_tucson.py b/py/desitransfer/test/test_tucson.py
index 69180c0..bebc47a 100644
--- a/py/desitransfer/test/test_tucson.py
+++ b/py/desitransfer/test/test_tucson.py
@@ -7,8 +7,10 @@
 import unittest
 import logging
 import subprocess as sub
-from unittest.mock import patch, call, mock_open
-from ..tucson import _options, _rsync, _configure_log, running
+from tempfile import mkdtemp
+from shutil import rmtree
+from unittest.mock import patch, call, mock_open, MagicMock
+from ..tucson import _options, _rsync, _configure_log, running, _get_proc
 from .. import __version__ as dtVersion
 
 
@@ -18,11 +20,11 @@ class TestTucson(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        pass
+        cls.temp_dir = mkdtemp()
 
     @classmethod
     def tearDownClass(cls):
-        pass
+        rmtree(cls.temp_dir)
 
     def setUp(self):
         pass
@@ -164,3 +166,34 @@ def test_running_read_exit(self, mock_log, mock_popen, mock_exists, mock_remove)
                                           stdout=sub.PIPE, stderr=sub.PIPE),
                                      call().communicate()])
         mock_popen().communicate.assert_called_once()
+
+    @patch('subprocess.Popen')
+    @patch('desitransfer.tucson.log')
+    def test_get_proc(self, mock_log, mock_popen):
+        """Test the function for generating external procedures.
+        """
+        directories = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']
+        exclude = set(['d', 'g'])
+        options = MagicMock()
+        options.test = False
+        options.log = self.temp_dir
+        proc, LOG_A, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'a')
+        LOG_A.close()
+        options.test = True
+        proc, LOG_B, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'b')
+        self.assertEqual(LOG_B, os.path.join(self.temp_dir, 'desi_tucson_transfer_b.log'))
+        proc, LOG_C, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'c')
+        proc, LOG_E, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'e')
+        proc, LOG_F, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'f')
+        proc, LOG_H, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'h')
+        proc, LOG_I, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertEqual(d, 'i')
+        proc, LOG_J, d = _get_proc(directories, exclude, '/src', '/dst', options)
+        self.assertIsNone(proc)
+
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 09edeb8..4904f5c 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -130,6 +130,9 @@ def _options():
     prsr.add_argument('-l', '--log', metavar='DIR',
                       default=os.path.join(os.environ['HOME'], 'Documents', 'Logfiles'),
                       help='Use DIR for log files (default %(default)s).')
+    prsr.add_argument('-p', '--processes', action='store', type=int,
+                      dest='nproc',  metavar="N", default=10,
+                      help="Number of simultaneous downloads (default %(default)s).")
     prsr.add_argument('-s', '--static', action='store_true', dest='static',
                       help='Also sync static data sets.')
     prsr.add_argument('-S', '--sleep', metavar='TIME', default='15m', dest='sleep',
@@ -166,6 +169,46 @@ def _rsync(src, dst, d, checksum=False):
     return cmd
 
 
+def _get_proc(directories, exclude, src, dst, options):
+    """Prepare the next download directory for processing.
+
+    Parameters
+    ----------
+    directories : :class:`list`
+        A list of directories to process.
+    exclude : :class:`set`
+        Do not process directories in this set.
+    src : :class:`str`
+        Root source directory.
+    dst : :class:`str`
+        Root destination directory.
+    options : :class:`argparse.Namespace`
+        The parsed command-line options.
+
+    Returns
+    -------
+    :class:`tuple`
+        A tuple containing information about the process.
+    """
+    global log
+    try:
+        d = directories.pop(0)
+        while d in exclude:
+            log.warning("%s skipped at user request.", d)
+            d = directories.pop(0)
+        log_file = os.path.join(options.log,
+                                'desi_tucson_transfer_' + d.replace('/', '_') + '.log')
+        command = _rsync(src, dst, d, checksum=options.checksum)
+        if options.test:
+            return (command, log_file, d)
+        else:
+            log.info(' '.join(command))
+            LOG = open(log_file, 'ab')
+            return (sub.Popen(command, stdout=LOG, stderr=sub.STDOUT), LOG, d)
+    except IndexError:
+        return (None, None, None)
+
+
 def running(pid_file):
     """Test for a duplicate process already running.
 
@@ -266,18 +309,24 @@ def main():
         directories = static + dynamic
     else:
         directories = dynamic
-    for d in directories:
-        if d in exclude:
-            log.warning("%s skipped at user request.", d)
-        else:
-            command = _rsync(src, dst, d, checksum=options.checksum)
-            log.info(' '.join(command))
-            if not options.test:
-                log_file = os.path.join(options.log,
-                                        'desi_tucson_transfer_' + d.replace('/', '_') + '.log')
-                with open(log_file, 'ab') as LOG:
-                    proc = sub.Popen(command, stdout=LOG, stderr=sub.STDOUT)
-                    status = proc.wait()
-                if status != 0:
-                    log.critical("rsync error detected for %s/%s/! Check logs!", dst, d)
+    proc_pool = dict()
+    for p in range(options.nproc):
+        proc_key = 'proc{0:03d}'.format(p)
+        proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)
+    while any([v[0] is not None for v in proc_pool.values()]):
+        for proc_key in proc_pool:
+            proc, LOG, d = proc_pool[proc_key]
+            if options.test:
+                log.debug("%s: %s -> %s", d, ' '.join(proc), LOG)
+            else:
+                if proc is None:
+                    status = None
+                else:
+                    status = proc.poll()
+                if status is not None:
+                    LOG.close()
+                    if status != 0:
+                        log.critical("rsync error detected for %s/%s/! Check logs!", dst, d)
+            proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)
+        time.sleep(sleepy_time)
     return 0

From 37aa3f31f8af7fc22c0158f414a86e6e4145bb49 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 20 Aug 2024 11:03:27 -0700
Subject: [PATCH 02/12] fix style

---
 doc/changes.rst                     | 2 +-
 py/desitransfer/test/test_tucson.py | 1 -
 py/desitransfer/tucson.py           | 8 +++++---
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/changes.rst b/doc/changes.rst
index 24609ec..b2305ca 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,7 +5,7 @@ Change Log
 1.0.3 (unreleased)
 ------------------
 
-* No changes yet.
+* Convert Tucson transfer to parallel operation.
 
 1.0.2 (2024-06-21)
 ------------------
diff --git a/py/desitransfer/test/test_tucson.py b/py/desitransfer/test/test_tucson.py
index bebc47a..b781bc7 100644
--- a/py/desitransfer/test/test_tucson.py
+++ b/py/desitransfer/test/test_tucson.py
@@ -196,4 +196,3 @@ def test_get_proc(self, mock_log, mock_popen):
         self.assertEqual(d, 'i')
         proc, LOG_J, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertIsNone(proc)
-
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 4904f5c..b3df778 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -131,7 +131,7 @@ def _options():
                       default=os.path.join(os.environ['HOME'], 'Documents', 'Logfiles'),
                       help='Use DIR for log files (default %(default)s).')
     prsr.add_argument('-p', '--processes', action='store', type=int,
-                      dest='nproc',  metavar="N", default=10,
+                      dest='nproc', metavar="N", default=10,
                       help="Number of simultaneous downloads (default %(default)s).")
     prsr.add_argument('-s', '--static', action='store_true', dest='static',
                       help='Also sync static data sets.')
@@ -327,6 +327,8 @@ def main():
                     LOG.close()
                     if status != 0:
                         log.critical("rsync error detected for %s/%s/! Check logs!", dst, d)
-            proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)
-        time.sleep(sleepy_time)
+                    proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)
+        if not options.test:
+            log.debug("Waiting for jobs to complete, sleeping %s.", options.sleep)
+            time.sleep(sleepy_time)
     return 0

From 74ba79947374a0f11b92a6f609ef72f640f8f145 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 20 Aug 2024 11:12:00 -0700
Subject: [PATCH 03/12] update change log

---
 doc/changes.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/changes.rst b/doc/changes.rst
index b2305ca..dd12491 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,7 +5,9 @@ Change Log
 1.0.3 (unreleased)
 ------------------
 
-* Convert Tucson transfer to parallel operation.
+* Convert Tucson transfer to parallel operation (PR `#61`_).
+
+.. _`#61`: https://github.com/desihub/desitransfer/pull/61
 
 1.0.2 (2024-06-21)
 ------------------

From 168944f468868c6f93590437b420b153e7351c81 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 20 Aug 2024 11:13:48 -0700
Subject: [PATCH 04/12] don't log processes that don't exist

---
 py/desitransfer/tucson.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index b3df778..c07785c 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -316,7 +316,7 @@ def main():
     while any([v[0] is not None for v in proc_pool.values()]):
         for proc_key in proc_pool:
             proc, LOG, d = proc_pool[proc_key]
-            if options.test:
+            if options.test and proc is not None:
                 log.debug("%s: %s -> %s", d, ' '.join(proc), LOG)
             else:
                 if proc is None:

From b3aea4de579d5c6e498c49258c51c2a7a3710e75 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 20 Aug 2024 11:15:54 -0700
Subject: [PATCH 05/12] avoid infinite loop

---
 py/desitransfer/tucson.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index c07785c..893f28e 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -318,16 +318,15 @@ def main():
             proc, LOG, d = proc_pool[proc_key]
             if options.test and proc is not None:
                 log.debug("%s: %s -> %s", d, ' '.join(proc), LOG)
+            if proc is None:
+                status = None
             else:
-                if proc is None:
-                    status = None
-                else:
-                    status = proc.poll()
-                if status is not None:
-                    LOG.close()
-                    if status != 0:
-                        log.critical("rsync error detected for %s/%s/! Check logs!", dst, d)
-                    proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)
+                status = proc.poll()
+            if status is not None:
+                LOG.close()
+                if status != 0:
+                    log.critical("rsync error detected for %s/%s/! Check logs!", dst, d)
+                proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)
         if not options.test:
             log.debug("Waiting for jobs to complete, sleeping %s.", options.sleep)
             time.sleep(sleepy_time)

From a0dd88dc23c13cb9abe13c3d4e63aa14ddbcfda8 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 20 Aug 2024 11:18:06 -0700
Subject: [PATCH 06/12] refactor test mode

---
 py/desitransfer/tucson.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 893f28e..59834a3 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -316,14 +316,17 @@ def main():
     while any([v[0] is not None for v in proc_pool.values()]):
         for proc_key in proc_pool:
             proc, LOG, d = proc_pool[proc_key]
-            if options.test and proc is not None:
-                log.debug("%s: %s -> %s", d, ' '.join(proc), LOG)
             if proc is None:
                 status = None
             else:
-                status = proc.poll()
+                if options.test:
+                    log.debug("%s: %s -> %s", d, ' '.join(proc), LOG)
+                    status = 0
+                else:
+                    status = proc.poll()
             if status is not None:
-                LOG.close()
+                if not options.test:
+                    LOG.close()
                 if status != 0:
                     log.critical("rsync error detected for %s/%s/! Check logs!", dst, d)
                 proc_pool[proc_key] = _get_proc(directories, exclude, src, dst, options)

From 413e0cb747f53b08f5bb6c3d79da32375b3432cc Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Mon, 9 Sep 2024 14:46:28 -0700
Subject: [PATCH 07/12] allow 'nice' transfers

---
 bin/desi_tucson_transfer_catchup.sh |  3 ++-
 py/desitransfer/test/test_tucson.py | 16 +++++++++++++++-
 py/desitransfer/tucson.py           | 27 +++++++++++++++++++++++++--
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh
index a9092bf..b4499e2 100755
--- a/bin/desi_tucson_transfer_catchup.sh
+++ b/bin/desi_tucson_transfer_catchup.sh
@@ -27,7 +27,8 @@ log_root=${HOME}/Documents/Logfiles
 for d in engineering/focalplane engineering/focalplane/hwtables \
     spectro/data \
     spectro/redux/daily spectro/redux/daily/exposures spectro/redux/daily/preproc spectro/redux/daily/tiles \
-    spectro/nightwatch/kpno spectro/staging/lost+found; do
+    spectro/nightwatch/kpno spectro/staging/lost+found \
+    software/AnyConnect software/CiscoSecureClient; do
     case ${d} in
         engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;;
         engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;
diff --git a/py/desitransfer/test/test_tucson.py b/py/desitransfer/test/test_tucson.py
index b781bc7..bbadbdc 100644
--- a/py/desitransfer/test/test_tucson.py
+++ b/py/desitransfer/test/test_tucson.py
@@ -169,11 +169,14 @@ def test_running_read_exit(self, mock_log, mock_popen, mock_exists, mock_remove)
 
     @patch('subprocess.Popen')
     @patch('desitransfer.tucson.log')
-    def test_get_proc(self, mock_log, mock_popen):
+    @patch('desitransfer.tucson.priority')
+    def test_get_proc(self, mock_priority, mock_log, mock_popen):
         """Test the function for generating external procedures.
         """
         directories = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']
         exclude = set(['d', 'g'])
+        mock_priority.__contains__ = lambda self, x: x == 'e' or x == 'f'
+        # mock_priority.__iter__.return_value = ('e', 'f')
         options = MagicMock()
         options.test = False
         options.log = self.temp_dir
@@ -186,6 +189,7 @@ def test_get_proc(self, mock_log, mock_popen):
         self.assertEqual(LOG_B, os.path.join(self.temp_dir, 'desi_tucson_transfer_b.log'))
         proc, LOG_C, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertEqual(d, 'c')
+        options.test = False
         proc, LOG_E, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertEqual(d, 'e')
         proc, LOG_F, d = _get_proc(directories, exclude, '/src', '/dst', options)
@@ -196,3 +200,13 @@ def test_get_proc(self, mock_log, mock_popen):
         self.assertEqual(d, 'i')
         proc, LOG_J, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertIsNone(proc)
+        mock_log.info.assert_has_calls([call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/a/ /dst/a/'),
+                                        call("Directory '%s' will be transferred with os.nice(%d)", 'a', 5),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/e/ /dst/e/'),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/f/ /dst/f/'),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/h/ /dst/h/'),
+                                        call("Directory '%s' will be transferred with os.nice(%d)", 'h', 5),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/i/ /dst/i/'),
+                                        call("Directory '%s' will be transferred with os.nice(%d)", 'i', 5)])
+        mock_log.warning.assert_has_calls([call('%s skipped at user request.', 'd'),
+                                           call('%s skipped at user request.', 'g')])
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 59834a3..0cb764e 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -55,6 +55,7 @@
            'spectro/redux/daily/preproc',
            'spectro/redux/daily/tiles',
            'engineering/focalplane',
+           'engineering/focalplane/hwtables',
            'software/AnyConnect',
            'software/CiscoSecureClient']
 
@@ -78,6 +79,13 @@
                                 "--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]}
 
 
+priority = ('spectro/data',
+            'spectro/redux/daily',
+            'spectro/redux/daily/exposures',
+            'spectro/redux/daily/preproc',
+            'spectro/redux/daily/tiles')
+
+
 def _configure_log(debug):
     """Re-configure the default logger returned by ``desiutil.log``.
 
@@ -169,7 +177,7 @@ def _rsync(src, dst, d, checksum=False):
     return cmd
 
 
-def _get_proc(directories, exclude, src, dst, options):
+def _get_proc(directories, exclude, src, dst, options, nice=5):
     """Prepare the next download directory for processing.
 
     Parameters
@@ -184,6 +192,9 @@ def _get_proc(directories, exclude, src, dst, options):
         Root destination directory.
     options : :class:`argparse.Namespace`
         The parsed command-line options.
+    nice : :class:`int`, optional.
+        Lower-priority transfers will be run with this value passed to :func:`os.nice`,
+        default 5.
 
     Returns
     -------
@@ -191,6 +202,13 @@ def _get_proc(directories, exclude, src, dst, options):
         A tuple containing information about the process.
     """
     global log
+
+    def preexec_nice():  # pragma: no cover
+        os.nice(nice)
+
+    def preexec_pass():  # pragma: no cover
+        pass
+
     try:
         d = directories.pop(0)
         while d in exclude:
@@ -204,7 +222,12 @@ def _get_proc(directories, exclude, src, dst, options):
         else:
             log.info(' '.join(command))
             LOG = open(log_file, 'ab')
-            return (sub.Popen(command, stdout=LOG, stderr=sub.STDOUT), LOG, d)
+            if d in priority:
+                preexec_fn = preexec_pass
+            else:
+                log.info("Directory '%s' will be transferred with os.nice(%d)", d, nice)
+                preexec_fn = preexec_nice
+            return (sub.Popen(command, preexec_fn=preexec_fn, stdout=LOG, stderr=sub.STDOUT), LOG, d)
     except IndexError:
         return (None, None, None)
 

From 44722d22213041476a2557fa8d5359e86ee1d15b Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Mon, 9 Sep 2024 14:52:23 -0700
Subject: [PATCH 08/12] fix home path in test

---
 py/desitransfer/test/test_tucson.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/py/desitransfer/test/test_tucson.py b/py/desitransfer/test/test_tucson.py
index bbadbdc..841291b 100644
--- a/py/desitransfer/test/test_tucson.py
+++ b/py/desitransfer/test/test_tucson.py
@@ -173,6 +173,7 @@ def test_running_read_exit(self, mock_log, mock_popen, mock_exists, mock_remove)
     def test_get_proc(self, mock_priority, mock_log, mock_popen):
         """Test the function for generating external procedures.
         """
+        home = os.environ['HOME']
         directories = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']
         exclude = set(['d', 'g'])
         mock_priority.__contains__ = lambda self, x: x == 'e' or x == 'f'
@@ -200,13 +201,13 @@ def test_get_proc(self, mock_priority, mock_log, mock_popen):
         self.assertEqual(d, 'i')
         proc, LOG_J, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertIsNone(proc)
-        mock_log.info.assert_has_calls([call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/a/ /dst/a/'),
+        mock_log.info.assert_has_calls([call(f'/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file {home}/.desi /src/a/ /dst/a/'),
                                         call("Directory '%s' will be transferred with os.nice(%d)", 'a', 5),
-                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/e/ /dst/e/'),
-                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/f/ /dst/f/'),
-                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/h/ /dst/h/'),
+                                        call(f'/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file {home}/.desi /src/e/ /dst/e/'),
+                                        call(f'/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file {home}/.desi /src/f/ /dst/f/'),
+                                        call(f'/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file {home}/.desi /src/h/ /dst/h/'),
                                         call("Directory '%s' will be transferred with os.nice(%d)", 'h', 5),
-                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/i/ /dst/i/'),
+                                        call(f'/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file {home}/.desi /src/i/ /dst/i/'),
                                         call("Directory '%s' will be transferred with os.nice(%d)", 'i', 5)])
         mock_log.warning.assert_has_calls([call('%s skipped at user request.', 'd'),
                                            call('%s skipped at user request.', 'g')])

From ed5175df664e390efd408054bf45ca50c16414dc Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 10 Sep 2024 14:54:53 -0700
Subject: [PATCH 09/12] add warning about number of processes

---
 bin/desi_tucson_transfer_catchup.sh |  5 ++---
 py/desitransfer/tucson.py           | 13 ++++++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh
index b4499e2..20831d6 100755
--- a/bin/desi_tucson_transfer_catchup.sh
+++ b/bin/desi_tucson_transfer_catchup.sh
@@ -22,13 +22,12 @@ src=rsync://${DESISYNC_HOSTNAME}/desi
 dst=${DESI_ROOT}
 log_root=${HOME}/Documents/Logfiles
 #
-# Execute rsync commands.
+# Execute rsync commands. Do not exceed 10 commands!
 #
 for d in engineering/focalplane engineering/focalplane/hwtables \
     spectro/data \
     spectro/redux/daily spectro/redux/daily/exposures spectro/redux/daily/preproc spectro/redux/daily/tiles \
-    spectro/nightwatch/kpno spectro/staging/lost+found \
-    software/AnyConnect software/CiscoSecureClient; do
+    spectro/nightwatch/kpno spectro/staging/lost+found; do
     case ${d} in
         engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;;
         engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 0cb764e..a734cc4 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -282,8 +282,15 @@ def main():
         try:
             foo = os.environ[e]
         except KeyError:
-            log.error("%s must be set!", e)
+            log.critical("%s must be set!", e)
             return 1
+
+    #
+    # Check other options.
+    #
+    if options.nproc > 10:
+        log.critical("Number of simultaneous transfers %d > 10!", options.nproc)
+        return 1
     #
     # Source and destination.
     #
@@ -292,7 +299,7 @@ def main():
         if 'DESI_ROOT' in os.environ:
             dst = os.environ['DESI_ROOT']
         else:
-            log.error("DESI_ROOT must be set, or destination directory set on the command-line (-d DIR)!")
+            log.critical("DESI_ROOT must be set, or destination directory set on the command-line (-d DIR)!")
             return 1
     else:
         dst = options.destination
@@ -314,7 +321,7 @@ def main():
                 try:
                     sleepy_time = int(options.sleep[0:-1]) * suffix[s]
                 except ValueError:
-                    log.error("Invalid value for sleep interval: '%s'!", options.sleep)
+                    log.critical("Invalid value for sleep interval: '%s'!", options.sleep)
                     return 1
     log.debug("requests.get('%s')", os.environ['DESISYNC_STATUS_URL'])
     if not options.test:

From 9b176332534245e2107e9c8f854250c0a7a87e12 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 17 Sep 2024 10:45:52 -0700
Subject: [PATCH 10/12] exclude years for exposures and preproc

---
 bin/desi_tucson_transfer_catchup.sh | 4 ++--
 py/desitransfer/tucson.py           | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh
index 20831d6..d474c3f 100755
--- a/bin/desi_tucson_transfer_catchup.sh
+++ b/bin/desi_tucson_transfer_catchup.sh
@@ -34,8 +34,8 @@ for d in engineering/focalplane engineering/focalplane/hwtables \
         spectro/data) priority=''; exclude='--exclude 2018* --exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022* --exclude 2023*' ;;
         spectro/nightwatch/kpno) priority='nice'; exclude='--exclude 2021* --exclude 2022* --exclude 2023*' ;;
         spectro/redux/daily) priority=''; exclude='--exclude *.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles' ;;
-        spectro/redux/daily/exposures) priority=''; exclude='--exclude *.tmp' ;;
-        spectro/redux/daily/preproc) priority=''; exclude='--exclude *.tmp --exclude preproc-*.fits --exclude preproc-*.fits.gz' ;;
+        spectro/redux/daily/exposures) priority=''; exclude='--exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022* --exclude 2023* --exclude *.tmp' ;;
+        spectro/redux/daily/preproc) priority=''; exclude='--exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022* --exclude 2023* --exclude *.tmp --exclude preproc-*.fits --exclude preproc-*.fits.gz' ;;
         spectro/redux/daily/tiles) priority=''; exclude='--exclude *.tmp --exclude temp' ;;
         *) priority='nice'; exclude='' ;;
     esac
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index a734cc4..8547fac 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -69,9 +69,9 @@
             'spectro/redux/daily': ["--exclude", "*.tmp", "--exclude", "attic",
                                     "--exclude", "exposures", "--exclude", "preproc",
                                     "--exclude", "temp", "--exclude", "tiles"],
-            'spectro/redux/daily/exposures': ["--exclude", "*.tmp"],
-            'spectro/redux/daily/preproc': ["--exclude", "*.tmp", "--exclude", "preproc-*.fits",
-                                            "--exclude", "preproc-*.fits.gz"],
+            'spectro/redux/daily/exposures': exclude_years(2019) + ["--exclude", "*.tmp"],
+            'spectro/redux/daily/preproc': exclude_years(2019) +  ["--exclude", "*.tmp", "--exclude", "preproc-*.fits",
+                                                                   "--exclude", "preproc-*.fits.gz"],
             'spectro/redux/daily/tiles': ["--exclude", "*.tmp", "--exclude", "temp"],
             'spectro/templates/basis_templates': ["--exclude", ".svn", "--exclude", "basis_templates_svn-old"],
             'survey/ops/surveyops/trunk': ["--exclude", ".svn", "--exclude", "cronupdate.log"],

From bf26b40a0a5dde6224985a6191c2d670cde7cdf2 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Tue, 17 Sep 2024 12:11:02 -0700
Subject: [PATCH 11/12] fix style

---
 py/desitransfer/tucson.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 8547fac..13141e1 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -70,8 +70,8 @@
                                     "--exclude", "exposures", "--exclude", "preproc",
                                     "--exclude", "temp", "--exclude", "tiles"],
             'spectro/redux/daily/exposures': exclude_years(2019) + ["--exclude", "*.tmp"],
-            'spectro/redux/daily/preproc': exclude_years(2019) +  ["--exclude", "*.tmp", "--exclude", "preproc-*.fits",
-                                                                   "--exclude", "preproc-*.fits.gz"],
+            'spectro/redux/daily/preproc': exclude_years(2019) + ["--exclude", "*.tmp", "--exclude", "preproc-*.fits",
+                                                                  "--exclude", "preproc-*.fits.gz"],
             'spectro/redux/daily/tiles': ["--exclude", "*.tmp", "--exclude", "temp"],
             'spectro/templates/basis_templates': ["--exclude", ".svn", "--exclude", "basis_templates_svn-old"],
             'survey/ops/surveyops/trunk': ["--exclude", ".svn", "--exclude", "cronupdate.log"],

From 12003761435786bceedc224e3e42ad37039157a5 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Fri, 20 Sep 2024 11:27:48 -0700
Subject: [PATCH 12/12] remove exclude from hwtables

---
 bin/desi_tucson_transfer_catchup.sh | 2 +-
 py/desitransfer/tucson.py           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh
index d474c3f..cab1499 100755
--- a/bin/desi_tucson_transfer_catchup.sh
+++ b/bin/desi_tucson_transfer_catchup.sh
@@ -30,7 +30,7 @@ for d in engineering/focalplane engineering/focalplane/hwtables \
     spectro/nightwatch/kpno spectro/staging/lost+found; do
     case ${d} in
         engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;;
-        engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;
+        # engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;
         spectro/data) priority=''; exclude='--exclude 2018* --exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022* --exclude 2023*' ;;
         spectro/nightwatch/kpno) priority='nice'; exclude='--exclude 2021* --exclude 2022* --exclude 2023*' ;;
         spectro/redux/daily) priority=''; exclude='--exclude *.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles' ;;
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 13141e1..1f5e999 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -62,7 +62,7 @@
 
 includes = {'engineering/focalplane': ["--exclude", "archive", "--exclude", "hwtables",
                                        "--exclude", ".ipynb_checkpoints", "--exclude", "*.ipynb"],
-            'engineering/focalplane/hwtables': ["--include", "*.csv", "--exclude", "*"],
+            # 'engineering/focalplane/hwtables': ["--include", "*.csv", "--exclude", "*"],
             'spectro/desi_spectro_calib': ["--exclude", ".svn"],
             'spectro/data': exclude_years(2018),
             'spectro/nightwatch/kpno': exclude_years(2021),