From 413e0cb747f53b08f5bb6c3d79da32375b3432cc Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Mon, 9 Sep 2024 14:46:28 -0700 Subject: [PATCH] allow 'nice' transfers --- bin/desi_tucson_transfer_catchup.sh | 3 ++- py/desitransfer/test/test_tucson.py | 16 +++++++++++++++- py/desitransfer/tucson.py | 27 +++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh index a9092bf..b4499e2 100755 --- a/bin/desi_tucson_transfer_catchup.sh +++ b/bin/desi_tucson_transfer_catchup.sh @@ -27,7 +27,8 @@ log_root=${HOME}/Documents/Logfiles for d in engineering/focalplane engineering/focalplane/hwtables \ spectro/data \ spectro/redux/daily spectro/redux/daily/exposures spectro/redux/daily/preproc spectro/redux/daily/tiles \ - spectro/nightwatch/kpno spectro/staging/lost+found; do + spectro/nightwatch/kpno spectro/staging/lost+found \ + software/AnyConnect software/CiscoSecureClient; do case ${d} in engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;; engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;; diff --git a/py/desitransfer/test/test_tucson.py b/py/desitransfer/test/test_tucson.py index b781bc7..bbadbdc 100644 --- a/py/desitransfer/test/test_tucson.py +++ b/py/desitransfer/test/test_tucson.py @@ -169,11 +169,14 @@ def test_running_read_exit(self, mock_log, mock_popen, mock_exists, mock_remove) @patch('subprocess.Popen') @patch('desitransfer.tucson.log') - def test_get_proc(self, mock_log, mock_popen): + @patch('desitransfer.tucson.priority') + def test_get_proc(self, mock_priority, mock_log, mock_popen): """Test the function for generating external procedures. """ directories = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] exclude = set(['d', 'g']) + mock_priority.__contains__ = lambda self, x: x == 'e' or x == 'f' + # mock_priority.__iter__.return_value = ('e', 'f') options = MagicMock() options.test = False options.log = self.temp_dir @@ -186,6 +189,7 @@ def test_get_proc(self, mock_log, mock_popen): self.assertEqual(LOG_B, os.path.join(self.temp_dir, 'desi_tucson_transfer_b.log')) proc, LOG_C, d = _get_proc(directories, exclude, '/src', '/dst', options) self.assertEqual(d, 'c') + options.test = False proc, LOG_E, d = _get_proc(directories, exclude, '/src', '/dst', options) self.assertEqual(d, 'e') proc, LOG_F, d = _get_proc(directories, exclude, '/src', '/dst', options) @@ -196,3 +200,13 @@ def test_get_proc(self, mock_log, mock_popen): self.assertEqual(d, 'i') proc, LOG_J, d = _get_proc(directories, exclude, '/src', '/dst', options) self.assertIsNone(proc) + mock_log.info.assert_has_calls([call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/a/ /dst/a/'), + call("Directory '%s' will be transferred with os.nice(%d)", 'a', 5), + call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/e/ /dst/e/'), + call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/f/ /dst/f/'), + call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/h/ /dst/h/'), + call("Directory '%s' will be transferred with os.nice(%d)", 'h', 5), + call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/i/ /dst/i/'), + call("Directory '%s' will be transferred with os.nice(%d)", 'i', 5)]) + mock_log.warning.assert_has_calls([call('%s skipped at user request.', 'd'), + call('%s skipped at user request.', 'g')]) diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py index 59834a3..0cb764e 100644 --- a/py/desitransfer/tucson.py +++ b/py/desitransfer/tucson.py @@ -55,6 +55,7 @@ 'spectro/redux/daily/preproc', 'spectro/redux/daily/tiles', 'engineering/focalplane', + 'engineering/focalplane/hwtables', 'software/AnyConnect', 'software/CiscoSecureClient'] @@ -78,6 +79,13 @@ "--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]} +priority = ('spectro/data', + 'spectro/redux/daily', + 'spectro/redux/daily/exposures', + 'spectro/redux/daily/preproc', + 'spectro/redux/daily/tiles') + + def _configure_log(debug): """Re-configure the default logger returned by ``desiutil.log``. @@ -169,7 +177,7 @@ def _rsync(src, dst, d, checksum=False): return cmd -def _get_proc(directories, exclude, src, dst, options): +def _get_proc(directories, exclude, src, dst, options, nice=5): """Prepare the next download directory for processing. Parameters @@ -184,6 +192,9 @@ def _get_proc(directories, exclude, src, dst, options): Root destination directory. options : :class:`argparse.Namespace` The parsed command-line options. + nice : :class:`int`, optional. + Lower-priority transfers will be run with this value passed to :func:`os.nice`, + default 5. Returns ------- @@ -191,6 +202,13 @@ def _get_proc(directories, exclude, src, dst, options): A tuple containing information about the process. """ global log + + def preexec_nice(): # pragma: no cover + os.nice(nice) + + def preexec_pass(): # pragma: no cover + pass + try: d = directories.pop(0) while d in exclude: @@ -204,7 +222,12 @@ def _get_proc(directories, exclude, src, dst, options): else: log.info(' '.join(command)) LOG = open(log_file, 'ab') - return (sub.Popen(command, stdout=LOG, stderr=sub.STDOUT), LOG, d) + if d in priority: + preexec_fn = preexec_pass + else: + log.info("Directory '%s' will be transferred with os.nice(%d)", d, nice) + preexec_fn = preexec_nice + return (sub.Popen(command, preexec_fn=preexec_fn, stdout=LOG, stderr=sub.STDOUT), LOG, d) except IndexError: return (None, None, None)