diff --git a/bin/desi_tucson_transfer_catchup.sh b/bin/desi_tucson_transfer_catchup.sh index 28ef2ef..a9092bf 100755 --- a/bin/desi_tucson_transfer_catchup.sh +++ b/bin/desi_tucson_transfer_catchup.sh @@ -32,6 +32,7 @@ for d in engineering/focalplane engineering/focalplane/hwtables \ engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;; engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;; spectro/data) priority=''; exclude='--exclude 2018* --exclude 2019* --exclude 2020* --exclude 2021* --exclude 2022* --exclude 2023*' ;; + spectro/nightwatch/kpno) priority='nice'; exclude='--exclude 2021* --exclude 2022* --exclude 2023*' ;; spectro/redux/daily) priority=''; exclude='--exclude *.tmp --exclude attic --exclude exposures --exclude preproc --exclude temp --exclude tiles' ;; spectro/redux/daily/exposures) priority=''; exclude='--exclude *.tmp' ;; spectro/redux/daily/preproc) priority=''; exclude='--exclude *.tmp --exclude preproc-*.fits --exclude preproc-*.fits.gz' ;; diff --git a/py/desitransfer/common.py b/py/desitransfer/common.py index e9029b4..37505a5 100644 --- a/py/desitransfer/common.py +++ b/py/desitransfer/common.py @@ -10,6 +10,7 @@ import os import re import stat +import time import pytz MST = pytz.timezone('America/Phoenix') @@ -34,7 +35,7 @@ def empty_rsync(out): ``True`` if there are no files to transfer. """ rr = re.compile(r'(receiving|sent [0-9]+ bytes|total size)') - return all([rr.match(l) is not None for l in out.split('\n') if l]) + return all([rr.match(out_line) is not None for out_line in out.split('\n') if out_line]) def new_exposures(out): @@ -52,8 +53,8 @@ def new_exposures(out): """ e = set() e_re = re.compile(r'([0-9]{8})/?') - for l in out.split('\n'): - m = e_re.match(l) + for out_line in out.split('\n'): + m = e_re.match(out_line) if m is not None: e.add(m.groups()[0]) return e @@ -125,7 +126,7 @@ def ensure_scratch(directories): """ for d in directories: try: - l = os.listdir(d) + dir_list = os.listdir(d) except FileNotFoundError: continue return d @@ -143,7 +144,7 @@ def today(): This formulation, with the offset ``7/24+0.5``, is inherited from previous nightwatch transfer scripts. """ - return (dt.datetime.utcnow() - dt.timedelta(7/24+0.5)).strftime('%Y%m%d') + return (dt.datetime.utcnow() - dt.timedelta(7 / 24 + 0.5)).strftime('%Y%m%d') def idle_time(start=8, end=12, tz=None): @@ -174,3 +175,19 @@ def idle_time(start=8, end=12, tz=None): return (i - s) // dt.timedelta(seconds=1) e = dt.datetime(i.year, i.month, i.day, end, 0, 0, tzinfo=tz) return (e - i) // dt.timedelta(seconds=1) + + +def exclude_years(start_year): + """Generate rsync ``--exclude`` statements of the form ``--exclude 2020*``. + + Parameters + ---------- + start_year : :class:`int` + First year to exclude. + + Returns + ------- + :class:`list` + A list suitable for appending to a command. + """ + return (' '.join([f'--exclude {y:d}*' for y in range(start_year, time.localtime().tm_year)])).split() diff --git a/py/desitransfer/daemon.py b/py/desitransfer/daemon.py index ca67715..202587a 100644 --- a/py/desitransfer/daemon.py +++ b/py/desitransfer/daemon.py @@ -78,11 +78,10 @@ def __init__(self, options): self._ini = options.configuration self.test = options.test self.tape = options.backup - getlist = lambda x: x.split(',') - getdict = lambda x: dict([tuple(i.split(':')) for i in x.split(',')]) self.conf = ConfigParser(defaults=os.environ, strict=True, interpolation=ExtendedInterpolation(), - converters={'list': getlist, 'dict': getdict}) + converters={'list': lambda x: x.split(','), + 'dict': lambda x: dict([tuple(i.split(':')) for i in x.split(',')])}) files = self.conf.read(self._ini) # assert files[0] == self._ini self.sections = [s for s in self.conf.sections() @@ -183,11 +182,11 @@ def directory(self, d): _, out, err = _popen(cmd) links = sorted([x for x in out.split('\n') if x]) if links: - for l in links: - if self._link_re.search(l) is None: - log.warning("Malformed symlink detected: %s. Skipping.", l) + for link in links: + if self._link_re.search(link) is None: + log.warning("Malformed symlink detected: %s. Skipping.", link) else: - self.exposure(d, l, status) + self.exposure(d, link, status) else: log.warning('No links found, check connection.') # @@ -440,9 +439,9 @@ def backup(self, d, night, status): if self.tape: log.debug(' '.join(cmd)) _, out, err = _popen(cmd) - with open(ls_file) as l: - data = l.read() - backup_files = [l.split()[-1] for l in data.split('\n') if l] + with open(ls_file) as ls_fileobj: + data = ls_fileobj.read() + backup_files = [ls_out.split()[-1] for ls_out in data.split('\n') if ls_out] else: backup_files = [] backup_file = hpss_file + '_' + night + '.tar' @@ -544,9 +543,9 @@ def verify_checksum(checksum_file): checksum_file, n_lines) errors += "{0:d} file(s) listed but not downloaded.\n".format(n_lines) if n_lines < 0: - log.error("%d files are not listed in %s!", -1*n_lines, checksum_file) - errors += "{0:d} file(s) downloaded but not listed.\n".format(-1*n_lines) - digest = dict([(l.split()[1], l.split()[0]) for l in lines if l]) + log.error("%d files are not listed in %s!", -1 * n_lines, checksum_file) + errors += "{0:d} file(s) downloaded but not listed.\n".format(-1 * n_lines) + digest = dict([(cl.split()[1], cl.split()[0]) for cl in lines if cl]) for f in files: ff = os.path.join(d, f) if ff != checksum_file: @@ -668,5 +667,5 @@ def main(): options.kill) return 0 transfer.transfer() - time.sleep(sleep*60) + time.sleep(sleep * 60) return 0 diff --git a/py/desitransfer/daily.py b/py/desitransfer/daily.py index ef67921..c66ec8c 100644 --- a/py/desitransfer/daily.py +++ b/py/desitransfer/daily.py @@ -56,14 +56,14 @@ def transfer(self, permission=True): if self.extra: for i, e in enumerate(self.extra): cmd.insert(cmd.index('--omit-dir-times') + 1 + i, e) - with open(self.log, 'ab') as l: - l.write(("DEBUG: desi_daily_transfer %s\n" % dtVersion).encode('utf-8')) - l.write(("DEBUG: %s\n" % ' '.join(cmd)).encode('utf-8')) - l.write(("DEBUG: Transfer start: %s\n" % stamp()).encode('utf-8')) - l.flush() - p = sub.Popen(cmd, stdout=l, stderr=sub.STDOUT) + with open(self.log, 'ab') as logfile: + logfile.write(("DEBUG: desi_daily_transfer %s\n" % dtVersion).encode('utf-8')) + logfile.write(("DEBUG: %s\n" % ' '.join(cmd)).encode('utf-8')) + logfile.write(("DEBUG: Transfer start: %s\n" % stamp()).encode('utf-8')) + logfile.flush() + p = sub.Popen(cmd, stdout=logfile, stderr=sub.STDOUT) status = p.wait() - l.write(("DEBUG: Transfer complete: %s\n" % stamp()).encode('utf-8')) + logfile.write(("DEBUG: Transfer complete: %s\n" % stamp()).encode('utf-8')) if status == 0: self.lock() if permission: @@ -80,8 +80,8 @@ def lock(self): fpath = os.path.join(dirpath, f) if stat.S_IMODE(os.stat(fpath).st_mode) != file_perm: os.chmod(fpath, file_perm) - with open(self.log, 'ab') as l: - l.write(("DEBUG: Lock complete: %s\n" % stamp()).encode('utf-8')) + with open(self.log, 'ab') as logfile: + logfile.write(("DEBUG: Lock complete: %s\n" % stamp()).encode('utf-8')) def permission(self): """Set permissions for DESI collaboration access. @@ -95,12 +95,12 @@ def permission(self): The status returned by :command:`fix_permissions.sh`. """ cmd = ['fix_permissions.sh', self.destination] - with open(self.log, 'ab') as l: - l.write(("DEBUG: %s\n" % ' '.join(cmd)).encode('utf-8')) - l.flush() - p = sub.Popen(cmd, stdout=l, stderr=sub.STDOUT) + with open(self.log, 'ab') as logfile: + logfile.write(("DEBUG: %s\n" % ' '.join(cmd)).encode('utf-8')) + logfile.flush() + p = sub.Popen(cmd, stdout=logfile, stderr=sub.STDOUT) status = p.wait() - l.write(("DEBUG: Permission reset complete: %s\n" % stamp()).encode('utf-8')) + logfile.write(("DEBUG: Permission reset complete: %s\n" % stamp()).encode('utf-8')) return status diff --git a/py/desitransfer/nightwatch.py b/py/desitransfer/nightwatch.py index e28d4d3..26da9ab 100644 --- a/py/desitransfer/nightwatch.py +++ b/py/desitransfer/nightwatch.py @@ -11,7 +11,8 @@ Catchup on a specific night:: - NIGHT=20200124 && rsync -rlvt --exclude-from ${DESITRANSFER}/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt dts:/exposures/nightwatch/${NIGHT}/ /global/cfs/cdirs/desi/spectro/nightwatch/kpno/${NIGHT}/ + NIGHT=20200124 && rsync -rlvt --exclude-from ${DESITRANSFER}/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt \ + dts:/exposures/nightwatch/${NIGHT}/ /global/cfs/cdirs/desi/spectro/nightwatch/kpno/${NIGHT}/ By-hand startup sequence (bash shell):: @@ -114,7 +115,7 @@ def main(): options = _options() _configure_log(options.debug) errcount = 0 - wait = options.sleep*60 + wait = options.sleep * 60 source = '/exposures/nightwatch' basedir = os.path.join(os.environ['DESI_ROOT'], 'spectro', 'nightwatch') kpnodir = os.path.join(basedir, 'kpno') diff --git a/py/desitransfer/spacewatch.py b/py/desitransfer/spacewatch.py index 5b5bd1f..e5ef418 100644 --- a/py/desitransfer/spacewatch.py +++ b/py/desitransfer/spacewatch.py @@ -117,7 +117,8 @@ def download_jpg(files, destination, overwrite=False, test=False): r = requests.get(jpg) if r.status_code == 200: downloaded += 1 - timestamp = int(datetime.datetime.strptime(r.headers['Last-Modified'], '%a, %d %b %Y %H:%M:%S %Z').replace(tzinfo=utc).timestamp()) + timestamp = int(datetime.datetime.strptime(r.headers['Last-Modified'], + '%a, %d %b %Y %H:%M:%S %Z').replace(tzinfo=utc).timestamp()) with open(dst_jpg, 'wb') as j: j.write(r.content) os.utime(dst_jpg, (timestamp, timestamp)) diff --git a/py/desitransfer/test/test_common.py b/py/desitransfer/test/test_common.py index 30d2ea1..f40f0a0 100644 --- a/py/desitransfer/test/test_common.py +++ b/py/desitransfer/test/test_common.py @@ -7,7 +7,7 @@ from unittest.mock import patch from tempfile import TemporaryDirectory from ..common import (dt, MST, dir_perm, file_perm, empty_rsync, new_exposures, rsync, - stamp, ensure_scratch, yesterday, today, idle_time) + stamp, ensure_scratch, yesterday, today, idle_time, exclude_years) class FakeDateTime(datetime): @@ -135,7 +135,7 @@ def test_today(self, mock_dt): """Test today's date. """ mock_dt.datetime.utcnow.return_value = datetime(2019, 7, 3, 5, 0, 0) - mock_dt.timedelta.return_value = timedelta(7/24+0.5) + mock_dt.timedelta.return_value = timedelta(7 / 24 + 0.5) y = today() self.assertEqual(y, '20190702') @@ -174,3 +174,11 @@ def test_idle_time_alt_time_zone(self): # mock_datetime.return_value = datetime(2021, 7, 3, 13, 0, 0, tzinfo=MST) i = idle_time(tz='US/Pacific') self.assertEqual(i, -3600) + + def test_exclude_years(self): + """Test exclude statements for a range of years. + """ + last_year = datetime.now().year - 1 + ex = exclude_years(2018) + self.assertEqual(ex[1], '2018*') + self.assertEqual(ex[-1], f'{last_year:d}*') diff --git a/py/desitransfer/test/test_daemon.py b/py/desitransfer/test/test_daemon.py index e7d41a4..7067157 100644 --- a/py/desitransfer/test/test_daemon.py +++ b/py/desitransfer/test/test_daemon.py @@ -316,7 +316,8 @@ def test_TransferDaemon_exposure_transfer(self, mock_cl, mock_log, mock_status, mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'), call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750), - call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), + call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'), call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'), call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'), @@ -369,7 +370,8 @@ def test_TransferDaemon_exposure_transfer_testmode(self, mock_cl, mock_log, mock mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'), call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750), - call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), + call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'), call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'True'), call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'), @@ -416,7 +418,8 @@ def test_TransferDaemon_exposure_rsync_failure(self, mock_cl, mock_log, mock_sta mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'), call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750), - call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), + call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), call("status.update('%s', '%s', 'rsync', failure=True)", '20190703', '00000127'), call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'), call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'), @@ -465,7 +468,8 @@ def test_TransferDaemon_exposure_checksum_missing(self, mock_cl, mock_log, mock_ mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'), call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750), - call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), + call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'), call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'), call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'), @@ -519,7 +523,8 @@ def test_TransferDaemon_exposure_checksum_failure(self, mock_cl, mock_log, mock_ mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'), call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750), - call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), + call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'), call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'), call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'), call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'), @@ -830,7 +835,8 @@ def test_TransferDaemon_backup_test(self, mock_cl, mock_log, mock_status, mock_i mock_log.debug.assert_has_calls([call("os.remove('%s')", ls_file), call("Failed to remove %s because it didn't exist. That's OK.", ls_file), call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)), - call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), + call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), call("os.chdir('%s')", '/desi/root/spectro/data'), call('%s -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703' % htar), call("os.chdir('%s')", self.tmp.name)]) @@ -877,7 +883,8 @@ def test_TransferDaemon_backup_no_test(self, mock_cl, mock_log, mock_status, moc mock_log.info.assert_has_calls([call('No files appear to have changed in %s.', '20190703')]) mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')), call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)), - call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), + call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2550), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001234', 0o2550), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001235', 0o2550), @@ -900,7 +907,11 @@ def test_TransferDaemon_backup_no_test(self, mock_cl, mock_log, mock_status, moc @patch('desitransfer.daemon.TransferStatus') @patch('desitransfer.daemon.log') @patch.object(TransferDaemon, '_configure_log') - def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status, mock_isdir, mock_rm, mock_popen, mock_empty, mock_getcwd, mock_chdir, mock_rsync, mock_chmod, mock_walk): + def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, + mock_status, mock_isdir, mock_rm, + mock_popen, mock_empty, mock_getcwd, + mock_chdir, mock_rsync, mock_chmod, + mock_walk): """Test HPSS backup of night with htar failure. """ with patch.dict('os.environ', @@ -916,7 +927,9 @@ def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status ('/desi/root/spectro/data/20190703/00001234', [], ['f1']), ('/desi/root/spectro/data/20190703/00001235', [], ['f2'])] mock_empty.return_value = True - mock_popen.return_value = ('1', '', 'Generating .netrc entry...\nMust run interactively to update .netrc\nUnable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n') + mock_popen.return_value = ('1', '', 'Generating .netrc entry...\n' + + 'Must run interactively to update .netrc\n' + + 'Unable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n') mock_getcwd.return_value = 'HOME' ls_file = os.path.join(self.tmp.name, 'desi_spectro_data.txt') with open(ls_file, 'w') as f: @@ -928,15 +941,18 @@ def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status mock_log.info.assert_has_calls([call('No files appear to have changed in %s.', '20190703')]) mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')), call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)), - call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), + call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2550), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001234', 0o2550), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001235', 0o2550), call("os.chdir('%s')", '/desi/root/spectro/data'), call('%s -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703' % htar), call("os.chdir('%s')", 'HOME')]) - mock_log.critical.assert_has_calls([call(("HTAR Backup failed! Command was: {0} -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703.".format(htar) + - "\nHTAR error message was: Generating .netrc entry...\nMust run interactively to update .netrc\nUnable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n"))]) + mock_log.critical.assert_has_calls([call("HTAR Backup failed! Command was: {0} -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703.".format(htar) + + "\nHTAR error message was: Generating .netrc entry...\n" + + "Must run interactively to update .netrc\n" + + "Unable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n")]) mock_popen.assert_has_calls([call([htar, '-cvhf', 'desi/spectro/data/desi_spectro_data_20190703.tar', '-H', 'crc:verify=all', '20190703'])]) mock_status.assert_not_called() mock_status.update.assert_not_called() @@ -953,7 +969,10 @@ def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status @patch('desitransfer.daemon.TransferStatus') @patch('desitransfer.daemon.log') @patch.object(TransferDaemon, '_configure_log') - def test_TransferDaemon_backup_delayed_data(self, mock_cl, mock_log, mock_status, mock_isdir, mock_rm, mock_popen, mock_empty, mock_getcwd, mock_chdir, mock_rsync, mock_chmod, mock_walk): + def test_TransferDaemon_backup_delayed_data(self, mock_cl, mock_log, mock_status, + mock_isdir, mock_rm, mock_popen, + mock_empty, mock_getcwd, mock_chdir, + mock_rsync, mock_chmod, mock_walk): """Test HPSS backup of night with delayed data. """ with patch.dict('os.environ', @@ -979,7 +998,8 @@ def test_TransferDaemon_backup_delayed_data(self, mock_cl, mock_log, mock_status htar = os.path.join(transfer.conf['common']['hpss'], 'htar') mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')), call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)), - call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), + call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' + + 'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2550), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001234', 0o2550), call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001235', 0o2550), @@ -1045,49 +1065,48 @@ def test_verify_checksum(self): d = os.path.dirname(c) with patch('os.listdir') as mock_listdir: mock_listdir.return_value = ['t.sha256sum', 'test_file_1.txt', 'test_file_2.txt'] - with patch('desitransfer.daemon.log') as l: + with patch('desitransfer.daemon.log') as mock_log: o = verify_checksum(c) self.assertEqual(o, "") - l.debug.assert_has_calls([call("%s is valid.", os.path.join(d, 'test_file_1.txt')), - call("%s is valid.", os.path.join(d, 'test_file_2.txt'))]) + mock_log.debug.assert_has_calls([call("%s is valid.", os.path.join(d, 'test_file_1.txt')), + call("%s is valid.", os.path.join(d, 'test_file_2.txt'))]) # # Wrong number of files. # with patch('os.listdir') as mock_listdir: mock_listdir.return_value = ['t.sha256sum', 'test_file_1.txt'] - with patch('desitransfer.daemon.log') as l: + with patch('desitransfer.daemon.log') as mock_log: o = verify_checksum(c) self.assertEqual(o, "1 file(s) listed but not downloaded.\n") - l.error.assert_has_calls([call("%s lists %d file(s) that are not present!", c, 1)]) + mock_log.error.assert_has_calls([call("%s lists %d file(s) that are not present!", c, 1)]) with patch('os.listdir') as mock_listdir: mock_listdir.return_value = ['t.sha256sum', 'test_file_1.txt', 'test_file_2.txt', 'test_file_3.txt'] - with patch('desitransfer.daemon.log') as l: + with patch('desitransfer.daemon.log') as mock_log: o = verify_checksum(c) self.assertEqual(o, "1 file(s) downloaded but not listed.\ntest_file_3.txt not listed in checksum file.\n") - l.error.assert_has_calls([call("%d files are not listed in %s!", 1, c)]) + mock_log.error.assert_has_calls([call("%d files are not listed in %s!", 1, c)]) # # Bad list of files. # with patch('os.listdir') as mock_listdir: mock_listdir.return_value = ['t.sha256sum', 'test_file_1.txt', 'test_file_3.txt'] - with patch('desitransfer.daemon.log') as l: + with patch('desitransfer.daemon.log') as mock_log: o = verify_checksum(c) self.assertEqual(o, "test_file_3.txt not listed in checksum file.\n") - l.debug.assert_has_calls([call("%s is valid.", os.path.join(d, 'test_file_1.txt'))]) - l.error.assert_has_calls([call("%s does not appear in %s!", os.path.join(d, 'test_file_3.txt'), c)]) + mock_log.debug.assert_has_calls([call("%s is valid.", os.path.join(d, 'test_file_1.txt'))]) + mock_log.error.assert_has_calls([call("%s does not appear in %s!", os.path.join(d, 'test_file_3.txt'), c)]) # # Hack hashlib to produce incorrect checksums. # with patch('os.listdir') as mock_listdir: mock_listdir.return_value = ['t.sha256sum', 'test_file_1.txt', 'test_file_2.txt'] - with patch('desitransfer.daemon.log') as l: - with patch('hashlib.sha256') as h: - # h.sha256 = MagicMock() - h.hexdigest.return_value = 'abcdef' + with patch('desitransfer.daemon.log') as mock_log: + with patch('hashlib.sha256') as mock_hash: + mock_hash.hexdigest.return_value = 'abcdef' o = verify_checksum(c) self.assertEqual(o, "test_file_1.txt had a checksum mismatch.\ntest_file_2.txt had a checksum mismatch.\n") - l.error.assert_has_calls([call("Checksum mismatch for %s in %s!", os.path.join(d, 'test_file_1.txt'), c), - call("Checksum mismatch for %s in %s!", os.path.join(d, 'test_file_2.txt'), c)]) + mock_log.error.assert_has_calls([call("Checksum mismatch for %s in %s!", os.path.join(d, 'test_file_1.txt'), c), + call("Checksum mismatch for %s in %s!", os.path.join(d, 'test_file_2.txt'), c)]) @patch('os.walk') @patch('os.chmod') diff --git a/py/desitransfer/test/test_status.py b/py/desitransfer/test/test_status.py index 8f1a0ed..6cfc073 100644 --- a/py/desitransfer/test/test_status.py +++ b/py/desitransfer/test/test_status.py @@ -75,17 +75,17 @@ def test_TransferStatus_init(self): # New directory. # d = '/desi/spectro/status' - with patch('desitransfer.status.log') as l: - with patch('os.makedirs') as m: - with patch('shutil.copy') as cp: - with patch('shutil.copyfile') as cf: + with patch('desitransfer.status.log') as mock_log: + with patch('os.makedirs') as mock_makedirs: + with patch('shutil.copy') as mock_copy: + with patch('shutil.copyfile') as mock_copyfile: s = TransferStatus(d) - l.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", d), - call("shutil.copyfile('%s', '%s')", h, os.path.join(d, 'index.html')), - call("shutil.copy('%s', '%s')", j, d)]) - m.assert_called_once_with(d, exist_ok=True) - cp.assert_called_once_with(j, d) - cf.assert_called_once_with(h, os.path.join(d, 'index.html')) + mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", d), + call("shutil.copyfile('%s', '%s')", h, os.path.join(d, 'index.html')), + call("shutil.copy('%s', '%s')", j, d)]) + mock_makedirs.assert_called_once_with(d, exist_ok=True) + mock_copy.assert_called_once_with(j, d) + mock_copyfile.assert_called_once_with(h, os.path.join(d, 'index.html')) @patch('desitransfer.status.log') def test_TransferStatus_handle_malformed_with_log(self, mock_log): diff --git a/py/desitransfer/test/test_top_level.py b/py/desitransfer/test/test_top_level.py index 22588b4..8393d93 100644 --- a/py/desitransfer/test/test_top_level.py +++ b/py/desitransfer/test/test_top_level.py @@ -13,8 +13,7 @@ class TestTopLevel(unittest.TestCase): @classmethod def setUpClass(cls): - cls.versionre = re.compile( - r'([0-9]+!)?([0-9]+)(\.[0-9]+)*((a|b|rc|\.post|\.dev)[0-9]+)?') + cls.versionre = re.compile(r'([0-9]+!)?([0-9]+)(\.[0-9]+)*((a|b|rc|\.post|\.dev)[0-9]+)?') @classmethod def tearDownClass(cls): diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py index d2d22a3..09edeb8 100644 --- a/py/desitransfer/tucson.py +++ b/py/desitransfer/tucson.py @@ -14,6 +14,7 @@ from logging.handlers import SMTPHandler import requests from . import __version__ as dtVersion +from .common import exclude_years from desiutil.log import get_logger @@ -58,17 +59,23 @@ 'software/CiscoSecureClient'] -includes = {'engineering/focalplane': ["--exclude", "archive", "--exclude", "hwtables", "--exclude", ".ipynb_checkpoints", "--exclude", "*.ipynb"], +includes = {'engineering/focalplane': ["--exclude", "archive", "--exclude", "hwtables", + "--exclude", ".ipynb_checkpoints", "--exclude", "*.ipynb"], 'engineering/focalplane/hwtables': ["--include", "*.csv", "--exclude", "*"], 'spectro/desi_spectro_calib': ["--exclude", ".svn"], - 'spectro/data': (' '.join([f'--exclude {y:d}*' for y in range(2018, time.localtime().tm_year)])).split(), - 'spectro/redux/daily': ["--exclude", "*.tmp", "--exclude", "attic", "--exclude", "exposures", "--exclude", "preproc", "--exclude", "temp", "--exclude", "tiles"], + 'spectro/data': exclude_years(2018), + 'spectro/nightwatch/kpno': exclude_years(2021), + 'spectro/redux/daily': ["--exclude", "*.tmp", "--exclude", "attic", + "--exclude", "exposures", "--exclude", "preproc", + "--exclude", "temp", "--exclude", "tiles"], 'spectro/redux/daily/exposures': ["--exclude", "*.tmp"], - 'spectro/redux/daily/preproc': ["--exclude", "*.tmp", "--exclude", "preproc-*.fits", "--exclude", "preproc-*.fits.gz"], + 'spectro/redux/daily/preproc': ["--exclude", "*.tmp", "--exclude", "preproc-*.fits", + "--exclude", "preproc-*.fits.gz"], 'spectro/redux/daily/tiles': ["--exclude", "*.tmp", "--exclude", "temp"], 'spectro/templates/basis_templates': ["--exclude", ".svn", "--exclude", "basis_templates_svn-old"], 'survey/ops/surveyops/trunk': ["--exclude", ".svn", "--exclude", "cronupdate.log"], - 'target/catalogs': ["--include", "dr8", "--include", "dr9", "--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]} + 'target/catalogs': ["--include", "dr8", "--include", "dr9", + "--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]} def _configure_log(debug): @@ -239,7 +246,7 @@ def main(): for s in suffix: if options.sleep.endswith(s): try: - sleepy_time = int(options.sleep[0:-1])*suffix[s] + sleepy_time = int(options.sleep[0:-1]) * suffix[s] except ValueError: log.error("Invalid value for sleep interval: '%s'!", options.sleep) return 1 diff --git a/setup.cfg b/setup.cfg index a5faf6c..b94f48f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -115,9 +115,8 @@ exclude_lines = # These are normally ignored by default: # ignore = E121, E123, E126, E133, E226, E241, E242, E704, W503, W504 # -# In addition to the default set we add: -# - E501: line too long (82 > 79 characters) -# - E731: do not assign a lambda expression, use a def -# - E741: do not use variables named 'l', 'O', or 'I' -- because, for example, -# 'l' might refer to Galactic longitude. -ignore = E121, E123, E126, E133, E226, E241, E242, E501, E704, E731, E741, W503, W504 +# These are the explicitly ignored styles: +# - W504: line break after binary operator +# - E501: not ignored, but 190 will eventually be reduced. +max-line-length = 190 +ignore = W504