Skip to content

Commit

Permalink
clean up some long lines
Browse files Browse the repository at this point in the history
  • Loading branch information
weaverba137 committed Jun 3, 2024
1 parent 7b55451 commit 80f5156
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 22 deletions.
3 changes: 2 additions & 1 deletion py/desitransfer/nightwatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
Catchup on a specific night::
NIGHT=20200124 && rsync -rlvt --exclude-from ${DESITRANSFER}/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt dts:/exposures/nightwatch/${NIGHT}/ /global/cfs/cdirs/desi/spectro/nightwatch/kpno/${NIGHT}/
NIGHT=20200124 && rsync -rlvt --exclude-from ${DESITRANSFER}/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt \
dts:/exposures/nightwatch/${NIGHT}/ /global/cfs/cdirs/desi/spectro/nightwatch/kpno/${NIGHT}/
By-hand startup sequence (bash shell)::
Expand Down
3 changes: 2 additions & 1 deletion py/desitransfer/spacewatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,8 @@ def download_jpg(files, destination, overwrite=False, test=False):
r = requests.get(jpg)
if r.status_code == 200:
downloaded += 1
timestamp = int(datetime.datetime.strptime(r.headers['Last-Modified'], '%a, %d %b %Y %H:%M:%S %Z').replace(tzinfo=utc).timestamp())
timestamp = int(datetime.datetime.strptime(r.headers['Last-Modified'],
'%a, %d %b %Y %H:%M:%S %Z').replace(tzinfo=utc).timestamp())
with open(dst_jpg, 'wb') as j:
j.write(r.content)
os.utime(dst_jpg, (timestamp, timestamp))
Expand Down
48 changes: 34 additions & 14 deletions py/desitransfer/test/test_daemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,8 @@ def test_TransferDaemon_exposure_transfer(self, mock_cl, mock_log, mock_status,
mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'),
call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times' +
'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'),
call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'),
call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'),
Expand Down Expand Up @@ -369,7 +370,8 @@ def test_TransferDaemon_exposure_transfer_testmode(self, mock_cl, mock_log, mock
mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'),
call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'),
call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'True'),
call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'),
Expand Down Expand Up @@ -416,7 +418,8 @@ def test_TransferDaemon_exposure_rsync_failure(self, mock_cl, mock_log, mock_sta
mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'),
call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call("status.update('%s', '%s', 'rsync', failure=True)", '20190703', '00000127'),
call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'),
call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'),
Expand Down Expand Up @@ -465,7 +468,8 @@ def test_TransferDaemon_exposure_checksum_missing(self, mock_cl, mock_log, mock_
mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'),
call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'),
call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'),
call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'),
Expand Down Expand Up @@ -519,7 +523,8 @@ def test_TransferDaemon_exposure_checksum_failure(self, mock_cl, mock_log, mock_
mock_log.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/staging/raw/20190703'),
call("os.makedirs('%s', exist_ok=True)", '/desi/root/spectro/data/20190703'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2750),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call('/bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/00000127/ /desi/root/spectro/staging/raw/20190703/00000127/'),
call("status.update('%s', '%s', 'rsync')", '20190703', '00000127'),
call("lock_directory('%s', %s)", '/desi/root/spectro/staging/raw/20190703/00000127', 'False'),
call("verify_checksum('%s')", '/desi/root/spectro/staging/raw/20190703/00000127/checksum-00000127.sha256sum'),
Expand Down Expand Up @@ -830,7 +835,8 @@ def test_TransferDaemon_backup_test(self, mock_cl, mock_log, mock_status, mock_i
mock_log.debug.assert_has_calls([call("os.remove('%s')", ls_file),
call("Failed to remove %s because it didn't exist. That's OK.", ls_file),
call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call("os.chdir('%s')", '/desi/root/spectro/data'),
call('%s -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703' % htar),
call("os.chdir('%s')", self.tmp.name)])
Expand Down Expand Up @@ -877,7 +883,8 @@ def test_TransferDaemon_backup_no_test(self, mock_cl, mock_log, mock_status, moc
mock_log.info.assert_has_calls([call('No files appear to have changed in %s.', '20190703')])
mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')),
call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2550),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001234', 0o2550),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001235', 0o2550),
Expand All @@ -900,7 +907,11 @@ def test_TransferDaemon_backup_no_test(self, mock_cl, mock_log, mock_status, moc
@patch('desitransfer.daemon.TransferStatus')
@patch('desitransfer.daemon.log')
@patch.object(TransferDaemon, '_configure_log')
def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status, mock_isdir, mock_rm, mock_popen, mock_empty, mock_getcwd, mock_chdir, mock_rsync, mock_chmod, mock_walk):
def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log,
mock_status, mock_isdir, mock_rm,
mock_popen, mock_empty, mock_getcwd,
mock_chdir, mock_rsync, mock_chmod,
mock_walk):
"""Test HPSS backup of night with htar failure.
"""
with patch.dict('os.environ',
Expand All @@ -916,7 +927,9 @@ def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status
('/desi/root/spectro/data/20190703/00001234', [], ['f1']),
('/desi/root/spectro/data/20190703/00001235', [], ['f2'])]
mock_empty.return_value = True
mock_popen.return_value = ('1', '', 'Generating .netrc entry...\nMust run interactively to update .netrc\nUnable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n')
mock_popen.return_value = ('1', '', 'Generating .netrc entry...\n' +
'Must run interactively to update .netrc\n' +
'Unable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n')
mock_getcwd.return_value = 'HOME'
ls_file = os.path.join(self.tmp.name, 'desi_spectro_data.txt')
with open(ls_file, 'w') as f:
Expand All @@ -928,15 +941,18 @@ def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status
mock_log.info.assert_has_calls([call('No files appear to have changed in %s.', '20190703')])
mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')),
call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2550),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001234', 0o2550),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001235', 0o2550),
call("os.chdir('%s')", '/desi/root/spectro/data'),
call('%s -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703' % htar),
call("os.chdir('%s')", 'HOME')])
mock_log.critical.assert_has_calls([call(("HTAR Backup failed! Command was: {0} -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703.".format(htar) +
"\nHTAR error message was: Generating .netrc entry...\nMust run interactively to update .netrc\nUnable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n"))])
mock_log.critical.assert_has_calls([call("HTAR Backup failed! Command was: {0} -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703.".format(htar) +
"\nHTAR error message was: Generating .netrc entry...\n" +
"Must run interactively to update .netrc\n" +
"Unable to update .netrc file\nFor help, see https://docs.nersc.gov/accounts/passwords/\n")])
mock_popen.assert_has_calls([call([htar, '-cvhf', 'desi/spectro/data/desi_spectro_data_20190703.tar', '-H', 'crc:verify=all', '20190703'])])
mock_status.assert_not_called()
mock_status.update.assert_not_called()
Expand All @@ -953,7 +969,10 @@ def test_TransferDaemon_backup_htar_failure(self, mock_cl, mock_log, mock_status
@patch('desitransfer.daemon.TransferStatus')
@patch('desitransfer.daemon.log')
@patch.object(TransferDaemon, '_configure_log')
def test_TransferDaemon_backup_delayed_data(self, mock_cl, mock_log, mock_status, mock_isdir, mock_rm, mock_popen, mock_empty, mock_getcwd, mock_chdir, mock_rsync, mock_chmod, mock_walk):
def test_TransferDaemon_backup_delayed_data(self, mock_cl, mock_log, mock_status,
mock_isdir, mock_rm, mock_popen,
mock_empty, mock_getcwd, mock_chdir,
mock_rsync, mock_chmod, mock_walk):
"""Test HPSS backup of night with delayed data.
"""
with patch.dict('os.environ',
Expand All @@ -979,7 +998,8 @@ def test_TransferDaemon_backup_delayed_data(self, mock_cl, mock_log, mock_status
htar = os.path.join(transfer.conf['common']['hpss'], 'htar')
mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')),
call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times ' +
'dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703', 0o2550),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001234', 0o2550),
call("os.chmod('%s', 0o%o)", '/desi/root/spectro/data/20190703/00001235', 0o2550),
Expand Down
13 changes: 9 additions & 4 deletions py/desitransfer/tucson.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,18 +59,23 @@
'software/CiscoSecureClient']


includes = {'engineering/focalplane': ["--exclude", "archive", "--exclude", "hwtables", "--exclude", ".ipynb_checkpoints", "--exclude", "*.ipynb"],
includes = {'engineering/focalplane': ["--exclude", "archive", "--exclude", "hwtables",
"--exclude", ".ipynb_checkpoints", "--exclude", "*.ipynb"],
'engineering/focalplane/hwtables': ["--include", "*.csv", "--exclude", "*"],
'spectro/desi_spectro_calib': ["--exclude", ".svn"],
'spectro/data': exclude_years(2018),
'spectro/nightwatch/kpno': exclude_years(2021),
'spectro/redux/daily': ["--exclude", "*.tmp", "--exclude", "attic", "--exclude", "exposures", "--exclude", "preproc", "--exclude", "temp", "--exclude", "tiles"],
'spectro/redux/daily': ["--exclude", "*.tmp", "--exclude", "attic",
"--exclude", "exposures", "--exclude", "preproc",
"--exclude", "temp", "--exclude", "tiles"],
'spectro/redux/daily/exposures': ["--exclude", "*.tmp"],
'spectro/redux/daily/preproc': ["--exclude", "*.tmp", "--exclude", "preproc-*.fits", "--exclude", "preproc-*.fits.gz"],
'spectro/redux/daily/preproc': ["--exclude", "*.tmp", "--exclude", "preproc-*.fits",
"--exclude", "preproc-*.fits.gz"],
'spectro/redux/daily/tiles': ["--exclude", "*.tmp", "--exclude", "temp"],
'spectro/templates/basis_templates': ["--exclude", ".svn", "--exclude", "basis_templates_svn-old"],
'survey/ops/surveyops/trunk': ["--exclude", ".svn", "--exclude", "cronupdate.log"],
'target/catalogs': ["--include", "dr8", "--include", "dr9", "--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]}
'target/catalogs': ["--include", "dr8", "--include", "dr9",
"--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]}


def _configure_log(debug):
Expand Down
5 changes: 3 additions & 2 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ exclude_lines =
# ignore = E121, E123, E126, E133, E226, E241, E242, E704, W503, W504
#
# These are the explicitly ignored styles:
# - E501: line too long (82 > 79 characters)
# - W504: line break after binary operator
ignore = E501, W504
# - E501: not ignored, but 190 will eventually be reduced.
max-line-length = 190
ignore = W504

0 comments on commit 80f5156

Please sign in to comment.