diff --git a/py/desispec/io/fibermap.py b/py/desispec/io/fibermap.py index ee40a9c68..376031bbf 100644 --- a/py/desispec/io/fibermap.py +++ b/py/desispec/io/fibermap.py @@ -1118,7 +1118,8 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None, except KeyError: log.debug("No camera {} in this file".format(camera)) continue - cfinder=CalibFinder([rawheader,camheader]) + + cfinder=CalibFinder([rawheader,camheader], fallback_on_dark_not_found=True) for key in badfibers_keywords_and_maskbits.keys() : newbadfibers = cfinder.badfibers([key]) if newbadfibers.size > 0 : diff --git a/py/desispec/test/test_binscripts.py b/py/desispec/test/test_binscripts.py index e8048ee8c..21523358f 100644 --- a/py/desispec/test/test_binscripts.py +++ b/py/desispec/test/test_binscripts.py @@ -2,6 +2,8 @@ import os, sys import unittest from uuid import uuid4 +import shutil +import tempfile import numpy as np @@ -26,6 +28,9 @@ class TestBinScripts(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.nspec = 6 cls.nwave = 2000 # Needed for QA cls.wave = 4000+np.arange(cls.nwave) @@ -86,16 +91,22 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): """Cleanup in case tests crashed and left files behind""" - for filename in [cls.framefile, cls.fiberflatfile, cls.fibermapfile, \ - cls.skyfile, cls.calibfile, cls.stdfile, cls.qa_calib_file, - cls.qa_data_file, cls.modelfile, cls.qafig]: - if os.path.exists(filename): - os.remove(filename) + + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + if cls.origPath is None: del os.environ['PYTHONPATH'] else: os.environ['PYTHONPATH'] = cls.origPath + #- back to where we started + os.chdir(cls.origdir) + + def setUp(self): + os.chdir(self.testdir) + def _write_frame(self, flavor='none', camera='b3', expid=1, night='20160607',gaia_only=False): """Write a fake frame""" flux = np.ones((self.nspec, self.nwave)) diff --git a/py/desispec/test/test_bootcalib.py b/py/desispec/test/test_bootcalib.py index 344f4a9a6..699727b7c 100644 --- a/py/desispec/test/test_bootcalib.py +++ b/py/desispec/test/test_bootcalib.py @@ -4,6 +4,8 @@ import unittest from uuid import uuid1 +import tempfile +import shutil import os import numpy as np import glob @@ -21,6 +23,9 @@ class TestBoot(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.testarc = 'test_arc.fits.gz' cls.testflat = 'test_flat.fits.gz' cls.testout = 'test_bootcalib_{}.fits'.format(uuid1()) @@ -28,8 +33,8 @@ def setUpClass(cls): cls.data_unavailable = False # Grab the data - url_arc = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000000.fits.gz' - url_flat = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000001.fits.gz' + url_arc = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_arc.fits.gz' + url_flat = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_flat.fits.gz' for url, outfile in [(url_arc, cls.testarc), (url_flat, cls.testflat)]: if not os.path.exists(outfile): try: @@ -47,17 +52,14 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - """We deliberately don't clean up the testarc and testflat files, - since they are useful for offline testing. - """ - # if os.path.exists(cls.testarc): - # os.unlink(cls.testarc) - # if os.path.exists(cls.testflat): - # os.unlink(cls.testflat) - if os.path.exists(cls.testout): - os.unlink(cls.testout) - if os.path.isfile(cls.qafile): - os.unlink(cls.qafile) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + + os.chdir(cls.origdir) + + def setUp(self): + os.chdir(self.testdir) def test_fiber_peaks(self): if self.data_unavailable: diff --git a/py/desispec/test/test_calibfinder.py b/py/desispec/test/test_calibfinder.py index 21f2da582..dbd379fb7 100644 --- a/py/desispec/test/test_calibfinder.py +++ b/py/desispec/test/test_calibfinder.py @@ -7,40 +7,74 @@ import os import shutil from importlib import resources - - +import tempfile from desispec.calibfinder import CalibFinder +_standard_calib_dirs = ('DESI_SPECTRO_CALIB' in os.environ) and ('DESI_SPECTRO_DARK' in os.environ) + class TestCalibFinder(unittest.TestCase): """Test desispec.calibfinder """ - def tearDown(self): - pass - if os.path.isdir(self.calibdir) : - shutil.rmtree(self.calibdir) - - def setUp(self): - #- Create temporary calib directory - self.calibdir = os.path.join(os.environ['HOME'], 'preproc_unit_test') - if not os.path.exists(self.calibdir): os.makedirs(self.calibdir) - #- Copy test calibration-data.yaml file - specdir=os.path.join(self.calibdir,"spec/sp0") - if not os.path.isdir(specdir) : - os.makedirs(specdir) + + @classmethod + def setUpClass(cls): + + #- Cache original environment + cls.origenv = dict() + for key in ['DESI_SPECTRO_CALIB', 'DESI_SPECTRO_DARK']: + cls.origenv[key] = os.getenv(key, None) + + #- Prepare alternate $DESI_SPECTRO_CALIB for testing + cls.calibdir = tempfile.mkdtemp() + specdir = os.path.join(cls.calibdir,"spec/sp0") + os.makedirs(specdir) for c in "brz" : shutil.copy(str(resources.files('desispec').joinpath(f'test/data/ql/{c}0.yaml')), os.path.join(specdir,f"{c}0.yaml")) - #- Set calibration environment variable - os.environ["DESI_SPECTRO_CALIB"] = self.calibdir - + + @classmethod + def tearDownClass(cls): + #- remove temporary calibration directory + if cls.calibdir.startswith(tempfile.gettempdir()) and os.path.isdir(cls.calibdir) : + shutil.rmtree(cls.calibdir) + + def tearDown(self): + #- restore original environment after every test; + #- some tests use default env; others use alternate $DESI_SPECTRO_CALIB + for key, value in self.origenv.items(): + if value is not None: + os.environ[key] = value + elif key in os.environ: + del os.environ[key] def test_init(self): - """Cleanup test files if they exist. + """Test basic initialization using test $DESI_SPECTRO_CALIB """ - + os.environ["DESI_SPECTRO_CALIB"] = self.calibdir pheader={"DATE-OBS":'2018-11-30T12:42:10.442593-05:00',"DOSVER":'SIM'} header={"DETECTOR":'SIM',"CAMERA":'b0 ',"FEEVER":'SIM'} cfinder = CalibFinder([pheader,header]) print(cfinder.value("DETECTOR")) if cfinder.haskey("BIAS") : print(cfinder.findfile("BIAS")) + + @unittest.skipIf(not _standard_calib_dirs, "$DESI_SPECTRO_CALIB or $DESI_SPECTRO_DARK not set") + def test_missing_darks(self): + """Missing dark files is only fatal if darks are requested + """ + + #- Commissioning era data from 20200219 expid 51053, for which we don't have + #- darks in $DESI_SPECTRO_DARK + phdr = {"DATE-OBS":"2020-02-20T08:59:59.104576", "DOSVER":"trunk"} + camhdr = {"DETECTOR":"sn22797", "CAMERA":"b0", "FEEVER":"v20160312", "SPECID":4, + "CCDCFG":"default_sta_20190717.cfg", + "CCDTMING":"default_sta_timing_20180905.txt"} + + #- without an entry in DESI_SPECTRO_DARK, even creating the CalibFinder fails + with self.assertRaises(OSError): + cfinder = CalibFinder([phdr,camhdr]) + + #- but fallback option should work + cfinder = CalibFinder([phdr,camhdr], fallback_on_dark_not_found=True) + darkfile = cfinder.findfile('DARK') + self.assertTrue(darkfile is not None) diff --git a/py/desispec/test/test_extract.py b/py/desispec/test/test_extract.py index caaaad600..a1baffb11 100644 --- a/py/desispec/test/test_extract.py +++ b/py/desispec/test/test_extract.py @@ -13,6 +13,8 @@ import unittest import uuid import os +import tempfile +import shutil from glob import glob from importlib import resources @@ -27,6 +29,9 @@ class TestExtract(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.testhash = uuid.uuid4() cls.imgfile = 'test-img-{}.fits'.format(cls.testhash) cls.outfile = 'test-out-{}.fits'.format(cls.testhash) @@ -49,15 +54,18 @@ def setUpClass(cls): cls.img = img def setUp(self): + os.chdir(self.testdir) for filename in (self.outfile, self.outmodel): if os.path.exists(filename): os.remove(filename) @classmethod def tearDownClass(cls): - for filename in glob('test-*{}*.fits'.format(cls.testhash)): - if os.path.exists(filename): - os.remove(filename) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + + os.chdir(cls.origdir) @unittest.skipIf(nospecter, 'specter not installed; skipping extraction test') def test_extract(self): diff --git a/py/desispec/test/test_fiberflat.py b/py/desispec/test/test_fiberflat.py index b63cb7c1d..d9e7faf94 100644 --- a/py/desispec/test/test_fiberflat.py +++ b/py/desispec/test/test_fiberflat.py @@ -7,6 +7,8 @@ import unittest import copy import os +import tempfile +import shutil from uuid import uuid1 import numpy as np @@ -44,15 +46,21 @@ def _get_data(): class TestFiberFlat(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) def setUp(self): + os.chdir(self.testdir) id = uuid1() self.testfibermap = 'test_fibermap_{}.fits'.format(id) self.testframe = 'test_frame_{}.fits'.format(id) self.testflat = 'test_fiberflat_{}.fits'.format(id) - def tearDown(self): + os.chdir(self.testdir) if os.path.isfile(self.testframe): os.unlink(self.testframe) if os.path.isfile(self.testflat): @@ -60,6 +68,14 @@ def tearDown(self): if os.path.isfile(self.testfibermap): os.unlink(self.testfibermap) + @classmethod + def tearDownClass(cls): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + + os.chdir(cls.origdir) + def test_interface(self): """ diff --git a/py/desispec/test/test_pixgroup.py b/py/desispec/test/test_pixgroup.py index 3893accdd..0a6ff5a44 100644 --- a/py/desispec/test/test_pixgroup.py +++ b/py/desispec/test/test_pixgroup.py @@ -19,6 +19,10 @@ class TestPixGroup(unittest.TestCase): def setUpClass(cls): cls.testdir = tempfile.mkdtemp() cls.outdir = os.path.join(cls.testdir, 'output') + + cls.origenv = dict() + for key in ['DESI_SPECTRO_REDUX', 'SPECPROD']: + cls.origenv[key] = os.getenv(key) #- will be None if not set os.environ['DESI_SPECTRO_REDUX'] = cls.testdir os.environ['SPECPROD'] = 'grouptest' @@ -92,8 +96,8 @@ def setUpClass(cls): cls.exptable.write(cls.expfile) # Setup a dummy SpectraLite for I/O tests - cls.fileio = 'test_spectralite.fits' - cls.fileiogz = 'test_spectralite.fits.gz' + cls.fileio = os.path.join(cls.testdir, 'test_spectralite.fits') + cls.fileiogz = os.path.join(cls.testdir, 'test_spectralite.fits.gz') cls.nwave = 100 cls.nspec = 5 @@ -136,9 +140,17 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - if os.path.exists(cls.testdir): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): shutil.rmtree(cls.testdir) + #- restore environment + for key, value in cls.origenv.items(): + if value is not None: + os.environ[key] = value + elif key in os.environ: + del os.environ[key] + def setUp(self): os.environ['DESI_SPECTRO_REDUX'] = self.testdir os.environ['SPECPROD'] = 'grouptest' diff --git a/py/desispec/test/test_qlextract.py b/py/desispec/test/test_qlextract.py index dfc67dcd9..edfe77ff4 100644 --- a/py/desispec/test/test_qlextract.py +++ b/py/desispec/test/test_qlextract.py @@ -12,6 +12,8 @@ import unittest import uuid import os +import tempfile +import shutil from glob import glob from importlib import resources @@ -26,6 +28,9 @@ class TestExtract(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.testhash = uuid.uuid4() cls.imgfile = 'test-img-{}.fits'.format(cls.testhash) cls.outfile = 'test-out-{}.fits'.format(cls.testhash) @@ -45,15 +50,18 @@ def setUpClass(cls): desispec.io.write_fibermap(cls.fibermapfile, fibermap) def setUp(self): + os.chdir(self.testdir) for filename in (self.outfile, self.outmodel): if os.path.exists(filename): os.remove(filename) @classmethod def tearDownClass(cls): - for filename in glob('test-*{}*.fits'.format(cls.testhash)): - if os.path.exists(filename): - os.remove(filename) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + + os.chdir(cls.origdir) def test_boxcar(self): from desispec.quicklook.qlboxcar import do_boxcar diff --git a/py/desispec/test/test_scripts.py b/py/desispec/test/test_scripts.py index 36a44a3b5..a3f3b2359 100644 --- a/py/desispec/test/test_scripts.py +++ b/py/desispec/test/test_scripts.py @@ -7,6 +7,8 @@ import os import unittest +import tempfile +import shutil from uuid import uuid4 from astropy.table import Table @@ -17,6 +19,9 @@ class TestScripts(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) # from os import environ # for k in ('DESI_SPECTRO_REDUX', 'SPECPROD'): # if k in environ: @@ -25,10 +30,15 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + cls.environ_cache.clear() + os.chdir(cls.origdir) def setUp(self): - pass + os.chdir(self.testdir) def tearDown(self): pass diff --git a/py/desispec/test/test_spectra.py b/py/desispec/test/test_spectra.py index b8b771f3d..e5f8ed0fb 100644 --- a/py/desispec/test/test_spectra.py +++ b/py/desispec/test/test_spectra.py @@ -40,7 +40,9 @@ class TestSpectra(unittest.TestCase): @classmethod def setUpClass(cls): """Create specprod directory structure""" + cls.origDir = os.getcwd() cls.testDir = tempfile.mkdtemp() + os.chdir(cls.testDir) cls.origEnv = { "SPECPROD": None, "DESI_SPECTRO_REDUX": None, @@ -72,15 +74,19 @@ def tearDownClass(cls): else: os.environ[e] = cls.origEnv[e] - if os.path.exists(cls.testDir): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testDir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testDir): shutil.rmtree(cls.testDir) + os.chdir(cls.origDir) + def setUp(self): #- catch specific warnings so that we can find and fix # warnings.filterwarnings("error", ".*did not parse as fits unit.*") #- Test data and files to work with + os.chdir(self.testDir) self.fileio = "test_spectra.fits" self.fileappend = "test_spectra_append.fits" self.filebuild = "test_spectra_build.fits" diff --git a/py/desispec/test/test_util.py b/py/desispec/test/test_util.py index 32b43d4c7..fa1bcb1f5 100644 --- a/py/desispec/test/test_util.py +++ b/py/desispec/test/test_util.py @@ -7,6 +7,8 @@ import unittest from uuid import uuid4 import importlib +import tempfile +import shutil import numpy as np from astropy.table import Table @@ -135,7 +137,7 @@ def test_parse_fibers(self): #- TODO: override log level to quiet down error messages that are supposed #- to be there from these tests class TestRunCmd(unittest.TestCase): - + def test_runcmd(self): """Test calling a script""" result, success = util.runcmd('echo hello > /dev/null') @@ -276,12 +278,17 @@ def test_newer_input(self): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) + cls.infile = 'test-'+uuid4().hex cls.outfile = 'test-'+uuid4().hex cls.testfile = 'test-'+uuid4().hex def setUp(self): # refresh timestamps so that outfile is older than infile + os.chdir(self.testdir) for filename in [self.infile, self.outfile]: with open(filename, 'w') as fx: fx.write('This file is leftover from a test; you can remove it\n') @@ -289,9 +296,11 @@ def setUp(self): @classmethod def tearDownClass(cls): - for filename in [cls.infile, cls.outfile, cls.testfile]: - if os.path.exists(filename): - os.remove(filename) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + + os.chdir(cls.origdir) class TestUtil(unittest.TestCase):