From 8d5f0bc76d64e4929e8870bae6fe5f6348dcaf8f Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 14 Aug 2024 13:22:11 -0700 Subject: [PATCH 1/6] fibermap calibfinder doesn't need darks --- py/desispec/io/fibermap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/desispec/io/fibermap.py b/py/desispec/io/fibermap.py index ee40a9c68..376031bbf 100644 --- a/py/desispec/io/fibermap.py +++ b/py/desispec/io/fibermap.py @@ -1118,7 +1118,8 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None, except KeyError: log.debug("No camera {} in this file".format(camera)) continue - cfinder=CalibFinder([rawheader,camheader]) + + cfinder=CalibFinder([rawheader,camheader], fallback_on_dark_not_found=True) for key in badfibers_keywords_and_maskbits.keys() : newbadfibers = cfinder.badfibers([key]) if newbadfibers.size > 0 : From 546a4554491fb9c1868e638346b2dbf0a2c75567 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 14 Aug 2024 13:26:09 -0700 Subject: [PATCH 2/6] test calibfinder restore env when done --- py/desispec/test/test_calibfinder.py | 70 ++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/py/desispec/test/test_calibfinder.py b/py/desispec/test/test_calibfinder.py index 21f2da582..5defcc116 100644 --- a/py/desispec/test/test_calibfinder.py +++ b/py/desispec/test/test_calibfinder.py @@ -7,7 +7,7 @@ import os import shutil from importlib import resources - +import tempfile from desispec.calibfinder import CalibFinder @@ -15,32 +15,64 @@ class TestCalibFinder(unittest.TestCase): """Test desispec.calibfinder """ - def tearDown(self): - pass - if os.path.isdir(self.calibdir) : - shutil.rmtree(self.calibdir) - - def setUp(self): - #- Create temporary calib directory - self.calibdir = os.path.join(os.environ['HOME'], 'preproc_unit_test') - if not os.path.exists(self.calibdir): os.makedirs(self.calibdir) - #- Copy test calibration-data.yaml file - specdir=os.path.join(self.calibdir,"spec/sp0") - if not os.path.isdir(specdir) : - os.makedirs(specdir) + + @classmethod + def setUpClass(cls): + + #- Cache original environment + cls.origenv = dict() + for key in ['DESI_SPECTRO_CALIB', 'DESI_SPECTRO_DARK']: + cls.origenv[key] = os.getenv(key, None) + + #- Prepare alternate $DESI_SPECTRO_CALIB for testing + cls.calibdir = tempfile.mkdtemp() + specdir = os.path.join(cls.calibdir,"spec/sp0") + os.makedirs(specdir) for c in "brz" : shutil.copy(str(resources.files('desispec').joinpath(f'test/data/ql/{c}0.yaml')), os.path.join(specdir,f"{c}0.yaml")) - #- Set calibration environment variable - os.environ["DESI_SPECTRO_CALIB"] = self.calibdir - + + @classmethod + def tearDownClass(cls): + #- remove temporary calibration directory + if os.path.isdir(cls.calibdir) : + shutil.rmtree(cls.calibdir) + + def tearDown(self): + #- restore original environment after every test; + #- some tests use default env; others use alternate $DESI_SPECTRO_CALIB + for key, value in self.origenv.items(): + if value is not None: + os.environ[key] = value + elif key in os.environ: + del os.environ[key] def test_init(self): - """Cleanup test files if they exist. + """Test basic initialization using test $DESI_SPECTRO_CALIB """ - + os.environ["DESI_SPECTRO_CALIB"] = self.calibdir pheader={"DATE-OBS":'2018-11-30T12:42:10.442593-05:00',"DOSVER":'SIM'} header={"DETECTOR":'SIM',"CAMERA":'b0 ',"FEEVER":'SIM'} cfinder = CalibFinder([pheader,header]) print(cfinder.value("DETECTOR")) if cfinder.haskey("BIAS") : print(cfinder.findfile("BIAS")) + + def test_missing_darks(self): + """Missing dark files is only fatal if darks are requested + """ + + #- Commissioning era data from 20200219 expid 51053, for which we don't have + #- darks in $DESI_SPECTRO_DARK + phdr = {"DATE-OBS":"2020-02-20T08:59:59.104576", "DOSVER":"trunk"} + camhdr = {"DETECTOR":"sn22797", "CAMERA":"b0", "FEEVER":"v20160312", "SPECID":4, + "CCDCFG":"default_sta_20190717.cfg", + "CCDTMING":"default_sta_timing_20180905.txt"} + + #- without an entry in DESI_SPECTRO_DARK, even creating the CalibFinder fails + with self.assertRaises(OSError): + cfinder = CalibFinder([phdr,camhdr]) + + #- but fallback option should work + cfinder = CalibFinder([phdr,camhdr], fallback_on_dark_not_found=True) + darkfile = cfinder.findfile('DARK') + self.assertTrue(darkfile is not None) From dc3f56428a2414b67c58375c5b0274555fabd4e7 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 14 Aug 2024 13:27:12 -0700 Subject: [PATCH 3/6] test bootcalib use tempdir and download from new URL --- py/desispec/test/test_bootcalib.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/py/desispec/test/test_bootcalib.py b/py/desispec/test/test_bootcalib.py index 344f4a9a6..bbc54d94b 100644 --- a/py/desispec/test/test_bootcalib.py +++ b/py/desispec/test/test_bootcalib.py @@ -4,6 +4,7 @@ import unittest from uuid import uuid1 +import tempfile import os import numpy as np import glob @@ -21,6 +22,9 @@ class TestBoot(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.testarc = 'test_arc.fits.gz' cls.testflat = 'test_flat.fits.gz' cls.testout = 'test_bootcalib_{}.fits'.format(uuid1()) @@ -28,8 +32,8 @@ def setUpClass(cls): cls.data_unavailable = False # Grab the data - url_arc = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000000.fits.gz' - url_flat = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000001.fits.gz' + url_arc = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_arc.fits.gz' + url_flat = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_flat.fits.gz' for url, outfile in [(url_arc, cls.testarc), (url_flat, cls.testflat)]: if not os.path.exists(outfile): try: @@ -50,6 +54,7 @@ def tearDownClass(cls): """We deliberately don't clean up the testarc and testflat files, since they are useful for offline testing. """ + os.chdir(cls.testdir) # if os.path.exists(cls.testarc): # os.unlink(cls.testarc) # if os.path.exists(cls.testflat): @@ -59,6 +64,11 @@ def tearDownClass(cls): if os.path.isfile(cls.qafile): os.unlink(cls.qafile) + os.chdir(cls.origdir) + + def setUp(self): + os.chdir(self.testdir) + def test_fiber_peaks(self): if self.data_unavailable: self.skipTest("Failed to download test data.") From 47f597d39ae450cd3421f31568ed6f027bc374fe Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 14 Aug 2024 13:27:43 -0700 Subject: [PATCH 4/6] update tests to use temp directories --- py/desispec/test/test_binscripts.py | 11 +++++++++++ py/desispec/test/test_extract.py | 8 ++++++++ py/desispec/test/test_fiberflat.py | 13 ++++++++++++- py/desispec/test/test_pixgroup.py | 4 ++-- py/desispec/test/test_qlextract.py | 8 ++++++++ py/desispec/test/test_scripts.py | 7 ++++++- py/desispec/test/test_spectra.py | 5 +++++ py/desispec/test/test_util.py | 11 ++++++++++- 8 files changed, 62 insertions(+), 5 deletions(-) diff --git a/py/desispec/test/test_binscripts.py b/py/desispec/test/test_binscripts.py index e8048ee8c..ff0be2e7c 100644 --- a/py/desispec/test/test_binscripts.py +++ b/py/desispec/test/test_binscripts.py @@ -2,6 +2,7 @@ import os, sys import unittest from uuid import uuid4 +import tempfile import numpy as np @@ -26,6 +27,9 @@ class TestBinScripts(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.nspec = 6 cls.nwave = 2000 # Needed for QA cls.wave = 4000+np.arange(cls.nwave) @@ -86,6 +90,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): """Cleanup in case tests crashed and left files behind""" + os.chdir(cls.testdir) for filename in [cls.framefile, cls.fiberflatfile, cls.fibermapfile, \ cls.skyfile, cls.calibfile, cls.stdfile, cls.qa_calib_file, cls.qa_data_file, cls.modelfile, cls.qafig]: @@ -96,6 +101,12 @@ def tearDownClass(cls): else: os.environ['PYTHONPATH'] = cls.origPath + #- back to where we started + os.chdir(cls.origdir) + + def setUp(self): + os.chdir(self.testdir) + def _write_frame(self, flavor='none', camera='b3', expid=1, night='20160607',gaia_only=False): """Write a fake frame""" flux = np.ones((self.nspec, self.nwave)) diff --git a/py/desispec/test/test_extract.py b/py/desispec/test/test_extract.py index caaaad600..1e3eee939 100644 --- a/py/desispec/test/test_extract.py +++ b/py/desispec/test/test_extract.py @@ -13,6 +13,7 @@ import unittest import uuid import os +import tempfile from glob import glob from importlib import resources @@ -27,6 +28,9 @@ class TestExtract(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.testhash = uuid.uuid4() cls.imgfile = 'test-img-{}.fits'.format(cls.testhash) cls.outfile = 'test-out-{}.fits'.format(cls.testhash) @@ -49,16 +53,20 @@ def setUpClass(cls): cls.img = img def setUp(self): + os.chdir(self.testdir) for filename in (self.outfile, self.outmodel): if os.path.exists(filename): os.remove(filename) @classmethod def tearDownClass(cls): + os.chdir(cls.testdir) for filename in glob('test-*{}*.fits'.format(cls.testhash)): if os.path.exists(filename): os.remove(filename) + os.chdir(cls.origdir) + @unittest.skipIf(nospecter, 'specter not installed; skipping extraction test') def test_extract(self): template = "desi_extract_spectra -i {} -p {} -w 7500,7600,0.75 -f {} -s 0 -n 5 --bundlesize 5 -o {} -m {}" diff --git a/py/desispec/test/test_fiberflat.py b/py/desispec/test/test_fiberflat.py index b63cb7c1d..5b2164e83 100644 --- a/py/desispec/test/test_fiberflat.py +++ b/py/desispec/test/test_fiberflat.py @@ -7,6 +7,7 @@ import unittest import copy import os +import tempfile from uuid import uuid1 import numpy as np @@ -44,15 +45,21 @@ def _get_data(): class TestFiberFlat(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) def setUp(self): + os.chdir(self.testdir) id = uuid1() self.testfibermap = 'test_fibermap_{}.fits'.format(id) self.testframe = 'test_frame_{}.fits'.format(id) self.testflat = 'test_fiberflat_{}.fits'.format(id) - def tearDown(self): + os.chdir(self.testdir) if os.path.isfile(self.testframe): os.unlink(self.testframe) if os.path.isfile(self.testflat): @@ -60,6 +67,10 @@ def tearDown(self): if os.path.isfile(self.testfibermap): os.unlink(self.testfibermap) + @classmethod + def tearDownClass(cls): + os.chdir(cls.origdir) + def test_interface(self): """ diff --git a/py/desispec/test/test_pixgroup.py b/py/desispec/test/test_pixgroup.py index 3893accdd..273be77a0 100644 --- a/py/desispec/test/test_pixgroup.py +++ b/py/desispec/test/test_pixgroup.py @@ -92,8 +92,8 @@ def setUpClass(cls): cls.exptable.write(cls.expfile) # Setup a dummy SpectraLite for I/O tests - cls.fileio = 'test_spectralite.fits' - cls.fileiogz = 'test_spectralite.fits.gz' + cls.fileio = os.path.join(cls.testdir, 'test_spectralite.fits') + cls.fileiogz = os.path.join(cls.testdir, 'test_spectralite.fits.gz') cls.nwave = 100 cls.nspec = 5 diff --git a/py/desispec/test/test_qlextract.py b/py/desispec/test/test_qlextract.py index dfc67dcd9..bc0007d16 100644 --- a/py/desispec/test/test_qlextract.py +++ b/py/desispec/test/test_qlextract.py @@ -12,6 +12,7 @@ import unittest import uuid import os +import tempfile from glob import glob from importlib import resources @@ -26,6 +27,9 @@ class TestExtract(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) cls.testhash = uuid.uuid4() cls.imgfile = 'test-img-{}.fits'.format(cls.testhash) cls.outfile = 'test-out-{}.fits'.format(cls.testhash) @@ -45,16 +49,20 @@ def setUpClass(cls): desispec.io.write_fibermap(cls.fibermapfile, fibermap) def setUp(self): + os.chdir(self.testdir) for filename in (self.outfile, self.outmodel): if os.path.exists(filename): os.remove(filename) @classmethod def tearDownClass(cls): + os.chdir(cls.testdir) for filename in glob('test-*{}*.fits'.format(cls.testhash)): if os.path.exists(filename): os.remove(filename) + os.chdir(cls.origdir) + def test_boxcar(self): from desispec.quicklook.qlboxcar import do_boxcar from desispec.io import read_xytraceset diff --git a/py/desispec/test/test_scripts.py b/py/desispec/test/test_scripts.py index 36a44a3b5..46cde5e13 100644 --- a/py/desispec/test/test_scripts.py +++ b/py/desispec/test/test_scripts.py @@ -7,6 +7,7 @@ import os import unittest +import tempfile from uuid import uuid4 from astropy.table import Table @@ -17,6 +18,9 @@ class TestScripts(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) # from os import environ # for k in ('DESI_SPECTRO_REDUX', 'SPECPROD'): # if k in environ: @@ -26,9 +30,10 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.environ_cache.clear() + os.chdir(cls.origdir) def setUp(self): - pass + os.chdir(self.testdir) def tearDown(self): pass diff --git a/py/desispec/test/test_spectra.py b/py/desispec/test/test_spectra.py index b8b771f3d..0328daab4 100644 --- a/py/desispec/test/test_spectra.py +++ b/py/desispec/test/test_spectra.py @@ -40,7 +40,9 @@ class TestSpectra(unittest.TestCase): @classmethod def setUpClass(cls): """Create specprod directory structure""" + cls.origDir = os.getcwd() cls.testDir = tempfile.mkdtemp() + os.chdir(cls.testDir) cls.origEnv = { "SPECPROD": None, "DESI_SPECTRO_REDUX": None, @@ -75,12 +77,15 @@ def tearDownClass(cls): if os.path.exists(cls.testDir): shutil.rmtree(cls.testDir) + os.chdir(cls.origDir) + def setUp(self): #- catch specific warnings so that we can find and fix # warnings.filterwarnings("error", ".*did not parse as fits unit.*") #- Test data and files to work with + os.chdir(self.testDir) self.fileio = "test_spectra.fits" self.fileappend = "test_spectra_append.fits" self.filebuild = "test_spectra_build.fits" diff --git a/py/desispec/test/test_util.py b/py/desispec/test/test_util.py index 32b43d4c7..71ecea9c9 100644 --- a/py/desispec/test/test_util.py +++ b/py/desispec/test/test_util.py @@ -7,6 +7,7 @@ import unittest from uuid import uuid4 import importlib +import tempfile import numpy as np from astropy.table import Table @@ -135,7 +136,7 @@ def test_parse_fibers(self): #- TODO: override log level to quiet down error messages that are supposed #- to be there from these tests class TestRunCmd(unittest.TestCase): - + def test_runcmd(self): """Test calling a script""" result, success = util.runcmd('echo hello > /dev/null') @@ -276,12 +277,17 @@ def test_newer_input(self): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls.testdir = tempfile.mkdtemp() + os.chdir(cls.testdir) + cls.infile = 'test-'+uuid4().hex cls.outfile = 'test-'+uuid4().hex cls.testfile = 'test-'+uuid4().hex def setUp(self): # refresh timestamps so that outfile is older than infile + os.chdir(self.testdir) for filename in [self.infile, self.outfile]: with open(filename, 'w') as fx: fx.write('This file is leftover from a test; you can remove it\n') @@ -289,10 +295,13 @@ def setUp(self): @classmethod def tearDownClass(cls): + os.chdir(cls.testdir) for filename in [cls.infile, cls.outfile, cls.testfile]: if os.path.exists(filename): os.remove(filename) + os.chdir(cls.origdir) + class TestUtil(unittest.TestCase): def test_utils_default_nproc(self): From 2ee5c1a9234341b2438788199a1f5ec58501f29b Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 14 Aug 2024 14:22:25 -0700 Subject: [PATCH 5/6] support github tests without DESI_SPECTRO_CALIB set --- py/desispec/test/test_calibfinder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py/desispec/test/test_calibfinder.py b/py/desispec/test/test_calibfinder.py index 5defcc116..334df3710 100644 --- a/py/desispec/test/test_calibfinder.py +++ b/py/desispec/test/test_calibfinder.py @@ -9,9 +9,10 @@ from importlib import resources import tempfile - from desispec.calibfinder import CalibFinder +_standard_calib_dirs = ('DESI_SPECTRO_CALIB' in os.environ) and ('DESI_SPECTRO_DARK' in os.environ) + class TestCalibFinder(unittest.TestCase): """Test desispec.calibfinder """ @@ -57,6 +58,7 @@ def test_init(self): if cfinder.haskey("BIAS") : print(cfinder.findfile("BIAS")) + @unittest.skipIf(not _standard_calib_dirs, "$DESI_SPECTRO_CALIB or $DESI_SPECTRO_DARK not set") def test_missing_darks(self): """Missing dark files is only fatal if darks are requested """ From 2fada6ad983cbd8395468d615af75dde48846c0e Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Thu, 15 Aug 2024 11:59:32 -0700 Subject: [PATCH 6/6] remove test mkdtemp dirs --- py/desispec/test/test_binscripts.py | 12 ++++++------ py/desispec/test/test_bootcalib.py | 16 ++++------------ py/desispec/test/test_calibfinder.py | 2 +- py/desispec/test/test_extract.py | 8 ++++---- py/desispec/test/test_fiberflat.py | 5 +++++ py/desispec/test/test_pixgroup.py | 14 +++++++++++++- py/desispec/test/test_qlextract.py | 8 ++++---- py/desispec/test/test_scripts.py | 5 +++++ py/desispec/test/test_spectra.py | 3 ++- py/desispec/test/test_util.py | 8 ++++---- 10 files changed, 48 insertions(+), 33 deletions(-) diff --git a/py/desispec/test/test_binscripts.py b/py/desispec/test/test_binscripts.py index ff0be2e7c..21523358f 100644 --- a/py/desispec/test/test_binscripts.py +++ b/py/desispec/test/test_binscripts.py @@ -2,6 +2,7 @@ import os, sys import unittest from uuid import uuid4 +import shutil import tempfile import numpy as np @@ -90,12 +91,11 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): """Cleanup in case tests crashed and left files behind""" - os.chdir(cls.testdir) - for filename in [cls.framefile, cls.fiberflatfile, cls.fibermapfile, \ - cls.skyfile, cls.calibfile, cls.stdfile, cls.qa_calib_file, - cls.qa_data_file, cls.modelfile, cls.qafig]: - if os.path.exists(filename): - os.remove(filename) + + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + if cls.origPath is None: del os.environ['PYTHONPATH'] else: diff --git a/py/desispec/test/test_bootcalib.py b/py/desispec/test/test_bootcalib.py index bbc54d94b..699727b7c 100644 --- a/py/desispec/test/test_bootcalib.py +++ b/py/desispec/test/test_bootcalib.py @@ -5,6 +5,7 @@ import unittest from uuid import uuid1 import tempfile +import shutil import os import numpy as np import glob @@ -51,18 +52,9 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - """We deliberately don't clean up the testarc and testflat files, - since they are useful for offline testing. - """ - os.chdir(cls.testdir) - # if os.path.exists(cls.testarc): - # os.unlink(cls.testarc) - # if os.path.exists(cls.testflat): - # os.unlink(cls.testflat) - if os.path.exists(cls.testout): - os.unlink(cls.testout) - if os.path.isfile(cls.qafile): - os.unlink(cls.qafile) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) os.chdir(cls.origdir) diff --git a/py/desispec/test/test_calibfinder.py b/py/desispec/test/test_calibfinder.py index 334df3710..dbd379fb7 100644 --- a/py/desispec/test/test_calibfinder.py +++ b/py/desispec/test/test_calibfinder.py @@ -35,7 +35,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): #- remove temporary calibration directory - if os.path.isdir(cls.calibdir) : + if cls.calibdir.startswith(tempfile.gettempdir()) and os.path.isdir(cls.calibdir) : shutil.rmtree(cls.calibdir) def tearDown(self): diff --git a/py/desispec/test/test_extract.py b/py/desispec/test/test_extract.py index 1e3eee939..a1baffb11 100644 --- a/py/desispec/test/test_extract.py +++ b/py/desispec/test/test_extract.py @@ -14,6 +14,7 @@ import uuid import os import tempfile +import shutil from glob import glob from importlib import resources @@ -60,10 +61,9 @@ def setUp(self): @classmethod def tearDownClass(cls): - os.chdir(cls.testdir) - for filename in glob('test-*{}*.fits'.format(cls.testhash)): - if os.path.exists(filename): - os.remove(filename) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) os.chdir(cls.origdir) diff --git a/py/desispec/test/test_fiberflat.py b/py/desispec/test/test_fiberflat.py index 5b2164e83..d9e7faf94 100644 --- a/py/desispec/test/test_fiberflat.py +++ b/py/desispec/test/test_fiberflat.py @@ -8,6 +8,7 @@ import copy import os import tempfile +import shutil from uuid import uuid1 import numpy as np @@ -69,6 +70,10 @@ def tearDown(self): @classmethod def tearDownClass(cls): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + os.chdir(cls.origdir) diff --git a/py/desispec/test/test_pixgroup.py b/py/desispec/test/test_pixgroup.py index 273be77a0..0a6ff5a44 100644 --- a/py/desispec/test/test_pixgroup.py +++ b/py/desispec/test/test_pixgroup.py @@ -19,6 +19,10 @@ class TestPixGroup(unittest.TestCase): def setUpClass(cls): cls.testdir = tempfile.mkdtemp() cls.outdir = os.path.join(cls.testdir, 'output') + + cls.origenv = dict() + for key in ['DESI_SPECTRO_REDUX', 'SPECPROD']: + cls.origenv[key] = os.getenv(key) #- will be None if not set os.environ['DESI_SPECTRO_REDUX'] = cls.testdir os.environ['SPECPROD'] = 'grouptest' @@ -136,9 +140,17 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - if os.path.exists(cls.testdir): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): shutil.rmtree(cls.testdir) + #- restore environment + for key, value in cls.origenv.items(): + if value is not None: + os.environ[key] = value + elif key in os.environ: + del os.environ[key] + def setUp(self): os.environ['DESI_SPECTRO_REDUX'] = self.testdir os.environ['SPECPROD'] = 'grouptest' diff --git a/py/desispec/test/test_qlextract.py b/py/desispec/test/test_qlextract.py index bc0007d16..edfe77ff4 100644 --- a/py/desispec/test/test_qlextract.py +++ b/py/desispec/test/test_qlextract.py @@ -13,6 +13,7 @@ import uuid import os import tempfile +import shutil from glob import glob from importlib import resources @@ -56,10 +57,9 @@ def setUp(self): @classmethod def tearDownClass(cls): - os.chdir(cls.testdir) - for filename in glob('test-*{}*.fits'.format(cls.testhash)): - if os.path.exists(filename): - os.remove(filename) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) os.chdir(cls.origdir) diff --git a/py/desispec/test/test_scripts.py b/py/desispec/test/test_scripts.py index 46cde5e13..a3f3b2359 100644 --- a/py/desispec/test/test_scripts.py +++ b/py/desispec/test/test_scripts.py @@ -8,6 +8,7 @@ import os import unittest import tempfile +import shutil from uuid import uuid4 from astropy.table import Table @@ -29,6 +30,10 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) + cls.environ_cache.clear() os.chdir(cls.origdir) diff --git a/py/desispec/test/test_spectra.py b/py/desispec/test/test_spectra.py index 0328daab4..e5f8ed0fb 100644 --- a/py/desispec/test/test_spectra.py +++ b/py/desispec/test/test_spectra.py @@ -74,7 +74,8 @@ def tearDownClass(cls): else: os.environ[e] = cls.origEnv[e] - if os.path.exists(cls.testDir): + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testDir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testDir): shutil.rmtree(cls.testDir) os.chdir(cls.origDir) diff --git a/py/desispec/test/test_util.py b/py/desispec/test/test_util.py index 71ecea9c9..fa1bcb1f5 100644 --- a/py/desispec/test/test_util.py +++ b/py/desispec/test/test_util.py @@ -8,6 +8,7 @@ from uuid import uuid4 import importlib import tempfile +import shutil import numpy as np from astropy.table import Table @@ -295,10 +296,9 @@ def setUp(self): @classmethod def tearDownClass(cls): - os.chdir(cls.testdir) - for filename in [cls.infile, cls.outfile, cls.testfile]: - if os.path.exists(filename): - os.remove(filename) + #- Remove testdir only if it was created by tempfile.mkdtemp + if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir): + shutil.rmtree(cls.testdir) os.chdir(cls.origdir)