Fix unit tests

py/desispec/io/fibermap.py

@@ -1118,7 +1118,8 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None,
             except KeyError:
                 log.debug("No camera {} in this file".format(camera))
                 continue
-            cfinder=CalibFinder([rawheader,camheader])
+
+            cfinder=CalibFinder([rawheader,camheader], fallback_on_dark_not_found=True)
             for key in badfibers_keywords_and_maskbits.keys() :
                 newbadfibers = cfinder.badfibers([key])
                 if newbadfibers.size > 0 :

py/desispec/test/test_binscripts.py

@@ -2,6 +2,7 @@
 import os, sys
 import unittest
 from uuid import uuid4
+import tempfile
 
 import numpy as np
 
@@ -26,6 +27,9 @@ class TestBinScripts(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.nspec = 6
         cls.nwave = 2000  # Needed for QA
         cls.wave = 4000+np.arange(cls.nwave)
@@ -86,6 +90,7 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         """Cleanup in case tests crashed and left files behind"""
+        os.chdir(cls.testdir)
         for filename in [cls.framefile, cls.fiberflatfile, cls.fibermapfile, \
             cls.skyfile, cls.calibfile, cls.stdfile, cls.qa_calib_file,
                          cls.qa_data_file, cls.modelfile, cls.qafig]:
@@ -96,6 +101,12 @@ def tearDownClass(cls):
         else:
             os.environ['PYTHONPATH'] = cls.origPath
 
+        #- back to where we started
+        os.chdir(cls.origdir)
+
+    def setUp(self):
+        os.chdir(self.testdir)
+
     def _write_frame(self, flavor='none', camera='b3', expid=1, night='20160607',gaia_only=False):
         """Write a fake frame"""
         flux = np.ones((self.nspec, self.nwave))

py/desispec/test/test_bootcalib.py

@@ -4,6 +4,7 @@
 
 import unittest
 from uuid import uuid1
+import tempfile
 import os
 import numpy as np
 import glob
@@ -21,15 +22,18 @@ class TestBoot(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.testarc = 'test_arc.fits.gz'
         cls.testflat = 'test_flat.fits.gz'
         cls.testout = 'test_bootcalib_{}.fits'.format(uuid1())
         cls.qafile = 'test-qa-123jkkjiuc4h123h12h3423sadfew.pdf'
         cls.data_unavailable = False
 
         # Grab the data
-        url_arc = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000000.fits.gz'
-        url_flat = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000001.fits.gz'
+        url_arc = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_arc.fits.gz'
+        url_flat = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_flat.fits.gz'
         for url, outfile in [(url_arc, cls.testarc), (url_flat, cls.testflat)]:
             if not os.path.exists(outfile):
                 try:
@@ -50,6 +54,7 @@ def tearDownClass(cls):
         """We deliberately don't clean up the testarc and testflat files,
         since they are useful for offline testing.
         """
+        os.chdir(cls.testdir)
         # if os.path.exists(cls.testarc):
         #     os.unlink(cls.testarc)
         # if os.path.exists(cls.testflat):
@@ -59,6 +64,11 @@ def tearDownClass(cls):
         if os.path.isfile(cls.qafile):
             os.unlink(cls.qafile)
 
+        os.chdir(cls.origdir)
+
+    def setUp(self):
+        os.chdir(self.testdir)
+
     def test_fiber_peaks(self):
         if self.data_unavailable:
             self.skipTest("Failed to download test data.")

py/desispec/test/test_calibfinder.py

@@ -7,40 +7,74 @@
 import os
 import shutil
 from importlib import resources
-
-
+import tempfile
 
 from desispec.calibfinder import CalibFinder
 
+_standard_calib_dirs = ('DESI_SPECTRO_CALIB' in os.environ) and ('DESI_SPECTRO_DARK' in os.environ)
+
 class TestCalibFinder(unittest.TestCase):
     """Test desispec.calibfinder
     """
-    def tearDown(self):
-        pass
-        if os.path.isdir(self.calibdir) :
-            shutil.rmtree(self.calibdir)
-
-    def setUp(self):    
-        #- Create temporary calib directory
-        self.calibdir  = os.path.join(os.environ['HOME'], 'preproc_unit_test')
-        if not os.path.exists(self.calibdir): os.makedirs(self.calibdir)
-        #- Copy test calibration-data.yaml file 
-        specdir=os.path.join(self.calibdir,"spec/sp0")
-        if not os.path.isdir(specdir) :
-            os.makedirs(specdir)
+
+    @classmethod
+    def setUpClass(cls):
+
+        #- Cache original environment
+        cls.origenv = dict()
+        for key in ['DESI_SPECTRO_CALIB', 'DESI_SPECTRO_DARK']:
+            cls.origenv[key] = os.getenv(key, None)
+
+        #- Prepare alternate $DESI_SPECTRO_CALIB for testing
+        cls.calibdir = tempfile.mkdtemp()
+        specdir = os.path.join(cls.calibdir,"spec/sp0")
+        os.makedirs(specdir)
         for c in "brz" :
             shutil.copy(str(resources.files('desispec').joinpath(f'test/data/ql/{c}0.yaml')), os.path.join(specdir,f"{c}0.yaml"))
-        #- Set calibration environment variable    
-        os.environ["DESI_SPECTRO_CALIB"] = self.calibdir
-
+
+    @classmethod
+    def tearDownClass(cls):
+        #- remove temporary calibration directory
+        if os.path.isdir(cls.calibdir) :
+            shutil.rmtree(cls.calibdir)
+
+    def tearDown(self):
+        #- restore original environment after every test;
+        #- some tests use default env; others use alternate $DESI_SPECTRO_CALIB
+        for key, value in self.origenv.items():
+            if value is not None:
+                os.environ[key] = value
+            elif key in os.environ:
+                del os.environ[key]
 
     def test_init(self):
-        """Cleanup test files if they exist.
+        """Test basic initialization using test $DESI_SPECTRO_CALIB
         """
-
+        os.environ["DESI_SPECTRO_CALIB"] = self.calibdir
         pheader={"DATE-OBS":'2018-11-30T12:42:10.442593-05:00',"DOSVER":'SIM'}
         header={"DETECTOR":'SIM',"CAMERA":'b0      ',"FEEVER":'SIM'}
         cfinder = CalibFinder([pheader,header])
         print(cfinder.value("DETECTOR"))
         if cfinder.haskey("BIAS") :
             print(cfinder.findfile("BIAS"))
+
+    @unittest.skipIf(not _standard_calib_dirs, "$DESI_SPECTRO_CALIB or $DESI_SPECTRO_DARK not set")
+    def test_missing_darks(self):
+        """Missing dark files is only fatal if darks are requested
+        """
+
+        #- Commissioning era data from 20200219 expid 51053, for which we don't have
+        #- darks in $DESI_SPECTRO_DARK
+        phdr = {"DATE-OBS":"2020-02-20T08:59:59.104576", "DOSVER":"trunk"}
+        camhdr = {"DETECTOR":"sn22797", "CAMERA":"b0", "FEEVER":"v20160312", "SPECID":4,
+                  "CCDCFG":"default_sta_20190717.cfg",
+                  "CCDTMING":"default_sta_timing_20180905.txt"}
+
+        #- without an entry in DESI_SPECTRO_DARK, even creating the CalibFinder fails
+        with self.assertRaises(OSError):
+            cfinder = CalibFinder([phdr,camhdr])
+
+        #- but fallback option should work
+        cfinder = CalibFinder([phdr,camhdr], fallback_on_dark_not_found=True)
+        darkfile = cfinder.findfile('DARK')
+        self.assertTrue(darkfile is not None)

py/desispec/test/test_extract.py

@@ -13,6 +13,7 @@
 import unittest
 import uuid
 import os
+import tempfile
 from glob import glob
 from importlib import resources
 
@@ -27,6 +28,9 @@ class TestExtract(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.testhash = uuid.uuid4()
         cls.imgfile = 'test-img-{}.fits'.format(cls.testhash)
         cls.outfile = 'test-out-{}.fits'.format(cls.testhash)
@@ -49,16 +53,20 @@ def setUpClass(cls):
         cls.img = img
 
     def setUp(self):
+        os.chdir(self.testdir)
         for filename in (self.outfile, self.outmodel):
             if os.path.exists(filename):
                 os.remove(filename)
 
     @classmethod
     def tearDownClass(cls):
+        os.chdir(cls.testdir)
         for filename in glob('test-*{}*.fits'.format(cls.testhash)):
             if os.path.exists(filename):
                 os.remove(filename)
 
+        os.chdir(cls.origdir)
+
     @unittest.skipIf(nospecter, 'specter not installed; skipping extraction test')
     def test_extract(self):
         template = "desi_extract_spectra -i {} -p {} -w 7500,7600,0.75 -f {} -s 0 -n 5 --bundlesize 5 -o {} -m {}"

py/desispec/test/test_fiberflat.py

@@ -7,6 +7,7 @@
 import unittest
 import copy
 import os
+import tempfile
 from uuid import uuid1
 
 import numpy as np
@@ -44,22 +45,32 @@ def _get_data():
 
 class TestFiberFlat(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
 
     def setUp(self):
+        os.chdir(self.testdir)
         id = uuid1()
         self.testfibermap = 'test_fibermap_{}.fits'.format(id)
         self.testframe = 'test_frame_{}.fits'.format(id)
         self.testflat = 'test_fiberflat_{}.fits'.format(id)
 
-
     def tearDown(self):
+        os.chdir(self.testdir)
         if os.path.isfile(self.testframe):
             os.unlink(self.testframe)
         if os.path.isfile(self.testflat):
             os.unlink(self.testflat)
         if os.path.isfile(self.testfibermap):
             os.unlink(self.testfibermap)
 
+    @classmethod
+    def tearDownClass(cls):
+        os.chdir(cls.origdir)
+
 
     def test_interface(self):
         """

py/desispec/test/test_pixgroup.py

@@ -92,8 +92,8 @@ def setUpClass(cls):
         cls.exptable.write(cls.expfile)
 
         # Setup a dummy SpectraLite for I/O tests
-        cls.fileio = 'test_spectralite.fits'
-        cls.fileiogz = 'test_spectralite.fits.gz'
+        cls.fileio = os.path.join(cls.testdir, 'test_spectralite.fits')
+        cls.fileiogz = os.path.join(cls.testdir, 'test_spectralite.fits.gz')
 
         cls.nwave = 100
         cls.nspec = 5

py/desispec/test/test_qlextract.py

@@ -12,6 +12,7 @@
 import unittest
 import uuid
 import os
+import tempfile
 from glob import glob
 from importlib import resources
 
@@ -26,6 +27,9 @@ class TestExtract(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.testhash = uuid.uuid4()
         cls.imgfile = 'test-img-{}.fits'.format(cls.testhash)
         cls.outfile = 'test-out-{}.fits'.format(cls.testhash)
@@ -45,16 +49,20 @@ def setUpClass(cls):
         desispec.io.write_fibermap(cls.fibermapfile, fibermap)
 
     def setUp(self):
+        os.chdir(self.testdir)
         for filename in (self.outfile, self.outmodel):
             if os.path.exists(filename):
                 os.remove(filename)
 
     @classmethod
     def tearDownClass(cls):
+        os.chdir(cls.testdir)
         for filename in glob('test-*{}*.fits'.format(cls.testhash)):
             if os.path.exists(filename):
                 os.remove(filename)
 
+        os.chdir(cls.origdir)
+
     def test_boxcar(self):
         from desispec.quicklook.qlboxcar import do_boxcar
         from desispec.io import read_xytraceset

py/desispec/test/test_scripts.py

@@ -7,6 +7,7 @@
 
 import os
 import unittest
+import tempfile
 from uuid import uuid4
 from astropy.table import Table
 
@@ -17,6 +18,9 @@ class TestScripts(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         # from os import environ
         # for k in ('DESI_SPECTRO_REDUX', 'SPECPROD'):
         #     if k in environ:
@@ -26,9 +30,10 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         cls.environ_cache.clear()
+        os.chdir(cls.origdir)
 
     def setUp(self):
-        pass
+        os.chdir(self.testdir)
 
     def tearDown(self):
         pass

py/desispec/test/test_spectra.py

@@ -40,7 +40,9 @@ class TestSpectra(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         """Create specprod directory structure"""
+        cls.origDir = os.getcwd()
         cls.testDir = tempfile.mkdtemp()
+        os.chdir(cls.testDir)
         cls.origEnv = {
             "SPECPROD": None,
             "DESI_SPECTRO_REDUX": None,
@@ -75,12 +77,15 @@ def tearDownClass(cls):
         if os.path.exists(cls.testDir):
             shutil.rmtree(cls.testDir)
 
+        os.chdir(cls.origDir)
+
 
     def setUp(self):
         #- catch specific warnings so that we can find and fix
         # warnings.filterwarnings("error", ".*did not parse as fits unit.*")
 
         #- Test data and files to work with
+        os.chdir(self.testDir)
         self.fileio = "test_spectra.fits"
         self.fileappend = "test_spectra_append.fits"
         self.filebuild = "test_spectra_build.fits"