Merge pull request #2319 from desihub/fix_tests

Fix unit tests

py/desispec/io/fibermap.py

@@ -1118,7 +1118,8 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None,
             except KeyError:
                 log.debug("No camera {} in this file".format(camera))
                 continue
-            cfinder=CalibFinder([rawheader,camheader])
+
+            cfinder=CalibFinder([rawheader,camheader], fallback_on_dark_not_found=True)
             for key in badfibers_keywords_and_maskbits.keys() :
                 newbadfibers = cfinder.badfibers([key])
                 if newbadfibers.size > 0 :

py/desispec/test/test_binscripts.py

@@ -2,6 +2,8 @@
 import os, sys
 import unittest
 from uuid import uuid4
+import shutil
+import tempfile
 
 import numpy as np
 
@@ -26,6 +28,9 @@ class TestBinScripts(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.nspec = 6
         cls.nwave = 2000  # Needed for QA
         cls.wave = 4000+np.arange(cls.nwave)
@@ -86,16 +91,22 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         """Cleanup in case tests crashed and left files behind"""
-        for filename in [cls.framefile, cls.fiberflatfile, cls.fibermapfile, \
-            cls.skyfile, cls.calibfile, cls.stdfile, cls.qa_calib_file,
-                         cls.qa_data_file, cls.modelfile, cls.qafig]:
-            if os.path.exists(filename):
-                os.remove(filename)
+
+        #- Remove testdir only if it was created by tempfile.mkdtemp
+        if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir):
+            shutil.rmtree(cls.testdir)
+
         if cls.origPath is None:
             del os.environ['PYTHONPATH']
         else:
             os.environ['PYTHONPATH'] = cls.origPath
 
+        #- back to where we started
+        os.chdir(cls.origdir)
+
+    def setUp(self):
+        os.chdir(self.testdir)
+
     def _write_frame(self, flavor='none', camera='b3', expid=1, night='20160607',gaia_only=False):
         """Write a fake frame"""
         flux = np.ones((self.nspec, self.nwave))

py/desispec/test/test_bootcalib.py

@@ -4,6 +4,8 @@
 
 import unittest
 from uuid import uuid1
+import tempfile
+import shutil
 import os
 import numpy as np
 import glob
@@ -21,15 +23,18 @@ class TestBoot(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.testarc = 'test_arc.fits.gz'
         cls.testflat = 'test_flat.fits.gz'
         cls.testout = 'test_bootcalib_{}.fits'.format(uuid1())
         cls.qafile = 'test-qa-123jkkjiuc4h123h12h3423sadfew.pdf'
         cls.data_unavailable = False
 
         # Grab the data
-        url_arc = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000000.fits.gz'
-        url_flat = 'https://portal.nersc.gov/project/desi/data/spectest/pix-sub_b0-00000001.fits.gz'
+        url_arc = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_arc.fits.gz'
+        url_flat = 'https://data.desi.lbl.gov/public/epo/example_files/spectest/test_flat.fits.gz'
         for url, outfile in [(url_arc, cls.testarc), (url_flat, cls.testflat)]:
             if not os.path.exists(outfile):
                 try:
@@ -47,17 +52,14 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        """We deliberately don't clean up the testarc and testflat files,
-        since they are useful for offline testing.
-        """
-        # if os.path.exists(cls.testarc):
-        #     os.unlink(cls.testarc)
-        # if os.path.exists(cls.testflat):
-        #     os.unlink(cls.testflat)
-        if os.path.exists(cls.testout):
-            os.unlink(cls.testout)
-        if os.path.isfile(cls.qafile):
-            os.unlink(cls.qafile)
+        #- Remove testdir only if it was created by tempfile.mkdtemp
+        if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir):
+            shutil.rmtree(cls.testdir)
+
+        os.chdir(cls.origdir)
+
+    def setUp(self):
+        os.chdir(self.testdir)
 
     def test_fiber_peaks(self):
         if self.data_unavailable:

py/desispec/test/test_calibfinder.py

@@ -7,40 +7,74 @@
 import os
 import shutil
 from importlib import resources
-
-
+import tempfile
 
 from desispec.calibfinder import CalibFinder
 
+_standard_calib_dirs = ('DESI_SPECTRO_CALIB' in os.environ) and ('DESI_SPECTRO_DARK' in os.environ)
+
 class TestCalibFinder(unittest.TestCase):
     """Test desispec.calibfinder
     """
-    def tearDown(self):
-        pass
-        if os.path.isdir(self.calibdir) :
-            shutil.rmtree(self.calibdir)
-
-    def setUp(self):    
-        #- Create temporary calib directory
-        self.calibdir  = os.path.join(os.environ['HOME'], 'preproc_unit_test')
-        if not os.path.exists(self.calibdir): os.makedirs(self.calibdir)
-        #- Copy test calibration-data.yaml file 
-        specdir=os.path.join(self.calibdir,"spec/sp0")
-        if not os.path.isdir(specdir) :
-            os.makedirs(specdir)
+
+    @classmethod
+    def setUpClass(cls):
+
+        #- Cache original environment
+        cls.origenv = dict()
+        for key in ['DESI_SPECTRO_CALIB', 'DESI_SPECTRO_DARK']:
+            cls.origenv[key] = os.getenv(key, None)
+
+        #- Prepare alternate $DESI_SPECTRO_CALIB for testing
+        cls.calibdir = tempfile.mkdtemp()
+        specdir = os.path.join(cls.calibdir,"spec/sp0")
+        os.makedirs(specdir)
         for c in "brz" :
             shutil.copy(str(resources.files('desispec').joinpath(f'test/data/ql/{c}0.yaml')), os.path.join(specdir,f"{c}0.yaml"))
-        #- Set calibration environment variable    
-        os.environ["DESI_SPECTRO_CALIB"] = self.calibdir
-
+
+    @classmethod
+    def tearDownClass(cls):
+        #- remove temporary calibration directory
+        if cls.calibdir.startswith(tempfile.gettempdir()) and os.path.isdir(cls.calibdir) :
+            shutil.rmtree(cls.calibdir)
+
+    def tearDown(self):
+        #- restore original environment after every test;
+        #- some tests use default env; others use alternate $DESI_SPECTRO_CALIB
+        for key, value in self.origenv.items():
+            if value is not None:
+                os.environ[key] = value
+            elif key in os.environ:
+                del os.environ[key]
 
     def test_init(self):
-        """Cleanup test files if they exist.
+        """Test basic initialization using test $DESI_SPECTRO_CALIB
         """
-
+        os.environ["DESI_SPECTRO_CALIB"] = self.calibdir
         pheader={"DATE-OBS":'2018-11-30T12:42:10.442593-05:00',"DOSVER":'SIM'}
         header={"DETECTOR":'SIM',"CAMERA":'b0      ',"FEEVER":'SIM'}
         cfinder = CalibFinder([pheader,header])
         print(cfinder.value("DETECTOR"))
         if cfinder.haskey("BIAS") :
             print(cfinder.findfile("BIAS"))
+
+    @unittest.skipIf(not _standard_calib_dirs, "$DESI_SPECTRO_CALIB or $DESI_SPECTRO_DARK not set")
+    def test_missing_darks(self):
+        """Missing dark files is only fatal if darks are requested
+        """
+
+        #- Commissioning era data from 20200219 expid 51053, for which we don't have
+        #- darks in $DESI_SPECTRO_DARK
+        phdr = {"DATE-OBS":"2020-02-20T08:59:59.104576", "DOSVER":"trunk"}
+        camhdr = {"DETECTOR":"sn22797", "CAMERA":"b0", "FEEVER":"v20160312", "SPECID":4,
+                  "CCDCFG":"default_sta_20190717.cfg",
+                  "CCDTMING":"default_sta_timing_20180905.txt"}
+
+        #- without an entry in DESI_SPECTRO_DARK, even creating the CalibFinder fails
+        with self.assertRaises(OSError):
+            cfinder = CalibFinder([phdr,camhdr])
+
+        #- but fallback option should work
+        cfinder = CalibFinder([phdr,camhdr], fallback_on_dark_not_found=True)
+        darkfile = cfinder.findfile('DARK')
+        self.assertTrue(darkfile is not None)

py/desispec/test/test_extract.py

@@ -13,6 +13,8 @@
 import unittest
 import uuid
 import os
+import tempfile
+import shutil
 from glob import glob
 from importlib import resources
 
@@ -27,6 +29,9 @@ class TestExtract(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.testhash = uuid.uuid4()
         cls.imgfile = 'test-img-{}.fits'.format(cls.testhash)
         cls.outfile = 'test-out-{}.fits'.format(cls.testhash)
@@ -49,15 +54,18 @@ def setUpClass(cls):
         cls.img = img
 
     def setUp(self):
+        os.chdir(self.testdir)
         for filename in (self.outfile, self.outmodel):
             if os.path.exists(filename):
                 os.remove(filename)
 
     @classmethod
     def tearDownClass(cls):
-        for filename in glob('test-*{}*.fits'.format(cls.testhash)):
-            if os.path.exists(filename):
-                os.remove(filename)
+        #- Remove testdir only if it was created by tempfile.mkdtemp
+        if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir):
+            shutil.rmtree(cls.testdir)
+
+        os.chdir(cls.origdir)
 
     @unittest.skipIf(nospecter, 'specter not installed; skipping extraction test')
     def test_extract(self):

py/desispec/test/test_fiberflat.py

@@ -7,6 +7,8 @@
 import unittest
 import copy
 import os
+import tempfile
+import shutil
 from uuid import uuid1
 
 import numpy as np
@@ -44,22 +46,36 @@ def _get_data():
 
 class TestFiberFlat(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
 
     def setUp(self):
+        os.chdir(self.testdir)
         id = uuid1()
         self.testfibermap = 'test_fibermap_{}.fits'.format(id)
         self.testframe = 'test_frame_{}.fits'.format(id)
         self.testflat = 'test_fiberflat_{}.fits'.format(id)
 
-
     def tearDown(self):
+        os.chdir(self.testdir)
         if os.path.isfile(self.testframe):
             os.unlink(self.testframe)
         if os.path.isfile(self.testflat):
             os.unlink(self.testflat)
         if os.path.isfile(self.testfibermap):
             os.unlink(self.testfibermap)
 
+    @classmethod
+    def tearDownClass(cls):
+        #- Remove testdir only if it was created by tempfile.mkdtemp
+        if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir):
+            shutil.rmtree(cls.testdir)
+
+        os.chdir(cls.origdir)
+
 
     def test_interface(self):
         """

py/desispec/test/test_pixgroup.py

@@ -19,6 +19,10 @@ class TestPixGroup(unittest.TestCase):
     def setUpClass(cls):
         cls.testdir = tempfile.mkdtemp()
         cls.outdir = os.path.join(cls.testdir, 'output')
+
+        cls.origenv = dict()
+        for key in ['DESI_SPECTRO_REDUX', 'SPECPROD']:
+            cls.origenv[key] = os.getenv(key)  #- will be None if not set
 
         os.environ['DESI_SPECTRO_REDUX'] = cls.testdir
         os.environ['SPECPROD'] = 'grouptest'
@@ -92,8 +96,8 @@ def setUpClass(cls):
         cls.exptable.write(cls.expfile)
 
         # Setup a dummy SpectraLite for I/O tests
-        cls.fileio = 'test_spectralite.fits'
-        cls.fileiogz = 'test_spectralite.fits.gz'
+        cls.fileio = os.path.join(cls.testdir, 'test_spectralite.fits')
+        cls.fileiogz = os.path.join(cls.testdir, 'test_spectralite.fits.gz')
 
         cls.nwave = 100
         cls.nspec = 5
@@ -136,9 +140,17 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        if os.path.exists(cls.testdir):
+        #- Remove testdir only if it was created by tempfile.mkdtemp
+        if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir):
             shutil.rmtree(cls.testdir)
 
+        #- restore environment
+        for key, value in cls.origenv.items():
+            if value is not None:
+                os.environ[key] = value
+            elif key in os.environ:
+                del os.environ[key]
+
     def setUp(self):
         os.environ['DESI_SPECTRO_REDUX'] = self.testdir
         os.environ['SPECPROD'] = 'grouptest'

py/desispec/test/test_qlextract.py

@@ -12,6 +12,8 @@
 import unittest
 import uuid
 import os
+import tempfile
+import shutil
 from glob import glob
 from importlib import resources
 
@@ -26,6 +28,9 @@ class TestExtract(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        cls.origdir = os.getcwd()
+        cls.testdir = tempfile.mkdtemp()
+        os.chdir(cls.testdir)
         cls.testhash = uuid.uuid4()
         cls.imgfile = 'test-img-{}.fits'.format(cls.testhash)
         cls.outfile = 'test-out-{}.fits'.format(cls.testhash)
@@ -45,15 +50,18 @@ def setUpClass(cls):
         desispec.io.write_fibermap(cls.fibermapfile, fibermap)
 
     def setUp(self):
+        os.chdir(self.testdir)
         for filename in (self.outfile, self.outmodel):
             if os.path.exists(filename):
                 os.remove(filename)
 
     @classmethod
     def tearDownClass(cls):
-        for filename in glob('test-*{}*.fits'.format(cls.testhash)):
-            if os.path.exists(filename):
-                os.remove(filename)
+        #- Remove testdir only if it was created by tempfile.mkdtemp
+        if cls.testdir.startswith(tempfile.gettempdir()) and os.path.exists(cls.testdir):
+            shutil.rmtree(cls.testdir)
+
+        os.chdir(cls.origdir)
 
     def test_boxcar(self):
         from desispec.quicklook.qlboxcar import do_boxcar