P200 DBSP misc updates

doc/help/run_pypeit.rst

@@ -4,7 +4,7 @@
     usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c]
                       pypeit_file
 
-    ##  PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev336+gdf3013372.d20240827
+    ##  PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev468+g832ee84e4
     ##  
     ##  Available spectrographs include:
     ##   aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2,

doc/releases/1.16.1dev.rst

@@ -62,7 +62,7 @@ Instrument-specific Updates
 
         - Platescale updated from an order-dependent value, to being 0.164
           arcsec/pixel for all orders
-
+- Add new P200/DBSP reid_arxiv template for 1200/7100 with D55 dichroic
 
 Script Changes
 --------------
@@ -100,6 +100,9 @@ Under-the-hood Improvements
 - Introduced :class:`~pypeit.pypeitdata.PypeItDataPaths` to handle all
   interactions with the ``pypeit/data`` directory, which provides a unified
   interface for accessing on-disk and cached files.
+- Updated general raw image reader so that it correctly accounts for
+  spectrographs that read the data and overscan sections directly from the file
+  headers.
 
 Bug Fixes
 ---------

pypeit/core/wavecal/spectrographs/templ_p200_dbsp.py

@@ -41,6 +41,22 @@ def p200_dbsp_red_1200_9400_d55(overwrite=False):  # DBSPr 1200/9400 D55
     templates.build_template([wfile0], slits, lcut, binspec, outroot,
         lowredux=False, ifiles=ifiles, normalize=True, overwrite=overwrite)
 
+# ##############################
+def p200_dbsp_red_1200_7100_d55(overwrite=False):  # DBSPr 1200/7100 D55
+
+    binspec = 1
+    outroot = 'p200_dbsp_red_1200_7100_d55_6680.fits'
+    #
+    ifiles = [0]
+    slits = [55]  # Be careful with the order..
+    lcut = None
+    wfile0 = os.path.join(templates.template_path, 'P200_DBSP',
+        'R1200_7100_D55', 'wvcalib_6680.fits')
+
+    #
+    templates.build_template([wfile0], slits, lcut, binspec, outroot,
+        lowredux=False, ifiles=ifiles, normalize=True, overwrite=overwrite)
+
 # ##############################
 def p200_dbsp_red_600_10000_d55(overwrite=False):  # DBSPr 600/10000 D55
 
@@ -144,4 +160,5 @@ def p200_dbsp_blue_1200_5000_d68(overwrite=False):  # DBSPb 1200/5000 D68
 if __name__ == '__main__':
     # p200_dbsp_red_316_7500_d55(overwrite=True)
     # p200_dbsp_red_600_10000_d55(overwrite=True)
-    p200_dbsp_red_1200_7100_d68(overwrite=True)
+    # p200_dbsp_red_1200_7100_d68(overwrite=True)
+    p200_dbsp_red_1200_7100_d55(overwrite=True)

pypeit/scripts/cache_github_data.py

@@ -130,7 +130,7 @@ def main(args):
             files = np.array(contents[path])[to_download[path]]
             if len(files) == 0:
                 continue
-            data_path = PypeItDataPath(path)
+            data_path = PypeItDataPath(path, remote_host="github")
             # NOTE: I'm using POSIX path here because I'm unsure what will
             # happen on Windows if the file is in a subdirectory.
             root = pathlib.PurePosixPath(f'pypeit/data/{path}')

pypeit/spectrographs/ldt_deveny.py

@@ -566,79 +566,17 @@ def config_specific_par(self, scifile, inp_par=None):
 
     def get_rawimage(self, raw_file, det):
         """
-        Read raw images and generate a few other bits and pieces
-        that are key for image processing.
+        Read raw spectrograph image files and return data and relevant metadata
+        needed for image processing.
 
         For LDT/DeVeny, the LOIS control system automatically adjusts the
-        ``DATASEC`` and ``OSCANSEC`` regions if the CCD is used in a binning other
-        than 1x1.  The :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.get_rawimage`
-        method in the base class assumes these sections are fixed and adjusts
-        them based on the binning -- an incorrect assumption for this instrument.
-
-        This method is a stripped-down version of the base class method and
-        additionally does *NOT* send the binning to :func:`~pypeit.core.parse.sec2slice`.
-
-        Parameters
-        ----------
-        raw_file : :obj:`str`
-            File to read
-        det : :obj:`int`
-            1-indexed detector to read
-
-        Returns
-        -------
-        detector_par : :class:`~pypeit.images.detector_container.DetectorContainer`
-            Detector metadata parameters.
-        raw_img : `numpy.ndarray`_
-            Raw image for this detector.
-        hdu : `astropy.io.fits.HDUList`_
-            Opened fits file
-        exptime : :obj:`float`
-            Exposure time *in seconds*.
-        rawdatasec_img : `numpy.ndarray`_
-            Data (Science) section of the detector as provided by setting the
-            (1-indexed) number of the amplifier used to read each detector
-            pixel. Pixels unassociated with any amplifier are set to 0.
-        oscansec_img : `numpy.ndarray`_
-            Overscan section of the detector as provided by setting the
-            (1-indexed) number of the amplifier used to read each detector
-            pixel. Pixels unassociated with any amplifier are set to 0.
+        ``DATASEC`` and ``OSCANSEC`` regions if the CCD is used in a binning
+        other than 1x1.  This is a simple wrapper for
+        :func:`pypeit.spectrographs.spectrograph.Spectrograph.get_rawimage` that
+        sets ``sec_includes_binning`` to True.  See the base-class function for
+        the detailed descriptions of the input parameters and returned objects.
         """
-        # Open
-        hdu = io.fits_open(raw_file)
-
-        # Grab the DetectorContainer and extract the raw image
-        detector = self.get_detector_par(det, hdu=hdu)
-        raw_img = hdu[detector['dataext']].data.astype(float)
-
-        # Exposure time (used by RawImage) from the header
-        headarr = self.get_headarr(hdu)
-        exptime = self.get_meta_value(headarr, 'exptime')
-
-        for section in ['datasec', 'oscansec']:
-            # Get the data section from Detector
-            image_sections = detector[section]
-
-            # Initialize the image (0 means no amplifier)
-            pix_img = np.zeros(raw_img.shape, dtype=int)
-            for i in range(detector['numamplifiers']):
-
-                if image_sections is not None:
-                    # Convert the (FITS) data section from a string to a slice
-                    # DO NOT send the binning (default: None)
-                    datasec = parse.sec2slice(image_sections[i], one_indexed=True,
-                                              include_end=True, require_dim=2)
-                    # Assign the amplifier
-                    pix_img[datasec] = i+1
-
-            # Finish
-            if section == 'datasec':
-                rawdatasec_img = pix_img.copy()
-            else:
-                oscansec_img = pix_img.copy()
-
-        # Return
-        return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
+        return super().get_rawimage(raw_file, det, sec_includes_binning=True)
 
     def calc_pattern_freq(self, frame, rawdatasec_img, oscansec_img, hdu):
         """

pypeit/spectrographs/p200_dbsp.py

@@ -169,6 +169,20 @@ def check_frame_type(self, ftype, fitstbl, exprng=None):
         msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
         return np.zeros(len(fitstbl), dtype=bool)
 
+    def get_rawimage(self, raw_file, det):
+        """
+        Read raw spectrograph image files and return data and relevant metadata
+        needed for image processing.
+
+        For P200/DBSP, the ``DATASEC`` and ``OSCANSEC`` regions are read
+        directly from the file header and are automatically adjusted to account
+        for the on-chip binning.  This is a simple wrapper for
+        :func:`pypeit.spectrographs.spectrograph.Spectrograph.get_rawimage` that
+        sets ``sec_includes_binning`` to True.  See the base-class function for
+        the detailed descriptions of the input parameters and returned objects.
+        """
+        return super().get_rawimage(raw_file, det, sec_includes_binning=True)
+
 
 class P200DBSPBlueSpectrograph(P200DBSPSpectrograph):
     """
@@ -575,6 +589,9 @@ def config_specific_par(self, scifile, inp_par=None):
                 'D68': {
                     7600: 'p200_dbsp_red_1200_7100_d68.fits',
                     8200: 'p200_dbsp_red_1200_7100_d68.fits'
+                },
+                'D55': {
+                    6680: 'p200_dbsp_red_1200_7100_d55_6680.fits'
                 }
             },
             '1200/9400': {

pypeit/spectrographs/spectrograph.py

@@ -1163,10 +1163,10 @@ def validate_det(self, det):
             msgs.error(f'Provided det must have type tuple or integer, not {type(det)}.')
         return 1, (det,)
 
-    def get_rawimage(self, raw_file, det):
+    def get_rawimage(self, raw_file, det, sec_includes_binning=False):
         """
-        Read raw images and generate a few other bits and pieces that are key
-        for image processing.
+        Read raw spectrograph image files and return data and relevant metadata
+        needed for image processing.
 
         .. warning::
 
@@ -1181,6 +1181,14 @@ def get_rawimage(self, raw_file, det):
             1-indexed detector(s) to read.  An image mosaic is selected using a
             :obj:`tuple` with the detectors in the mosaic, which must be one of
             the allowed mosaics returned by :func:`allowed_mosaics`.
+        sec_includes_binning : :obj:`bool`, optional
+            Some instruments use hard-coded image-section strings to define the
+            data and overscan regions, which are then automatically adjusted by
+            the on-chip binning read from the header.  Others read the data and
+            overscan sections directly from the header.  If these sections
+            *include* the on-chip binning automatically when the image is
+            written, this flag should be set to true so that this reader returns
+            the correct image sections.
 
         Returns
         -------
@@ -1207,7 +1215,8 @@ def get_rawimage(self, raw_file, det):
         """
         # Check extension and then open
         self._check_extensions(raw_file)
-        hdu = io.fits_open(raw_file, ignore_missing_end=True, output_verify = 'ignore', ignore_blank=True)
+        hdu = io.fits_open(raw_file, ignore_missing_end=True, output_verify='ignore',
+                           ignore_blank=True)
 
         # Validate the entered (list of) detector(s)
         nimg, _det = self.validate_det(det)
@@ -1225,15 +1234,26 @@ def get_rawimage(self, raw_file, det):
         # NOTE: This *must* be (converted to) seconds.
         exptime = self.get_meta_value(headarr, 'exptime')
 
-        # Rawdatasec, oscansec images
-        binning = self.get_meta_value(headarr, 'binning')
-        # NOTE: This means that `specaxis` must be the same for all detectors in
-        # a mosaic
-        if detectors[0]['specaxis'] == 1:
-            binning_raw = (',').join(binning.split(',')[::-1])
+        # Binning
+        if sec_includes_binning:
+            # The section in the header includes the binning, so set it to None
+            # here.
+            binning_raw = None
         else:
-            binning_raw = binning
+            binning = self.get_meta_value(headarr, 'binning')
+            # NOTE: This means that `specaxis` must be the same for all detectors in
+            # a mosaic
+            if detectors[0]['specaxis'] == 1:
+                binning_raw = (',').join(binning.split(',')[::-1])
+            else:
+                binning_raw = binning
+
+        # Always assume normal FITS header formatting
+        one_indexed = True
+        include_last = True
+        require_dim = 2
 
+        # Read the image(s)
         raw_img = [None]*nimg
         rawdatasec_img = [None]*nimg
         oscansec_img = [None]*nimg
@@ -1254,28 +1274,18 @@ def get_rawimage(self, raw_file, det):
 
             for section in ['datasec', 'oscansec']:
 
-                # Get the data section
-                # Try using the image sections as header keywords
-                # TODO -- Deal with user windowing of the CCD (e.g. Kast red)
-                #  Code like the following maybe useful
-                #hdr = hdu[detector[det - 1]['dataext']].header
-                #image_sections = [hdr[key] for key in detector[det - 1][section]]
-                # Grab from Detector
+                # Get the data sections from the detector object (see get_detector_par above)
+                # TODO: Add ability to incude user windowing (e.g., Kast Red)
                 image_sections = detectors[i][section]
-                #if not isinstance(image_sections, list):
-                #    image_sections = [image_sections]
-                # Always assume normal FITS header formatting
-                one_indexed = True
-                include_last = True
 
                 # Initialize the image (0 means no amplifier)
                 pix_img = np.zeros(raw_img[i].shape, dtype=int)
                 for j in range(detectors[i]['numamplifiers']):
 
                     if image_sections is not None:  # and image_sections[i] is not None:
-                        # Convert the data section from a string to a slice
+                        # Convert the (FITS) data section from a string to a slice
                         datasec = parse.sec2slice(image_sections[j], one_indexed=one_indexed,
-                                                  include_end=include_last, require_dim=2,
+                                                  include_end=include_last, require_dim=require_dim,
                                                   binning=binning_raw)
                         # Assign the amplifier
                         pix_img[datasec] = j+1