Update .tpns, add env scripts for all deployments, update romancal im…

…ports, remove unused code

crds/roman/__init__.py

@@ -4,14 +4,6 @@
 
 HERE  = os.path.abspath(os.path.dirname(__file__) or ".")
 
-#
-# These are added globally to requested parameters for all rmaps,  roughly
-# equivalent to config_manager.pipeline_exceptions.keys()
-#
-# XXX needed for Roman??
-# extra_keys = ["META.VISIT.TSOVISIT", "META.INSTRUMENT.LAMP_STATE"]
-extra_keys = []
-
 TYPES = reftypes.from_package_file("roman", __file__)
 
 OBSERVATORY = TYPES.observatory
@@ -27,7 +19,7 @@
 }
 
 PROVENANCE_KEYWORDS = ("META.DESCRIPTION", "META.PEDIGREE", "META.USEAFTER","HISTORY", "META.AUTHOR", "META.MODEL_TYPE")
-# PROVENANCE_KEYWORDS = ("DESCRIP", "PEDIGREE", "USEAFTER","HISTORY", "AUTHOR")
+
 USEAFTER_KEYWORDS = ("META.OBSERVATION.DATE", "META.OBSERVATION.TIME") # Dataset keywords matching in UseAfter selectors
 
 DEFAULT_SELECTORS = ("Match", "UseAfter")   # Normal selector hierarchy in rmap

crds/roman/locate.py

@@ -7,7 +7,7 @@
 in the certification of reference files.
 
 XXXX Roman NOTE: This code was derived from the JWST locate.py module and
-contains substantial dupliucation.  However, because the functions often depend
+contains substantial duplication.  However, because the functions often depend
 on project-specific modules, globals, or functions the code is not usable
 without some refactoring.  Other plugins may vary simply because
 ASDF+datamodels Roman is already different than the FITS+datamodels world of
@@ -71,16 +71,16 @@ def get_datamodels():
     dependencies when supporting other observatories.
     """
     try:
-        from roman import datamodels
+        from romancal import datamodels
     except ImportError:
         log.error(
-            "CRDS requires installation of the 'roman' package to operate on Roman files.")
+            "CRDS requires installation of the 'romancal' package to operate on Roman files.")
         raise
     global MODEL
     if MODEL is None:
         with log.error_on_exception(
-                "Failed constructing basic Roman DataModel"):
-            MODEL = datamodels.DataModel()
+                "Failed constructing basic RomanDataModel"):
+            MODEL = datamodels.RomanDataModel()
     return datamodels
 
 # =============================================================================
@@ -377,9 +377,6 @@ def reference_keys_to_dataset_keys(rmapping, header):
                          "to value of", repr(rkey), "=", repr(rval))
                 header[dkey] = rval
 
-    #  XXXX  roman  no more FITS
-    # header = abstract.cross_strap_header(header)
-
     if "META.SUBARRAY.NAME" not in header:
         header["META.SUBARRAY.NAME"] = "UNDEFINED"
 
@@ -418,24 +415,9 @@ def get_env_prefix(instrument):
     """Return the environment variable prefix (IRAF prefix) for `instrument`."""
     return "crds://"
 
-# META.REF_FILE.SPECWCS.NAME.FITS_KEYWORD
-
-
-# XXXX Roman TODO   this is a guess;  define the dotted object path needed to
-# set the datamodels field for these.  Then transform it to a "flat form" string.
 def filekind_to_keyword(filekind):
-    """Return the "keyword" at which a assigned reference should be recorded.
-
-    See also the JWST version of this plugin which translated to R_ FITS keywords.
-
-    >>> filekind_to_keyword("flat")
-    'META.REF_FILE.FLAT.NAME'
-    """
-    return f"META.REF_FILE.{filekind.upper()}.NAME"
-
-@utils.cached
-def warn_filekind_once(filekind):
-    log.warning("No apparent Roman cal code data models schema support for", log.srepr(filekind))
+    """Return the "keyword" at which a assigned reference should be recorded."""
+    raise NotImplementedError("filekind_to_keyword not implemented for Roman")
 
 def locate_file(refname, mode=None):
     """Given a valid reffilename in CDBS or CRDS format,  return a cache path for the file.
@@ -493,7 +475,7 @@ def hijack_warnings(func, *args, **keys):
         from astropy.utils.exceptions import AstropyUserWarning
         warnings.simplefilter("always", AstropyUserWarning)
 
-        from romancal.datamodels.validate import ValidationWarning
+        from sdatamodels.validate import ValidationWarning
         warnings.filterwarnings("always", r".*", ValidationWarning, f".*roman.*")
         if not config.ALLOW_SCHEMA_VIOLATIONS:
             warnings.filterwarnings("error", r".*is not one of.*", ValidationWarning, f".*roman.*")

crds/roman/schema.py

@@ -1,21 +1,19 @@
-"""This module defines functions for loading JWST's data model schema files which
-describe reference parameters and their values.   The schema files are used to
+"""This module defines functions for loading Roman's data model schema files which
+describe reference parameters and their values.  The schema files are used to
 validate reference and rmap parameters to screen out illegal values.
 
 The primary functions provided by this module are:
 
-1. Scraping the cal code data models schema for keyword value enumerations
-2. Defining translations to and from "data model paths" like META.SUBARRAY.NAME and
-"physical keyword values, e.g. FITS" like SUBARRAY.
+1. Scraping the romancal data model schema for keyword value enumerations
 
-See the tpn.py and locator.py modules,  as well as crds.certify and crds.rmap,
-and crds.selectors for more information.
+See the locator.py module, as well as crds.certify, crds.rmap, and crds.selectors
+for more information.
 """
 import re
 
 # ====================================================================================
 
-# from jwst import datamodels   # deferred
+# from romancal import datamodels   # deferred
 # from . import locate  # deferred
 
 import crds
@@ -24,25 +22,11 @@
 
 # ====================================================================================
 
-
-INSTR_PREFIX = {
-    "wfi" : "WFI_",
-    }
-
 def get_exptypes(instrument=None):
-    """Using an arbitrary reference from an instrument that matches using EXP_TYPE,  return
-    the set of all EXP_TYPE values defined in the JWST schema.   XXX kludged
+    """Using an arbitrary reference from an instrument that matches using EXP_TYPE, return
+    the set of all EXP_TYPE values defined in the Roman schema.
     """
-    tpninfos = get_schema_tpninfos(config.locate_file("roman_area_0005.asdf", "roman"))
-    values = set()
-    for info in tpninfos:
-        if info.name in ["EXP_TYPE", "META.EXPOSURE.TYPE"]:
-            values = values | set(info.values)
-    if instrument is None:
-        return sorted(list(values))
-    else:
-        return sorted([value for value in values
-                       if value.startswith(INSTR_PREFIX[instrument.lower()])])
+    raise NotImplementedError("get_exptypes is not yet implemented for Roman")
 
 def get_schema_tpninfos(refpath):
     """Load the list of TPN info tuples corresponding to `instrument` and
@@ -59,7 +43,7 @@ def reference_to_schema_name(reference_name):
     """This function will eventually identify the schema associated with `reference_name`
     unless replaced by similar functionality in the models package.
 
-    Returns None  meaning "default/core schema"
+    Returns None meaning "default/core schema"
     """
     return None
 
@@ -87,11 +71,6 @@ def refpath_to_parkeys(refpath):
         p = crds.get_pickled_mapping(context)   # reviewed
         instrument, filekind = locate.get_file_properties(refpath)
         keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys()
-        # The below turned out to be bad because CRDS-only values aren't in datamodels
-        # so in this case the datamodels schema are too narrow for these and will reject
-        # e.g. SYSTEM or it's types.
-        #         keys.append("META.INSTRUMENT.NAME")
-        #         keys.append("META.REFTYPE")
     return sorted(keys)
 
 # =============================================================================
@@ -103,7 +82,7 @@ def _load_schema(schema_name=None):
     """Return the core data model schema."""
     from . import locate
     datamodels = locate.get_datamodels()
-    model = datamodels.DataModel(schema=schema_name)
+    model = datamodels.RomanDataModel(schema=schema_name)
     return model.schema
 
 def _schema_to_flat(schema):
@@ -214,48 +193,6 @@ def _flat_to_tpns(flat=None, schema_name=None):
                 log.warning("No TPN form for", repr(key), repr(value))
     return sorted(tpns)
 
-# XXXX roman  fits_to_dm  not needed,  no more FITS?
-
-# def _get_dm_to_fits(schema=None):
-#     """Return mapping from DM dotted path string to FITS keyword."""
-#     if schema is None:
-#         schema = _load_schema()
-#     fits = {}
-#     flat = _schema_to_flat(schema)
-#     for key, val in flat.items():
-#         if key.lower().endswith(".fits_keyword"):
-#             fits[str(key[:-len(".fits_keyword")])] = str(val)
-#     return fits
-
-# def _get_fits_to_dm(schema=None):
-#     """Return mapping from FITS keyword to DM dotted path string."""
-#     return utils.invert_dict(_get_dm_to_fits(schema))
-
-# DM_TO_FITS = None
-# FITS_TO_DM = None
-
-# def dm_to_fits(key):
-#     """Return the FITS keyword for DM `key` or None.
-
-#     >>> dm_to_fits('META.SUBARRAY.NAME')
-#     'SUBARRAY'
-#     """
-#     global DM_TO_FITS
-#     if DM_TO_FITS is None:
-#         DM_TO_FITS = _get_dm_to_fits()
-#     return DM_TO_FITS.get(key.upper(), None)
-
-# def fits_to_dm(key):
-#     """Return the DM keyword for FITS `key` or None.
-
-#     >>> fits_to_dm('SUBARRAY')
-#     'META.SUBARRAY.NAME'
-#     """
-#     global FITS_TO_DM
-#     if FITS_TO_DM is None:
-#         FITS_TO_DM = _get_fits_to_dm()
-#     return FITS_TO_DM.get(key.upper(), None)
-
 # =============================================================================
 
 def main():

crds/roman/specs/wfi_all.spec

@@ -4,7 +4,7 @@
     'filekind' : 'ALL',
     'filetype' : 'ALL',
     'suffix' : 'ALL',
-    'text_descr' : 'Group for constraints for all types of WFI instrument.',
+    'text_descr' : 'All WFI reference file types.',
     'ld_tpn': 'wfi_all_ld.tpn',
     'tpn': 'wfi_all.tpn',
 }

crds/roman/tpns/all_all.tpn

@@ -11,14 +11,9 @@
 # NAME  KEYTYPE  DATATYPE   PRESENCE    VALUES
 #----------------------------------------------------------
 META.TELESCOPE          H   C   R       ROMAN
-META.INSTRUMENT.NAME   H   C   R
-META.REFTYPE       H   C   R
-META.USEAFTER   H   C   R               &JWSTDATE
-META.PEDIGREE   H   C   R               &JWSTPEDIGREE
-META.AUTHOR     H   C   R
-META.DESCRIPTION H   C  R
-HISTORY     H   C  R
-
-# Added automatically when files are created with the CAL code data models
-# Determines verification performed by CAL code data models so can be critical
-META.MODEL_TYPE  H   C  W
+META.INSTRUMENT.NAME    H   C   R
+META.REFTYPE            H   C   R
+META.USEAFTER           H   C   R       &JWSTDATE
+META.PEDIGREE           H   C   R       &JWSTPEDIGREE
+META.AUTHOR             H   C   R
+META.DESCRIPTION        H   C   R

crds/roman/tpns/wfi_all.tpn

@@ -1,4 +1,4 @@
-# Contraints defined  in this file apply to every REFTYPE of the WFI instrument
+# Contraints defined in this file apply to every REFTYPE of the WFI instrument
 
 # Template file used by certify to check reference files
 # Some fields may be abbreviated to their first character:
@@ -9,12 +9,11 @@
 #
 # NAME  KEYTYPE  DATATYPE   PRESENCE    VALUES
 #----------------------------------------------------------
-META.INSTRUMENT.NAME        H   C   R                   WFI
+META.INSTRUMENT.NAME        H   C   R   WFI
 
-META.INSTRUMENT.PUPIL       H   C   O   CLEAR,ANY,N/A
+META.INSTRUMENT.DETECTOR    H   C   O   WFI01,WFI02,WFI03,WFI04,WFI05,WFI06,WFI07,WFI08,WFI09,\
+                                        WFI10,WFI11,WFI12,WFI13,WFI14,WFI15,WFI16,WFI17,WFI18,\
+                                        ANY,N/A
 
-META.INSTRUMENT.DETECTOR    H   C   O   WFI-1,ANY,N/A
-
-META.INSTRUMENT.FILTER      H   C   O   MULTIPLE,ANY,N/A
-
-# include includes/nir_subarray_baseline.tpn
+META.INSTRUMENT.FILTER      H   C   O   F062,F087,F106,F129,W146,F158,F184,GRISM,PRISM,CLEAR,\
+                                        N/A,ANY,UNKNOWN

envs/roman-crds-dev.csh

@@ -0,0 +1,3 @@
+setenv CRDS_SERVER_URL https://roman-crds-dev.stsci.edu
+setenv CRDS_PATH ${HOME}/crds_cache_dev
+setenv CRDS_READONLY_CACHE 0

envs/roman-crds-dev.sh

@@ -0,0 +1,3 @@
+export CRDS_SERVER_URL="https://roman-crds-dev.stsci.edu"
+export CRDS_PATH="${HOME}/crds_cache_dev"
+export CRDS_READONLY_CACHE=0

envs/roman-crds-int.csh

@@ -0,0 +1,3 @@
+setenv CRDS_SERVER_URL https://roman-crds-int.stsci.edu
+setenv CRDS_PATH ${HOME}/crds_cache_int
+setenv CRDS_READONLY_CACHE 0

envs/roman-crds-int.sh

@@ -0,0 +1,3 @@
+export CRDS_SERVER_URL=https://roman-crds-int.stsci.edu
+export CRDS_PATH=${HOME}/crds_cache_int
+export CRDS_READONLY_CACHE=0

envs/roman-crds-ops.csh

@@ -1,43 +1,3 @@
-# This file defines required environment variables for the CRDS client
-
-# If you don't care too much about CRDS file organization,  use CRDS_PATH
-# to define the root of a single tree of CRDS mapping and reference cache files.
-
-# --------------------------------------------------------------------------------------------------------------------
-# setenv CRDS_PATH /Users/jmiller/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/crds
-# unsetenv CRDS_PATH
-
-# Where reference files are stored,  not necessary except for special cases:
-# setenv CRDS_REFPATH  ${CRDS_PATH}/references
-
-# Where mapping files are stored, not necessary except for special cases:
-# setenv CRDS_MAPPATH ${CRDS_PATH}/mappings
-
-# Where CRDS server status is stored,  not necessary except for special cases: (should be writeable unless serverless)
-# setenv CRDS_CFGPATH ${CRD_PATH}/config
-
-# --------------------------------------------------------------------------------------------------------------------
-# The URL used to locate the CRDS server which distributes files, status, and computes best references
-# setenv CRDS_SERVER_URL  http://hst-crds.stsci.edu
-# setenv CRDS_SERVER_URL  http://jwst-crds.stsci.edu
-# setenv CRDS_SERVER_URL  http://crds-non-networked.foo.bar
-setenv CRDS_SERVER_URL  https://roman-crds.stsci.edu
+setenv CRDS_SERVER_URL https://roman-crds.stsci.edu
 setenv CRDS_PATH ${HOME}/crds_cache_ops
 setenv CRDS_READONLY_CACHE 0
-
-# --------------------------------------------------------------------------------------------------------------------
-# To override server recommendations for the operational pipeline context (.pmap)
-# This override is likewise superceded by the getreferences() function parameter
-# which can be used to implement command line overrides in pipeline software.
-# setenv CRDS_CONTEXT  jwst.pmap
-
-# --------------------------------------------------------------------------------------------------------------------
-# To force where CRDS computes best references,  using local client code,  or server code,
-# set CRDS_MODE to "local", "remote", or "auto"
-#
-#  local  -- CRDS will compute best references using the local installation
-#  remote -- CRDS will ask the server to compute best references
-#  auto   -- CRDS will only compute best references on the server if the client is deemed obsolete
-#            by comparing client and server software versions.
-#
-# setenv CRDS_MODE auto