From f3c8836581a0f50645e6e8490feae20e091434f2 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 12:39:53 -0400
Subject: [PATCH 01/41] correct logging statement

---
 jwql/shared_tasks/shared_tasks.py | 34 +++++++++++++++----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 516738ed0..c2f7626a1 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -231,7 +231,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
     with process.stderr:
         log_subprocess_output(process.stderr)
-    result = process.wait()    
+    result = process.wait()
     logging.info("Subprocess result was {}".format(result))
 
     if not os.path.isfile(res_file):
@@ -241,7 +241,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
             for line in status:
                 logging.error(line.strip())
             return status
-        
+
     with open(res_file, 'r') as inf:
         status = inf.readlines()
     return status
@@ -251,17 +251,17 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
 def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}):
     """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each
     step as a separate output file, then return the name-and-path of the file as reduced
-    in the reduction directory. Once all requested extensions have been produced, the 
+    in the reduction directory. Once all requested extensions have been produced, the
     pipeline will return.
 
     Parameters
     ----------
     input_file_name : str
         File on which to run the pipeline steps
-    
+
     short_name : str
         Name of the file to be calibrated after any extensions have been stripped off.
-    
+
     ext_or_exts : list
         List of extensions to be retrieved.
 
@@ -291,7 +291,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     output_dir = os.path.join(config['transfer_dir'], "outgoing")
     msg = "Input from {}, calibrate in {}, output to {}"
     logging.info(msg.format(input_dir, cal_dir, output_dir))
-    
+
     input_file = os.path.join(input_dir, input_file_name)
     current_dir = os.path.dirname(__file__)
     cmd_name = os.path.join(current_dir, "run_pipeline.py")
@@ -308,9 +308,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         logging.info("Requesting {}".format(calibrated_files))
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -323,7 +323,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -337,7 +337,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -346,7 +346,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                         logging.error("Pipeline subprocess failed.")
         if not managed:
             raise ValueError("Pipeline Failed")
-    
+
     for file in calibrated_files:
         logging.info("Checking for output {}".format(file))
         if not os.path.isfile(os.path.join(cal_dir, file)):
@@ -420,14 +420,14 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
     short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "")
     ensure_dir_exists(cal_dir)
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
-    
+
     cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py")
     result_file = os.path.join(cal_dir, short_name+"_status.txt")
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -454,7 +454,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -549,7 +549,7 @@ def prep_file(input_file, in_ext):
         logging.critical(msg.format(short_name))
         raise ValueError("Redis lock for {} is in an unknown state".format(short_name))
     logging.info("\t\tAcquired Lock.")
-    logging.info("\t\tCopying {} to {}".format(input_file, send_path))
+    logging.info("\t\tCopying {} to {}".format(uncal_file, send_path))
     copy_files([uncal_file], send_path)
     return short_name, cal_lock, os.path.join(send_path, uncal_name)
 

From 90d79e38e7882ae40f3eb3e529d849790ad5ea16 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 15:20:53 -0400
Subject: [PATCH 02/41] update run_save_jump

---
 jwql/shared_tasks/run_pipeline.py | 57 +++++++++++++++----------------
 jwql/shared_tasks/shared_tasks.py | 38 ++++++++++-----------
 2 files changed, 47 insertions(+), 48 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 9a196ecdd..124f51a15 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -43,17 +43,17 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
     status_file_name = short_name + "_status.txt"
     status_file = os.path.join(work_directory, status_file_name)
     uncal_file = os.path.join(work_directory, input_file_basename)
-    
+
     with open(status_file, 'a+') as status_f:
         status_f.write("Running run_pipe\n")
         status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename)))
         status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir)))
         status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file)))
-    
+
     try:
         copy_files([input_file], work_directory)
         set_permissions(uncal_file)
-    
+
         steps = get_pipeline_steps(instrument)
         first_step_to_be_run = True
         for step_name in steps:
@@ -94,7 +94,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                             # If the dither_points entry is not populated, then ignore this change
                             pass
                         model[0].save(output_file)
-                
+
                     done = True
                     for output in outputs:
                         output_name = "{}_{}.fits".format(short_name, output)
@@ -110,7 +110,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
             status_f.write("{}\n".format(e))
             status_f.write("FAILED")
         sys.exit(1)
-    
+
     with open(status_file, "a+") as status_f:
         status_f.write("SUCCEEDED")
     # Done.
@@ -130,7 +130,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     sys.stderr.write("Starting pipeline\n")
     with open(status_file, 'a+') as status_f:
         status_f.write("Starting pipeline\n")
-    
+
     try:
         copy_files([input_file], work_directory)
         set_permissions(uncal_file)
@@ -157,51 +157,50 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         # and use the run() method so that we can set parameters
         # progammatically.
         model = Detector1Pipeline()
+        params = {}
 
         # Always true
         if instrument == 'nircam':
-            model.refpix.odd_even_rows = False
+            params['refpix'] = dict(odd_even_rows=False)
 
         # Default CR rejection threshold is too low
-        model.jump.rejection_threshold = 15
+        #params['jump'] = dict(rejection_threshold=15)
 
         # Turn off IPC step until it is put in the right place
-        model.ipc.skip = True
+        params['ipc'] = dict(skip=True)
 
-        model.jump.save_results = True
-        model.jump.output_dir = work_directory
-        model.jump.maximum_cores = max_cores
-        jump_output = uncal_file.replace('uncal', 'jump')
+        # Set up to save jump step output
+        params['jump']['save_results'] = True
+        params['jump']['output_dir'] = work_directory
+        params['jump']['maximum_cores'] = max_cores
+        jump_output = short_name + '_jump.fits'
 
         # Check to see if the jump version of the requested file is already
         # present
         run_jump = not os.path.isfile(jump_output)
 
         if ramp_fit:
-            model.ramp_fit.save_results = True
-            model.ramp_fit.maximum_cores = max_cores
-            # model.save_results = True
-            model.output_dir = work_directory
-            # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate'))
-            pipe_output = uncal_file.replace('uncal', '0_ramp_fit')
+            params['ramp_fit'] = dict(save_results=True, maximum_cores=max_cores)
+
+            pipe_output = os.path.join(work_directory, short_name + '_0_ramp_fit.fits')
             run_slope = not os.path.isfile(pipe_output)
             if save_fitopt:
-                model.ramp_fit.save_opt = True
-                fitopt_output = uncal_file.replace('uncal', 'fitopt')
+                params['ramp_fit']['save_opt'] = True
+                fitopt_output = os.path.join(work_directory, short_name + '_fitopt.fits')
                 run_fitopt = not os.path.isfile(fitopt_output)
             else:
-                model.ramp_fit.save_opt = False
+                params['ramp_fit']['save_opt'] = False
                 fitopt_output = None
                 run_fitopt = False
         else:
-            model.ramp_fit.skip = True
+            params['ramp_fit']['skip'] = True
             pipe_output = None
             fitopt_output = None
             run_slope = False
             run_fitopt = False
 
         if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
-            model.run(datamodel)
+            model.call(datamodel, output_dir=work_directory)
         else:
             print(("Files with all requested calibration states for {} already present in "
                    "output directory. Skipping pipeline call.".format(uncal_file)))
@@ -211,7 +210,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
             status_f.write("{}\n".format(e))
             status_f.write("FAILED")
         sys.exit(1)
-    
+
     with open(status_file, "a+") as status_f:
         status_f.write("{}\n".format(jump_output))
         status_f.write("{}\n".format(pipe_output))
@@ -258,14 +257,14 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
 
     with open(general_status_file, "a+") as status_file:
         status_file.write("Finished parsing args at {}\n".format(time.ctime()))
-    
+
     input_file = args.input_file
     instrument = args.instrument
     short_name = args.short_name
     working_path = args.working_path
     pipe_type = args.pipe
     outputs = args.outputs
-    
+
     status_file = os.path.join(working_path, short_name+"_status.txt")
     with open(status_file, 'w') as out_file:
         out_file.write("Starting Process\n")
@@ -275,10 +274,10 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument)))
         out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file)))
         out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name)))
-    
+
     if not os.path.isfile(args.input_file):
         raise FileNotFoundError("No input file {}".format(args.input_file))
-    
+
     if pipe_type not in ['jump', 'cal']:
         raise ValueError("Unknown calibration type {}".format(pipe_type))
 
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 516738ed0..c4dceee32 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -231,7 +231,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
     with process.stderr:
         log_subprocess_output(process.stderr)
-    result = process.wait()    
+    result = process.wait()
     logging.info("Subprocess result was {}".format(result))
 
     if not os.path.isfile(res_file):
@@ -241,7 +241,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
             for line in status:
                 logging.error(line.strip())
             return status
-        
+
     with open(res_file, 'r') as inf:
         status = inf.readlines()
     return status
@@ -251,17 +251,17 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
 def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}):
     """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each
     step as a separate output file, then return the name-and-path of the file as reduced
-    in the reduction directory. Once all requested extensions have been produced, the 
+    in the reduction directory. Once all requested extensions have been produced, the
     pipeline will return.
 
     Parameters
     ----------
     input_file_name : str
         File on which to run the pipeline steps
-    
+
     short_name : str
         Name of the file to be calibrated after any extensions have been stripped off.
-    
+
     ext_or_exts : list
         List of extensions to be retrieved.
 
@@ -291,7 +291,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     output_dir = os.path.join(config['transfer_dir'], "outgoing")
     msg = "Input from {}, calibrate in {}, output to {}"
     logging.info(msg.format(input_dir, cal_dir, output_dir))
-    
+
     input_file = os.path.join(input_dir, input_file_name)
     current_dir = os.path.dirname(__file__)
     cmd_name = os.path.join(current_dir, "run_pipeline.py")
@@ -308,9 +308,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         logging.info("Requesting {}".format(calibrated_files))
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -323,7 +323,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -337,7 +337,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -346,7 +346,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                         logging.error("Pipeline subprocess failed.")
         if not managed:
             raise ValueError("Pipeline Failed")
-    
+
     for file in calibrated_files:
         logging.info("Checking for output {}".format(file))
         if not os.path.isfile(os.path.join(cal_dir, file)):
@@ -417,17 +417,17 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
         logging.error("File {} not found!".format(input_file))
         raise FileNotFoundError("{} not found".format(input_file))
 
-    short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "")
+    short_name = input_file_name[0: input_file_name.rfind('_')]
     ensure_dir_exists(cal_dir)
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
-    
+
     cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py")
     result_file = os.path.join(cal_dir, short_name+"_status.txt")
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -454,7 +454,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -528,7 +528,7 @@ def prep_file(input_file, in_ext):
     input_path, input_name = os.path.split(input_file)
     logging.info("\tPath is {}, file is {}".format(input_path, input_name))
 
-    if "uncal" not in in_ext:
+    if "uncal" not in in_ext and "dark" not in in_ext:
         logging.info("\tSwitching from {} to uncal".format(in_ext))
         uncal_name = os.path.basename(input_file).replace(in_ext, "uncal")
         uncal_file = filesystem_path(uncal_name, check_existence=True)
@@ -762,7 +762,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
     file_or_files : str or list-of-str
         Name (or names) of the result file(s), including path(s)
     """
-    logging.info("Pipeline call requestion calibrated extensions {}".format(ext_or_exts))
+    logging.info("Pipeline call requested calibrated extensions {}".format(ext_or_exts))
     for input_file in input_files:
         logging.info("\tCalibrating {}".format(input_file))
 

From 00e9058c6a8189363e43a00b57bafd975b151523 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 15:45:02 -0400
Subject: [PATCH 03/41] Turn off steps when input is dark.fits

---
 jwql/shared_tasks/run_pipeline.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 124f51a15..361501ac5 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -166,9 +166,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         # Default CR rejection threshold is too low
         #params['jump'] = dict(rejection_threshold=15)
 
-        # Turn off IPC step until it is put in the right place
-        params['ipc'] = dict(skip=True)
-
         # Set up to save jump step output
         params['jump']['save_results'] = True
         params['jump']['output_dir'] = work_directory
@@ -199,8 +196,23 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
             run_slope = False
             run_fitopt = False
 
+        # If the input file is dark.fits rather than uncal.fits, then skip
+        # all of the pipeline steps that are run prior to dark subtraction
+        if 'dark.fits' in input_file:
+            if instrument.lower() == 'miri':
+                steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe',
+                                 'lastframe', 'reset', 'linearity', 'rscd']
+            else:
+                steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias',
+                                 'refpix', 'linearity']
+            for step in steps_to_skip:
+                params[step] = dict(skip: True)
+        else:
+            # Turn off IPC step until it is put in the right place
+            params['ipc'] = dict(skip=True)
+
         if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
-            model.call(datamodel, output_dir=work_directory)
+            model.call(datamodel, output_dir=work_directory, steps=params)
         else:
             print(("Files with all requested calibration states for {} already present in "
                    "output directory. Skipping pipeline call.".format(uncal_file)))

From 858b06eeed934d97995c336b68900d632d5c42b5 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 17:30:03 -0400
Subject: [PATCH 04/41] Update run_pipe to optionally skip steps

---
 .../common_monitors/dark_monitor.py           |  8 +++----
 jwql/shared_tasks/run_pipeline.py             | 24 +++++++++++++++----
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 26d433ed1..8750914d5 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -402,11 +402,11 @@ def exclude_existing_badpix(self, badpix, pixel_type):
         new_pixels_y : list
             List of y coordinates of new bad pixels
         """
-        
+
         if len(badpix[0]) == 0:
             logging.warning("\tNo new {} pixels to check.".format(pixel_type))
             return ([], [])
-        
+
         logging.info("\tChecking {} potential new {} pixels".format(len(badpix[0]), pixel_type))
 
         if pixel_type not in ['hot', 'dead', 'noisy']:
@@ -440,7 +440,7 @@ def exclude_existing_badpix(self, badpix, pixel_type):
             if len(np.intersect1d(ind_x[0], ind_y[0])) == 0:
                 new_pixels_x.append(x)
                 new_pixels_y.append(y)
-        
+
         logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type))
 #             pixel = (x, y)
 #             if pixel not in already_found:
@@ -810,7 +810,7 @@ def process(self, file_list):
                 # Add new noisy pixels to the database
                 logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0])))
                 self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time)
-            
+
             logging.info("Creating Mean Slope Image {}".format(slope_image))
             # Create png file of mean slope image. Add bad pixels only for full frame apertures
             self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix,
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 361501ac5..c59126ebe 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -27,7 +27,7 @@
 from jwst.saturation import SaturationStep
 from jwst.superbias import SuperBiasStep
 
-from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, get_pipeline_steps
+from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, completed_pipeline_steps, get_pipeline_steps
 from jwql.utils.logging_functions import configure_logging
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
@@ -55,15 +55,26 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         set_permissions(uncal_file)
 
         steps = get_pipeline_steps(instrument)
+
+        # If the input file is a file other than uncal.fits, then we may only need to run a
+        # subset of steps. Check the completed steps in the input file and set steps such
+        # that anything that is already complete is not re-run.
+        if 'uncal.fits' not in input_file:
+            completed_steps = completed_pipeline_steps(filename)
+            for step in steps:
+                if step in completed_steps:
+                    if completed_steps[step]:
+                        steps[step] = False
+
         first_step_to_be_run = True
         for step_name in steps:
-            sys.stderr.write("Running step {}\n".format(step_name))
-            with open(status_file, 'a+') as status_f:
-                status_f.write("Running step {}\n".format(step_name))
             kwargs = {}
             if step_name in ['jump', 'rate']:
                 kwargs = {'maximum_cores': max_cores}
             if steps[step_name]:
+                sys.stderr.write("Running step {}\n".format(step_name))
+                with open(status_file, 'a+') as status_f:
+                    status_f.write("Running step {}\n".format(step_name))
                 output_file_name = short_name + "_{}.fits".format(step_name)
                 output_file = os.path.join(work_directory, output_file_name)
                 # skip already-done steps
@@ -104,6 +115,11 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                     if done:
                         sys.stderr.write("Done pipeline.\n")
                         break
+            else:
+                sys.stderr.write("Skipping step {}\n".format(step_name))
+                with open(status_file, 'a+') as status_f:
+                    status_f.write("Skipping step {}\n".format(step_name))
+
     except Exception as e:
         with open(status_file, "a+") as status_f:
             status_f.write("EXCEPTION\n")

From e7527f7b62df8605407741356c67bd5f68e63e35 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 9 May 2023 10:33:36 -0400
Subject: [PATCH 05/41] Fix logic for skipping completed steps

---
 jwql/shared_tasks/run_pipeline.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index c59126ebe..82e85fd1a 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -57,15 +57,19 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         steps = get_pipeline_steps(instrument)
 
         # If the input file is a file other than uncal.fits, then we may only need to run a
-        # subset of steps. Check the completed steps in the input file and set steps such
-        # that anything that is already complete is not re-run.
+        # subset of steps. Check the completed steps in the input file. Find the latest step
+        # that has been completed, and skip that plus all prior steps
         if 'uncal.fits' not in input_file:
             completed_steps = completed_pipeline_steps(filename)
             for step in steps:
-                if step in completed_steps:
-                    if completed_steps[step]:
-                        steps[step] = False
+                steps[step] = not completed_steps[step]
 
+        # Special case: if the input file is a dark.fits file, then we want to skip the
+        # dark_current subtraction step
+        if 'dark.fits' in input_file:
+            step['dark_current'] = False
+
+        # Run each specified step
         first_step_to_be_run = True
         for step_name in steps:
             kwargs = {}

From 712894b86bdb1d982e38df78c9f9ae605aacaf59 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 9 May 2023 11:02:46 -0400
Subject: [PATCH 06/41] Add missing steps to step list

---
 jwql/instrument_monitors/pipeline_tools.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py
index e9754bd17..9390802e8 100644
--- a/jwql/instrument_monitors/pipeline_tools.py
+++ b/jwql/instrument_monitors/pipeline_tools.py
@@ -165,12 +165,8 @@ def get_pipeline_steps(instrument):
 
     # Order is important in 'steps' lists below!!
     if instrument == 'MIRI':
-        steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe',
-                 'linearity', 'rscd', 'dark_current', 'refpix', 'persistence', 'jump', 'rate']
-        # No persistence correction for MIRI
-        steps.remove('persistence')
-        # MIRI is limited to one frame per group
-        steps.remove('group_scale')
+        steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe', 'reset',
+                 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate', 'gain_scale']
     else:
         steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', 'refpix', 'linearity',
                  'persistence', 'dark_current', 'jump', 'rate']

From 3cd5cd618526705db3c0ea6eb84221dfbb80498c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 9 May 2023 12:31:09 -0400
Subject: [PATCH 07/41] rateints support

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +-
 jwql/shared_tasks/run_pipeline.py                        | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 8750914d5..36b56a5cb 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -712,7 +712,7 @@ def process(self, file_list):
                 logging.info("\t\tAdding {} to calibration set".format(filename))
                 pipeline_files.append(filename)
 
-        outputs = run_parallel_pipeline(pipeline_files, "dark", "rate", self.instrument)
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 82e85fd1a..aef81fd56 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -60,7 +60,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # subset of steps. Check the completed steps in the input file. Find the latest step
         # that has been completed, and skip that plus all prior steps
         if 'uncal.fits' not in input_file:
-            completed_steps = completed_pipeline_steps(filename)
+            completed_steps = completed_pipeline_steps(input_file)
             for step in steps:
                 steps[step] = not completed_steps[step]
 
@@ -109,6 +109,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                             # If the dither_points entry is not populated, then ignore this change
                             pass
                         model[0].save(output_file)
+                        if 'rateints' in outputs:
+                            model[1].save(output_file.replace('rate', 'rateints'))
 
                     done = True
                     for output in outputs:

From fb2bb90ca20c206a1b23c09d640cdc589c67703d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 10:11:10 -0400
Subject: [PATCH 08/41] trivial commit

---
 jwql/shared_tasks/shared_tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index c4dceee32..d191311cd 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -702,7 +702,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False):
         uncal_name = os.path.basename(uncal_file)
         result = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe)
         logging.info("\t\tStarting with ID {}".format(result.id))
-        processed_path = result.get()
+        #processed_path = result.get()
         logging.info("\t\tPipeline Complete")
         output = retrieve_files(short_name, ext_or_exts, retrieve_dir)
     except Exception as e:
@@ -789,7 +789,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
         for short_name in results:
             try:
                 logging.info("\tWaiting for {} ({})".format(short_name, results[short_name].id))
-                processed_path = results[short_name].get()
+                #processed_path = results[short_name].get()
                 logging.info("\t{} retrieved".format(short_name))
                 outputs[input_file_paths[short_name]] = retrieve_files(short_name, ext_or_exts, output_dirs[short_name])
                 logging.info("\tFiles copied for {}".format(short_name))

From 3e3af74672ece149aab129aeaa3cd3c37a7fd363 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 22 May 2023 16:21:47 -0400
Subject: [PATCH 09/41] update step_args in shared_tasks scripts, to be able to
 skip pipe steps

---
 .../common_monitors/dark_monitor.py           |  6 +-
 jwql/instrument_monitors/pipeline_tools.py    | 23 ++++---
 jwql/shared_tasks/run_pipeline.py             | 46 ++++++++++---
 jwql/shared_tasks/shared_tasks.py             | 65 +++++++++++++++----
 4 files changed, 110 insertions(+), 30 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 36b56a5cb..a04080ed5 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -712,7 +712,11 @@ def process(self, file_list):
                 logging.info("\t\tAdding {} to calibration set".format(filename))
                 pipeline_files.append(filename)
 
-        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument)
+        # Specify that we want to skip the dark current correction step
+        step_args = {'dark_current': {'skip: True'}}
+
+        # Call the pipeline
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):
diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py
index 9390802e8..a5fd4a464 100644
--- a/jwql/instrument_monitors/pipeline_tools.py
+++ b/jwql/instrument_monitors/pipeline_tools.py
@@ -25,6 +25,7 @@
 from jwst.dq_init import DQInitStep
 from jwst.dark_current import DarkCurrentStep
 from jwst.firstframe import FirstFrameStep
+from jwst.gain_scale import GainScaleStep
 from jwst.group_scale import GroupScaleStep
 from jwst.ipc import IPCStep
 from jwst.jump import JumpStep
@@ -34,6 +35,7 @@
 from jwst.pipeline.calwebb_detector1 import Detector1Pipeline
 from jwst.ramp_fitting import RampFitStep
 from jwst.refpix import RefPixStep
+from jwst.reset import ResetStep
 from jwst.rscd import RscdStep
 from jwst.saturation import SaturationStep
 from jwst.superbias import SuperBiasStep
@@ -43,16 +45,17 @@
 
 # Define the fits header keyword that accompanies each step
 PIPE_KEYWORDS = {'S_GRPSCL': 'group_scale', 'S_DQINIT': 'dq_init', 'S_SATURA': 'saturation',
-                 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias',
+                 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias', 'S_RESET': 'reset',
                  'S_PERSIS': 'persistence', 'S_DARK': 'dark_current', 'S_LINEAR': 'linearity',
                  'S_FRSTFR': 'firstframe', 'S_LASTFR': 'lastframe', 'S_RSCD': 'rscd',
-                 'S_JUMP': 'jump', 'S_RAMP': 'rate'}
+                 'S_JUMP': 'jump', 'S_RAMP': 'rate', 'S_GANSCL': 'gain_scale', 'S_IPC': 'ipc'}
 
 PIPELINE_STEP_MAPPING = {'dq_init': DQInitStep, 'dark_current': DarkCurrentStep,
-                         'firstframe': FirstFrameStep, 'group_scale': GroupScaleStep,
-                         'ipc': IPCStep, 'jump': JumpStep, 'lastframe': LastFrameStep,
-                         'linearity': LinearityStep, 'persistence': PersistenceStep,
-                         'rate': RampFitStep, 'refpix': RefPixStep, 'rscd': RscdStep,
+                         'firstframe': FirstFrameStep, 'gain_scale': GainScaleStep,
+                         'group_scale': GroupScaleStep, 'ipc': IPCStep, 'jump': JumpStep,
+                         'lastframe': LastFrameStep, 'linearity': LinearityStep,
+                         'persistence': PersistenceStep, 'rate': RampFitStep,
+                         'refpix': RefPixStep, 'reset': ResetStep, 'rscd': RscdStep,
                          'saturation': SaturationStep, 'superbias': SuperBiasStep}
 
 # Readout patterns that have nframes != a power of 2. These readout patterns
@@ -182,13 +185,13 @@ def get_pipeline_steps(instrument):
     # IPC correction currently not done for any instrument
     steps.remove('ipc')
 
-    # Initialize using PIPE_KEYWORDS so the steps will be in the right order
+    # Initialize using OrderedDict so the steps will be in the right order
     required_steps = OrderedDict({})
     for key in steps:
         required_steps[key] = True
-    for key in PIPE_KEYWORDS.values():
-        if key not in required_steps.keys():
-            required_steps[key] = False
+    #for key in PIPE_KEYWORDS.values():
+    #    if key not in required_steps.keys():
+    #        required_steps[key] = False
 
     return required_steps
 
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index aef81fd56..79fa8d260 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -5,6 +5,7 @@
 from collections import OrderedDict
 from copy import deepcopy
 from glob import glob
+import json
 import os
 import shutil
 import sys
@@ -33,7 +34,7 @@
 from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
 
 
-def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all'):
+def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all', step_args={}):
     """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each
     step as a separate output file, then return the name-and-path of the file as reduced
     in the reduction directory.
@@ -61,20 +62,47 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # that has been completed, and skip that plus all prior steps
         if 'uncal.fits' not in input_file:
             completed_steps = completed_pipeline_steps(input_file)
+
+            # Reverse the boolean value, so that now steps answers the question: "Do we need
+            # to run this step?""
             for step in steps:
                 steps[step] = not completed_steps[step]
 
         # Special case: if the input file is a dark.fits file, then we want to skip the
         # dark_current subtraction step
-        if 'dark.fits' in input_file:
-            step['dark_current'] = False
+        #if 'dark.fits' in input_file:
+        #    steps['dark_current'] = False
+
+        # Make sure we don't run steps out of order. Find the latest step that has been
+        # run, and only run subsequent steps. This protects against cases where some early
+        # step was not run. In that case, we don't want to go back and run it because running
+        # pipeline steps out of order doesn't work.
+        last_run = 'group_scale'  # initialize to the first step
+        for step in steps:
+            if not steps[step]:
+                last_run = deepcopy(step)
+
+        for step in steps:
+            if step == last_run:
+                break
+            if step != last_run:
+                steps[step] = False
+
+        # Set any steps the user specifically asks to skip
+        for step, step_dict in step_args.items():
+            if 'skip' in step_dict:
+                print(f'SKIP the {step}!!')
+                if step_dict['skip']:
+                    steps[step] = False
 
         # Run each specified step
         first_step_to_be_run = True
         for step_name in steps:
             kwargs = {}
+            if step_name in step_args:
+                kwargs = step_args[step_name]
             if step_name in ['jump', 'rate']:
-                kwargs = {'maximum_cores': max_cores}
+                kwargs['maximum_cores'] = max_cores
             if steps[step_name]:
                 sys.stderr.write("Running step {}\n".format(step_name))
                 with open(status_file, 'a+') as status_f:
@@ -138,7 +166,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
     # Done.
 
 
-def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all'):
+def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all', step_args={}):
     """Call ``calwebb_detector1`` on the provided file, running all
     steps up to the ``ramp_fit`` step, and save the result. Optionally
     run the ``ramp_fit`` step and save the resulting slope file as well.
@@ -228,7 +256,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
                 steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias',
                                  'refpix', 'linearity']
             for step in steps_to_skip:
-                params[step] = dict(skip: True)
+                params[step] = dict(skip=True)
         else:
             # Turn off IPC step until it is put in the right place
             params['ipc'] = dict(skip=True)
@@ -269,6 +297,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     out_help = 'Comma-separated list of output extensions (for cal only, otherwise just "all")'
     name_help = 'Input file name with no path or extensions'
     cores_help = 'Maximum cores to use (default "all")'
+    step_args_help = 'Step-specific parameter value nested dictionary'
     parser = argparse.ArgumentParser(description='Run local calibration')
     parser.add_argument('pipe', metavar='PIPE', type=str, help=pipe_help)
     parser.add_argument('outputs', metavar='OUTPUTS', type=str, help=out_help)
@@ -277,6 +306,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     parser.add_argument('input_file', metavar='FILE', type=str, help=file_help)
     parser.add_argument('short_name', metavar='NAME', type=str, help=name_help)
     parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help)
+    parser.add_argument('step_args', metavar='STEP_ARGS', type=json.loads, help=step_args_help)
 
     with open(general_status_file, "a+") as status_file:
         status_file.write("Created argument parser at {}\n".format(time.ctime()))
@@ -319,12 +349,12 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         if pipe_type == 'jump':
             with open(status_file, 'a+') as out_file:
                 out_file.write("Running jump pipeline.\n")
-            run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores)
+            run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores, step_args=args.step_args)
         elif pipe_type == 'cal':
             with open(status_file, 'a+') as out_file:
                 out_file.write("Running cal pipeline.\n")
             outputs = outputs.split(",")
-            run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores)
+            run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores, step_args=args.step_args)
     except Exception as e:
         with open(status_file, 'a+') as out_file:
             out_file.write("Exception when starting pipeline.\n")
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index d191311cd..eef421ecd 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -224,9 +224,42 @@ def collect_after_task(**kwargs):
     gc.collect()
 
 
-def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores):
-    command = "{} {} {} '{}' {} {} {} {}"
-    command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores)
+def convert_step_args_to_string(args_dict):
+    """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
+    to a string so that it can be passed via command line
+
+    Parameters
+    ----------
+    args_dict : dict
+        Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing
+        keyword value pairs for that step.
+
+    Returns
+    -------
+    args_str : str
+        String representation of ``args_dict``
+    """
+    args_str='{'
+
+    for i, step in enumerate(args_dict):
+        args_str += f'"{step}":'
+        args_str += '{'
+        for j, (param, val) in enumerate(args_dict[step].items()):
+            args_str += f'"{param}":"{val}"'
+            if j < len(args_dict[step])-1:
+                args_str += ', '
+        args_str += '}'
+        if i < len(args_dict)-1:
+            args_str += ','
+    return args_str
+
+
+def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args):
+    # Convert step_args dictionary to a string so that it can be passed via command line
+    step_args_str = convert_step_args_to_string(step_args)
+
+    command = "{} {} {} '{}' {} {} {} {} {}"
+    command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
     with process.stderr:
@@ -309,7 +342,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
 
     cores = 'all'
     status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file,
-                            short_name, result_file, cores)
+                            short_name, result_file, cores, step_args)
 
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
@@ -324,7 +357,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         if core_fail:
             cores = "half"
             status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
-                                    input_file, short_name, result_file, cores)
+                                    input_file, short_name, result_file, cores, step_args)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
                 managed = True
@@ -338,7 +371,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                 if core_fail:
                     cores = "none"
                     status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
-                                            input_file, short_name, result_file, cores)
+                                            input_file, short_name, result_file, cores, step_args)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
                         managed = True
@@ -554,7 +587,7 @@ def prep_file(input_file, in_ext):
     return short_name, cal_lock, os.path.join(send_path, uncal_name)
 
 
-def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False):
+def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False, step_args={}):
     """Starts the standard or save_jump pipeline for the provided file.
 
     .. warning::
@@ -591,6 +624,11 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa
     jump_pipe : bool
         Whether the detector1 jump pipeline is being used (e.g. the bad pixel monitor)
 
+    step_args : dict
+        Pipeline step arguments to be passed to the pipeline call. Nested dictionary with keys that
+        are the step names (as seen in pipeline_tools.PIPELINE_STEP_MAPPING). Each value is a
+        dictionary of keyword value pairs that are relevant for that step.
+
     Returns
     -------
     result : celery.result.AsyncResult
@@ -606,9 +644,9 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa
                 ramp_fit = True
             elif "fitopt" in ext:
                 save_fitopt = True
-        result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt)
+        result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt, step_args=step_args)
     else:
-        result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument)
+        result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument, step_args=step_args)
     return result
 
 
@@ -716,7 +754,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False):
     return output
 
 
-def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pipe=False):
+def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pipe=False, step_args={}):
     """Convenience function for using the ``run_calwebb_detector1`` function on a list of
     data files, breaking them into parallel celery calls, collecting the results together,
     and returning the results as another list. In particular, this function will do the
@@ -757,6 +795,11 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
     jump_pipe : bool
         Whether the detector1 jump pipeline is being used (e.g. the bad pixel monitor)
 
+    step_args : dict
+        Pipeline step arguments to be passed to the pipeline call. Nested dictionary with keys that
+        are the step names (as seen in pipeline_tools.PIPELINE_STEP_MAPPING). Each value is a
+        dictionary of keyword value pairs that are relevant for that step.
+
     Returns
     -------
     file_or_files : str or list-of-str
@@ -782,7 +825,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
             output_dirs[short_name] = retrieve_dir
             input_file_paths[short_name] = input_file
             locks[short_name] = cal_lock
-            results[short_name] = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe)
+            results[short_name] = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe, step_args=step_args)
             logging.info("\tStarting {} with ID {}".format(short_name, results[short_name].id))
         logging.info("Celery tasks submitted.")
         logging.info("Waiting for task results")

From 458c789fd130249a2accfcac76779bddff540394 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 22 May 2023 16:50:51 -0400
Subject: [PATCH 10/41] Propogate user-input pipeline keywords.

---
 jwql/shared_tasks/run_pipeline.py | 13 ++++++++++++-
 jwql/shared_tasks/shared_tasks.py |  9 +++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 79fa8d260..ba321eafa 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -256,11 +256,22 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
                 steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias',
                                  'refpix', 'linearity']
             for step in steps_to_skip:
-                params[step] = dict(skip=True)
+                step_dict = dict(skip=True)
+                if step in params:
+                    params[step] = params[step].update(step_dict)
+                else:
+                    params[step] = dict(skip=True)
         else:
             # Turn off IPC step until it is put in the right place
             params['ipc'] = dict(skip=True)
 
+        # Include any user-specified parameters
+        for step_name in step_args:
+            if step_name in params:
+                params[step_name] = params[step_name].update(step_args[step_name])
+            else:
+                params[step_name] = step_args[step_name]
+
         if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
             model.call(datamodel, output_dir=work_directory, steps=params)
         else:
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index eef421ecd..450405058 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -332,8 +332,13 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     result_file = os.path.join(cal_dir, short_name+"_status.txt")
     if "all" in ext_or_exts:
         logging.info("All outputs requested")
-        out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity",
-                    "persistence", "dark_current", "jump", "rate"]
+        if instrument.lower() != 'miri':
+            out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity",
+                        "persistence", "dark_current", "jump", "rate"]
+        else:
+            out_exts = ["group_scale", "dq_init", "saturation", "firstframe", "lastframe", "reset",
+                        "linearity", "rscd", "dark_current", "refpix", "jump", "rate", "gain_scale"]
+
         calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in out_exts]
         logging.info("Requesting {}".format(calibrated_files))
     else:

From 2acadedace5bcf859a860e3caae9c3c614f868f8 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 23 May 2023 11:35:32 -0400
Subject: [PATCH 11/41] Make step_args optional in run_pipeline.py

---
 jwql/shared_tasks/run_pipeline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index ba321eafa..6f4250349 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -317,7 +317,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     parser.add_argument('input_file', metavar='FILE', type=str, help=file_help)
     parser.add_argument('short_name', metavar='NAME', type=str, help=name_help)
     parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help)
-    parser.add_argument('step_args', metavar='STEP_ARGS', type=json.loads, help=step_args_help)
+    parser.add_argument('--step_args', metavar='STEP_ARGS', type=json.loads, default='{}', help=step_args_help)
 
     with open(general_status_file, "a+") as status_file:
         status_file.write("Created argument parser at {}\n".format(time.ctime()))

From 8b3ac9ef08e2c12aa17e48c9fc21fb2ecb696c3d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 23 May 2023 12:00:10 -0400
Subject: [PATCH 12/41] Fix pipeline step test

---
 jwql/tests/test_pipeline_tools.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/jwql/tests/test_pipeline_tools.py b/jwql/tests/test_pipeline_tools.py
index 841cdb6d4..a79d687c8 100644
--- a/jwql/tests/test_pipeline_tools.py
+++ b/jwql/tests/test_pipeline_tools.py
@@ -107,9 +107,10 @@ def test_get_pipeline_steps():
 
     # MIRI
     miri_req_steps = pipeline_tools.get_pipeline_steps('miri')
-    miri_steps = ['dq_init', 'saturation', 'firstframe', 'lastframe',
-                  'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate']
-    not_required = ['group_scale', 'ipc', 'superbias', 'persistence']
+    miri_steps = ['group_scale', 'dq_init', 'saturation', 'firstframe', 'lastframe',
+                  'reset', 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate',
+                  'gain_scale']
+    not_required = ['ipc', 'superbias', 'persistence']
     miri_dict = OrderedDict({})
     for step in miri_steps:
         miri_dict[step] = True

From ac4dde8866e5dbe42e8ffaf8ccd792976dcf8227 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 23 May 2023 12:15:03 -0400
Subject: [PATCH 13/41] Clean up

---
 jwql/instrument_monitors/pipeline_tools.py | 3 ---
 jwql/shared_tasks/run_pipeline.py          | 5 -----
 jwql/shared_tasks/shared_tasks.py          | 2 +-
 3 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py
index a5fd4a464..c39e3bda4 100644
--- a/jwql/instrument_monitors/pipeline_tools.py
+++ b/jwql/instrument_monitors/pipeline_tools.py
@@ -189,9 +189,6 @@ def get_pipeline_steps(instrument):
     required_steps = OrderedDict({})
     for key in steps:
         required_steps[key] = True
-    #for key in PIPE_KEYWORDS.values():
-    #    if key not in required_steps.keys():
-    #        required_steps[key] = False
 
     return required_steps
 
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 6f4250349..2e34f58e3 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -68,11 +68,6 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
             for step in steps:
                 steps[step] = not completed_steps[step]
 
-        # Special case: if the input file is a dark.fits file, then we want to skip the
-        # dark_current subtraction step
-        #if 'dark.fits' in input_file:
-        #    steps['dark_current'] = False
-
         # Make sure we don't run steps out of order. Find the latest step that has been
         # run, and only run subsequent steps. This protects against cases where some early
         # step was not run. In that case, we don't want to go back and run it because running
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 450405058..047084d62 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -258,7 +258,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     # Convert step_args dictionary to a string so that it can be passed via command line
     step_args_str = convert_step_args_to_string(step_args)
 
-    command = "{} {} {} '{}' {} {} {} {} {}"
+    command = "{} {} {} '{}' {} {} {} {} --step_args {}"
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)

From 153727854c50bb1e05753198d38df543d88f982c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 3 Jul 2023 17:22:38 -0400
Subject: [PATCH 14/41] Tweaks from testing on server

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +-
 jwql/shared_tasks/run_pipeline.py                        | 2 ++
 jwql/shared_tasks/shared_tasks.py                        | 7 ++++---
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index b8399b1d9..929d9d099 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -712,7 +712,7 @@ def process(self, file_list):
                 pipeline_files.append(filename)
 
         # Specify that we want to skip the dark current correction step
-        step_args = {'dark_current': {'skip: True'}}
+        step_args = {'dark_current': {'skip': True}}
 
         # Call the pipeline
         outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args)
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 2e34f58e3..2ab107318 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -334,6 +334,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     working_path = args.working_path
     pipe_type = args.pipe
     outputs = args.outputs
+    step_args = args.step_args
 
     status_file = os.path.join(working_path, short_name+"_status.txt")
     with open(status_file, 'w') as out_file:
@@ -344,6 +345,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument)))
         out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file)))
         out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name)))
+        out_file.write("\tstep_args is {} ({})\n".format(step_args, type(step_args)))
 
     if not os.path.isfile(args.input_file):
         raise FileNotFoundError("No input file {}".format(args.input_file))
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 047084d62..42ee9021b 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -239,7 +239,7 @@ def convert_step_args_to_string(args_dict):
     args_str : str
         String representation of ``args_dict``
     """
-    args_str='{'
+    args_str="'{"
 
     for i, step in enumerate(args_dict):
         args_str += f'"{step}":'
@@ -248,9 +248,10 @@ def convert_step_args_to_string(args_dict):
             args_str += f'"{param}":"{val}"'
             if j < len(args_dict[step])-1:
                 args_str += ', '
-        args_str += '}'
+        args_str += "}"
         if i < len(args_dict)-1:
             args_str += ','
+    args_str += "}'"
     return args_str
 
 
@@ -262,7 +263,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
-    with process.stderr:
+    with process.stderr:<log_subprocess
         log_subprocess_output(process.stderr)
     result = process.wait()
     logging.info("Subprocess result was {}".format(result))

From 13dcb142b0c71d0136fe03b27f315b0057b9b4dc Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 12 Jul 2023 10:05:27 -0400
Subject: [PATCH 15/41] prepare models for external postgres db

---
 jwql/example_config.json           | 8 ++++++++
 jwql/website/jwql_proj/settings.py | 5 +----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/jwql/example_config.json b/jwql/example_config.json
index 2f9c3ad8c..112b13755 100644
--- a/jwql/example_config.json
+++ b/jwql/example_config.json
@@ -10,6 +10,14 @@
         "host" : "",
         "port" : ""
     },
+    "django_database" : {
+        "engine" : "",
+        "name" : "",
+        "user" : "",
+        "password" : "",
+        "host" : "",
+        "port" : ""
+    },
     "jwql_dir" : "",
     "jwql_version": "",
     "server_type": "",
diff --git a/jwql/website/jwql_proj/settings.py b/jwql/website/jwql_proj/settings.py
index 676d164d9..6dc070ca7 100644
--- a/jwql/website/jwql_proj/settings.py
+++ b/jwql/website/jwql_proj/settings.py
@@ -108,10 +108,7 @@
 # Database
 # https://docs.djangoproject.com/en/2.0/ref/settings/#databases
 DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    },
+    'default': get_config()['django_database']
 }
 
 # Password validation

From 06458e6968c30a8be05b098724f00d6b95f87f89 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 12 Jul 2023 11:56:52 -0400
Subject: [PATCH 16/41] db keys must be all caps

---
 jwql/example_config.json | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jwql/example_config.json b/jwql/example_config.json
index 112b13755..937a1d63b 100644
--- a/jwql/example_config.json
+++ b/jwql/example_config.json
@@ -11,12 +11,12 @@
         "port" : ""
     },
     "django_database" : {
-        "engine" : "",
-        "name" : "",
-        "user" : "",
-        "password" : "",
-        "host" : "",
-        "port" : ""
+        "ENGINE" : "",
+        "NAME" : "",
+        "USER" : "",
+        "PASSWORD" : "",
+        "HOST" : "",
+        "PORT" : ""
     },
     "jwql_dir" : "",
     "jwql_version": "",

From 9f97ca9a799b6db2fe4f8532b0d56c176596d87b Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 13 Jul 2023 17:39:46 -0400
Subject: [PATCH 17/41] update tab_service search for all intents

---
 jwql/website/apps/jwql/data_containers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index c3828b11a..1b6cdcae7 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1248,9 +1248,9 @@ def get_instrument_proposals(instrument):
         List of proposals for the given instrument
     """
     tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx")
-    tap_results = tap_service.search(f"select distinct proposal_id from dbo.ObsPointing where obs_collection='JWST' and calib_level>0 and instrument_name like '{instrument.lower()}%'")
+    tap_results = tap_service.search(f"select distinct prpID from CaomObservation where collection='JWST' and maxLevel>0 and insName like '{instrument.lower()}%'")
     prop_table = tap_results.to_table()
-    proposals = prop_table['proposal_id'].data
+    proposals = prop_table['prpID'].data
     inst_proposals = sorted(proposals.compressed(), reverse=True)
     return inst_proposals
 

From 02a6b63ef686357b7920ad8603ddd15b828080a4 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 12:47:08 -0400
Subject: [PATCH 18/41] rateints case working

---
 jwql/shared_tasks/run_pipeline.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 2ab107318..7e743bbf8 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -50,6 +50,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename)))
         status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir)))
         status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file)))
+        status_f.write(f"\t outputs is {outputs}\n")
 
     try:
         copy_files([input_file], work_directory)
@@ -133,8 +134,12 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                             pass
                         model[0].save(output_file)
                         if 'rateints' in outputs:
-                            model[1].save(output_file.replace('rate', 'rateints'))
-
+                            outbase = os.path.basename(output_file)
+                            outbase = outbase.replace('rate', 'rateints')
+                            output_file = os.path.join(work_directory, outbase)
+                            model[1].save(output_file)
+                            with open(status_file, 'a+') as status_f:
+                                status_f.write(f"Saved rateints model to {output_file}\n")
                     done = True
                     for output in outputs:
                         output_name = "{}_{}.fits".format(short_name, output)

From 9f5f05d1fc8271faaf36554d21aff59dbbcacabd Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 12:54:01 -0400
Subject: [PATCH 19/41] Uncomment the get() lines

---
 jwql/shared_tasks/shared_tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 42ee9021b..244d1a3e8 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -746,7 +746,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False):
         uncal_name = os.path.basename(uncal_file)
         result = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe)
         logging.info("\t\tStarting with ID {}".format(result.id))
-        #processed_path = result.get()
+        processed_path = result.get()
         logging.info("\t\tPipeline Complete")
         output = retrieve_files(short_name, ext_or_exts, retrieve_dir)
     except Exception as e:
@@ -838,7 +838,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
         for short_name in results:
             try:
                 logging.info("\tWaiting for {} ({})".format(short_name, results[short_name].id))
-                #processed_path = results[short_name].get()
+                processed_path = results[short_name].get()
                 logging.info("\t{} retrieved".format(short_name))
                 outputs[input_file_paths[short_name]] = retrieve_files(short_name, ext_or_exts, output_dirs[short_name])
                 logging.info("\tFiles copied for {}".format(short_name))

From 21e0d3eb5ad810fccf8a930a4ee5c67f239dedf3 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 15:07:18 -0400
Subject: [PATCH 20/41] Remove bad cut and paste text

---
 jwql/shared_tasks/shared_tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index e83dd66ce..b35410291 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -263,7 +263,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
-    with process.stderr:<log_subprocess
+    with process.stderr:
         log_subprocess_output(process.stderr)
     result = process.wait()
     logging.info("Subprocess result was {}".format(result))

From 6330d2bb6ab845826a6b83bc3dede0115c85d343 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 15:20:48 -0400
Subject: [PATCH 21/41] Return dark monitor to specifying rate outputs.
 rateints will be another PR

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 929d9d099..8ffab9146 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -715,7 +715,7 @@ def process(self, file_list):
         step_args = {'dark_current': {'skip': True}}
 
         # Call the pipeline
-        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args)
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate"], self.instrument, step_args=step_args)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):

From 95de7d98f945b681d951897e52d74d0298275e1a Mon Sep 17 00:00:00 2001
From: Bradley Sappington
 <101193271+BradleySappington@users.noreply.github.com>
Date: Tue, 25 Jul 2023 16:39:30 -0400
Subject: [PATCH 22/41] Update README.md

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 5909b8889..70b4e173b 100644
--- a/README.md
+++ b/README.md
@@ -154,10 +154,11 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Mees Fix (Technical Lead, INS) [@mfixstsci](https://github.com/mfixstsci)
 - Misty Cracraft (INS) [@cracraft](https://github.com/cracraft)
 - Mike Engesser (INS) [@mengesser](https://github.com/mengesser)
-- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne)
 - Maria Pena-Guerrero [@penaguerrero](https://github.com/penaguerrero)
 - Ben Sunnquist (INS) [@bsunnquist](https://github.com/bsunnquist)
 - Brian York (INS) [@york-stsci](https://github.com/york-stsci)
+- Bradley Sappington (INS) [@bradleysappington](https://github.com/bradleysappington)
+- Melanie Clarke (INS) [@melanieclarke](https://github.com/melanieclarke)
 
 ## Past Development Team Members
 - Matthew Bourque (INS) [@bourque](https://github.com/bourque)
@@ -168,7 +169,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Sara Ogaz (DMD) [@SaOgaz](https://github.com/SaOgaz)
 - Catherine Martlin (INS) [@catherine-martlin](https://github.com/catherine-martlin)
 - Johannes Sahlmann (INS) [@Johannes-Sahlmann](https://github.com/johannes-sahlmann)
-
+- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne)
 
 ## Acknowledgments:
 - Faith Abney (DMD)

From 4adc5853acfc0398b3035f8fbbdcafd048afd480 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 27 Jul 2023 15:28:36 -0400
Subject: [PATCH 23/41] use json loads rather than custom function. resp to
 review comments

---
 jwql/shared_tasks/run_pipeline.py |  3 --
 jwql/shared_tasks/shared_tasks.py | 54 +++++--------------------------
 2 files changed, 8 insertions(+), 49 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 7e743bbf8..d9af19086 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -213,9 +213,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         if instrument == 'nircam':
             params['refpix'] = dict(odd_even_rows=False)
 
-        # Default CR rejection threshold is too low
-        #params['jump'] = dict(rejection_threshold=15)
-
         # Set up to save jump step output
         params['jump']['save_results'] = True
         params['jump']['output_dir'] = work_directory
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index b35410291..b7164b042 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -75,6 +75,7 @@ def some_function(some_arguments):
 from copy import deepcopy
 import gc
 from glob import glob
+import json
 import logging
 from logging import FileHandler, StreamHandler
 import os
@@ -224,40 +225,9 @@ def collect_after_task(**kwargs):
     gc.collect()
 
 
-def convert_step_args_to_string(args_dict):
-    """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
-    to a string so that it can be passed via command line
-
-    Parameters
-    ----------
-    args_dict : dict
-        Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing
-        keyword value pairs for that step.
-
-    Returns
-    -------
-    args_str : str
-        String representation of ``args_dict``
-    """
-    args_str="'{"
-
-    for i, step in enumerate(args_dict):
-        args_str += f'"{step}":'
-        args_str += '{'
-        for j, (param, val) in enumerate(args_dict[step].items()):
-            args_str += f'"{param}":"{val}"'
-            if j < len(args_dict[step])-1:
-                args_str += ', '
-        args_str += "}"
-        if i < len(args_dict)-1:
-            args_str += ','
-    args_str += "}'"
-    return args_str
-
-
 def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args):
     # Convert step_args dictionary to a string so that it can be passed via command line
-    step_args_str = convert_step_args_to_string(step_args)
+    step_args_str = json.loads(step_args)
 
     command = "{} {} {} '{}' {} {} {} {} --step_args {}"
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
@@ -557,7 +527,7 @@ def prep_file(input_file, in_ext):
     short_name : str
         The exposure ID with the calibration tag and the fits extension chopped off.
 
-    uncal_file : str
+    input_name : str
         The raw file to be calibrated
     """
     config = get_config()
@@ -569,16 +539,8 @@ def prep_file(input_file, in_ext):
     input_path, input_name = os.path.split(input_file)
     logging.info("\tPath is {}, file is {}".format(input_path, input_name))
 
-    if "uncal" not in in_ext and "dark" not in in_ext:
-        logging.info("\tSwitching from {} to uncal".format(in_ext))
-        uncal_name = os.path.basename(input_file).replace(in_ext, "uncal")
-        uncal_file = filesystem_path(uncal_name, check_existence=True)
-    else:
-        uncal_file = input_file
-        uncal_name = input_name
-
-    if not os.path.isfile(uncal_file):
-        raise FileNotFoundError("Input File {} does not exist.".format(uncal_file))
+    if not os.path.isfile(input_file):
+        raise FileNotFoundError("Input File {} does not exist.".format(input_file))
 
     output_file_or_files = []
     short_name = input_name.replace("_" + in_ext, "").replace(".fits", "")
@@ -590,9 +552,9 @@ def prep_file(input_file, in_ext):
         logging.critical(msg.format(short_name))
         raise ValueError("Redis lock for {} is in an unknown state".format(short_name))
     logging.info("\t\tAcquired Lock.")
-    logging.info("\t\tCopying {} to {}".format(uncal_file, send_path))
-    copy_files([uncal_file], send_path)
-    return short_name, cal_lock, os.path.join(send_path, uncal_name)
+    logging.info("\t\tCopying {} to {}".format(input_file, send_path))
+    copy_files([input_file], send_path)
+    return short_name, cal_lock, os.path.join(send_path, input_name)
 
 
 def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False, step_args={}):

From 5fdcec1fee1a96732786f3e1aef1fe41b1b7d8ab Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 28 Jul 2023 12:33:20 -0400
Subject: [PATCH 24/41] Returning to custom function for dictioanry to string
 conversion

---
 jwql/shared_tasks/run_pipeline.py |  1 -
 jwql/shared_tasks/shared_tasks.py | 35 +++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index d9af19086..74754afd0 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -87,7 +87,6 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # Set any steps the user specifically asks to skip
         for step, step_dict in step_args.items():
             if 'skip' in step_dict:
-                print(f'SKIP the {step}!!')
                 if step_dict['skip']:
                     steps[step] = False
 
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index b7164b042..7a05f6f92 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -224,10 +224,41 @@ def after_setup_celery_logger(logger, **kwargs):
 def collect_after_task(**kwargs):
     gc.collect()
 
+def convert_step_args_to_string(args_dict):
+    """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
+    to a string so that it can be passed via command line
+
+    Parameters
+    ----------
+    args_dict : dict
+        Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing
+        keyword value pairs for that step.
+
+    Returns
+    -------
+    args_str : str
+        String representation of ``args_dict``
+    """
+    args_str="'{"
+
+    for i, step in enumerate(args_dict):
+        args_str += f'"{step}":'
+        args_str += '{'
+        for j, (param, val) in enumerate(args_dict[step].items()):
+            args_str += f'"{param}":"{val}"'
+            if j < len(args_dict[step])-1:
+                args_str += ', '
+        args_str += "}"
+        if i < len(args_dict)-1:
+            args_str += ','
+    args_str += "}'"
+    return args_str
+
 
 def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args):
-    # Convert step_args dictionary to a string so that it can be passed via command line
-    step_args_str = json.loads(step_args)
+    # Convert step_args dictionary to a string so that it can be passed via command line.
+    # For some reason, json.dumps() doesn't seem to work correctly, so we use a custom function.
+    step_args_str = convert_step_args_to_string(step_args)
 
     command = "{} {} {} '{}' {} {} {} {} --step_args {}"
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)

From 7b1fe988bd52cbfbd8ba4052e58b3efd02831da1 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 28 Jul 2023 13:54:06 -0400
Subject: [PATCH 25/41] Return jump threshold statement.

---
 jwql/shared_tasks/run_pipeline.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 74754afd0..a79167e0b 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -212,6 +212,9 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         if instrument == 'nircam':
             params['refpix'] = dict(odd_even_rows=False)
 
+        # Default CR rejection threshold is too low
+        params['jump']['rejection_threshold'] = 15
+
         # Set up to save jump step output
         params['jump']['save_results'] = True
         params['jump']['output_dir'] = work_directory

From dd136258b7436e32685e2c09874ed87453b014af Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 28 Jul 2023 14:48:26 -0400
Subject: [PATCH 26/41] update anomaly search backend

---
 jwql/website/apps/jwql/data_containers.py | 42 +++--------------------
 1 file changed, 5 insertions(+), 37 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 1b6cdcae7..f4ecb6aa7 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1498,8 +1498,7 @@ def get_rootnames_from_query(parameters):
     filtered_rootnames : list
         A list of all root filenames filtered from the given parameters
     """
-    # TODO - This code setup is temporary until the merge to Postgres is complete.
-    # selected_rootfileinfos = RootFileInfo.objects.none()
+
     filtered_rootnames = []
     # Each Query Selection is Instrument specific
     for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
@@ -1544,23 +1543,13 @@ def get_rootnames_from_query(parameters):
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(read_patt__in=inst_read_patt)
         if (inst_subarray != []):
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(subarray__in=inst_subarray)
-
-        # TODO - This uncommented CODE is what we will use while DJANGO is still using the SQLITE3 DB.
-        #        ONCE DB IS MIGRATED TO POSTGRES WE CAN REPLACE THIS CODE WITH THE UNTESTED CODE COMMENTED OUT BELOW
-        #      >>>  START CODE PRE DB MIGRATION HERE  <<<
         if (inst_anomalies != []):
-            # If the rootfile info has any of the marked anomalies we want it
-            # Make union of marked anomalies from our current query set, then save intersection.
-            anomaly_filters = None
+            anomaly_rootfileinfos = RootFileInfo.objects.none()
             for anomaly in inst_anomalies:
+                # If the rootfile info has any of the marked anomalies we want it
                 anomaly_filter = "anomalies__" + str(anomaly).lower()
-                this_anomaly_rootnames = current_ins_rootfileinfos.filter(**{anomaly_filter: True})
-                if anomaly_filters is None:
-                    anomaly_filters = this_anomaly_rootnames
-                else:
-                    anomaly_filters |= this_anomaly_rootnames
-            current_ins_rootfileinfos = anomaly_filters
-        #      >>>  END CODE PRE DB MIGRATION HERE  <<<
+                anomaly_rootfileinfos = anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True}))
+            current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos)
 
         # sort as desired
         if sort_type.upper() == 'ASCENDING':
@@ -1577,27 +1566,6 @@ def get_rootnames_from_query(parameters):
 
     return filtered_rootnames
 
-    # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE
-    #             ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER
-    #             NOTE: THIS CODE OUTLINE IS UNTESTED and is only a rough outline!
-    #      >>>  START CODE AFTER DB MIGRATION HERE  <<<
-    #     if (inst_anomalies != []):
-    #         # If the rootfile info has any of the marked anomalies we want it
-    #         # Make union of marked anomalies from our current query set, then save intersection.
-    #         anomaly_rootfileinfos = RootFileInfo.objects.none()
-
-    #         for anomaly in inst_anomalies:
-    #             anomaly_filter = "anomalies__" + str(anomaly).lower()
-    #             anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True}))
-
-    #         current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos)
-
-    #     # Add this instrument's query set to our return queryset
-    #     selected_rootfileinfos = selected_rootfileinfos.union(current_ins_rootfileinfos)
-
-    # return selected_rootfileinfos  # OR RETURN LIST OF ROOTNAMES
-    #      >>>  END CODE AFTER DB MIGRATION HERE  <<<
-
 
 def get_thumbnails_by_instrument(inst):
     """Return a list of thumbnails available in the filesystem for the

From 7f4334566f787a1f75dc94b8ab85dd76d9437ab4 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 1 Aug 2023 16:37:32 -0400
Subject: [PATCH 27/41] exploring

---
 environment_python_3.10.yml | 64 ++++++++++++++++++-------------------
 environment_python_3.9.yml  | 64 ++++++++++++++++++-------------------
 2 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 207f9d658..32bef050d 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -15,58 +15,58 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.10
 
-name: jwql-3.10
+name: jwql-3.10-deptest
 
 channels:
   - conda-forge
   - defaults
 
 dependencies:
-  - astropy=5.2.2
+  - astropy=5.3.1
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
-  - celery=5.2.7
-  - cryptography=40.0.2
-  - django=4.2.1
+  - celery=5.3.1
+  - cryptography=41.0.2
+  - django=4.2.3
   - inflection=0.5.1
-  - ipython=8.13.2
+  - ipython=8.14.0
   - jinja2=3.1.2
   - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.15.0
-  - numpy=1.24.3
+  - matplotlib=3.7.2
+  - nodejs=18.16.1
+  - numpy=1.25.1
   - numpydoc=1.5.0
-  - pandas=2.0.1
-  - pip=23.1.2
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.3.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.10.9
+  - pandas=2.0.3
+  - pip=23.2.1
+  - postgresql=15.3
+  - psycopg2=2.9.6
+  - pytest=7.4.0
+  - pytest-cov=4.1.0
+  - pytest-mock=3.11.1
+  - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.269
+  - ruff=0.0.280
   - scipy=1.9.3
-  - setuptools=67.7.2
+  - setuptools=68.0.0
   - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.15
+  - sphinx_rtd_theme=1.2.2
+  - sqlalchemy=2.0.19
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery==0.4.6
-    - bandit==1.7.5
-    - jwst==1.10.2
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4.1
-    - redis==4.5.5
-    - selenium==4.9.1
-    - stdatamodels==1.3.1
-    - stsci_rtd_theme==1.0.0
-    - vine==5.0.0
+    - astroquery=0.4.6
+    - bandit=1.7.5
+    - jwst=1.11.3
+    - pysiaf=0.20.0
+    - pysqlite3=0.5.1
+    - pyvo=1.4.1
+    - redis=4.6.0
+    - selenium=4.10.0
+    - stdatamodels=1.7.1
+    - stsci_rtd_theme=1.0.0
+    - vine=5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index cf20b4108..dd677fa50 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -15,58 +15,58 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.9
 
-name: jwql-3.9
+name: jwql-3.9-deptest
 
 channels:
   - conda-forge
   - defaults
 
 dependencies:
-  - astropy=5.2.2
+  - astropy=5.3.1
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
-  - celery=5.2.7
-  - cryptography=40.0.2
-  - django=4.2.1
+  - celery=5.3.1
+  - cryptography=41.0.2
+  - django=4.2.3
   - inflection=0.5.1
-  - ipython=8.13.2
+  - ipython=8.14.0
   - jinja2=3.1.2
   - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.15.0
-  - numpy=1.24.3
+  - matplotlib=3.7.2
+  - nodejs=18.16.1
+  - numpy=1.25.1
   - numpydoc=1.5.0
-  - pandas=2.0.1
-  - pip=23.1.2
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.3.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.9.16
+  - pandas=2.0.3
+  - pip=23.2.1
+  - postgresql=15.3
+  - psycopg2=2.9.6
+  - pytest=7.4.0
+  - pytest-cov=4.1.0
+  - pytest-mock=3.11.1
+  - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.269
+  - ruff=0.0.280
   - scipy=1.9.3
-  - setuptools=67.7.2
+  - setuptools=68.0.0
   - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.15
+  - sphinx_rtd_theme=1.2.2
+  - sqlalchemy=2.0.19
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery==0.4.6
-    - bandit==1.7.5
-    - jwst==1.10.2
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4.1
-    - redis==4.5.5
-    - selenium==4.9.1
-    - stdatamodels==1.3.1
-    - stsci_rtd_theme==1.0.0
-    - vine==5.0.0
+    - astroquery=0.4.6
+    - bandit=1.7.5
+    - jwst=1.11.3
+    - pysiaf=0.20.0
+    - pysqlite3=0.5.1
+    - pyvo=1.4.1
+    - redis=4.6.0
+    - selenium=4.10.0
+    - stdatamodels=1.7.1
+    - stsci_rtd_theme=1.0.0
+    - vine=5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package

From 2673b3cee46078019dfac099826482358aba2a8b Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 1 Aug 2023 16:46:30 -0400
Subject: [PATCH 28/41] Smarter about base filename

---
 jwql/shared_tasks/shared_tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 7a05f6f92..be232935d 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -459,7 +459,8 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
         logging.error("File {} not found!".format(input_file))
         raise FileNotFoundError("{} not found".format(input_file))
 
-    short_name = input_file_name[0: input_file_name.rfind('_')]
+    parts = input_file_name.split('_')
+    short_name = f'{parts[0]}_{parts[1]}_{parts[2]}_{parts[3]}'
     ensure_dir_exists(cal_dir)
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
 

From 99bcb7a83bcab8a7c7205ada5b51ab7b1275745c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 12:55:54 -0400
Subject: [PATCH 29/41] Remove unused badpix values from columndatasource

---
 .../common_monitors/dark_monitor.py                 | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 8ffab9146..75cdf1062 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -649,8 +649,7 @@ def overplot_bad_pix(self, pix_type, coords, values):
             values = []
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
-                                                       pixels_y=coords[1],
-                                                       values=values
+                                                       pixels_y=coords[1]
                                                       )
                                              )
 
@@ -658,14 +657,6 @@ def overplot_bad_pix(self, pix_type, coords, values):
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
                                                  source=sources[pix_type], color=colors[pix_type])
 
-        # Create hover tools for the bad pixel types
-        #hover_tools[pix_type] = HoverTool(tooltips=[(f'{pix_type} (x, y):', '(@pixels_x, @pixels_y)'),
-        #                                            ('value:', f'@values'),
-        #                                            ],
-        #                                    renderers=[badpixplots[pix_type]])
-        # Add tool to plot
-        #self.plot.tools.append(hover_tools[pix_type])
-
         # Add to the legend
         if  numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
@@ -814,7 +805,7 @@ def process(self, file_list):
                 logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0])))
                 self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time)
 
-            logging.info("Creating Mean Slope Image {}".format(slope_image))
+            logging.info("Creating Mean Slope Image")
             # Create png file of mean slope image. Add bad pixels only for full frame apertures
             self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix,
                                           noisyxy=new_noisy_pixels, baseline_file=baseline_file)

From 9f45a404b5ea7f0f3e6fb5021e1800848c5fa7b0 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 13:22:53 -0400
Subject: [PATCH 30/41] Set first pipeline step name per instrument

---
 jwql/shared_tasks/run_pipeline.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index a79167e0b..0531f0337 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -61,7 +61,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # If the input file is a file other than uncal.fits, then we may only need to run a
         # subset of steps. Check the completed steps in the input file. Find the latest step
         # that has been completed, and skip that plus all prior steps
-        if 'uncal.fits' not in input_file:
+        if 'uncal' not in input_file:
             completed_steps = completed_pipeline_steps(input_file)
 
             # Reverse the boolean value, so that now steps answers the question: "Do we need
@@ -73,7 +73,11 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # run, and only run subsequent steps. This protects against cases where some early
         # step was not run. In that case, we don't want to go back and run it because running
         # pipeline steps out of order doesn't work.
-        last_run = 'group_scale'  # initialize to the first step
+        if instrument in ['miri', 'nirspec']:
+            last_run = 'group_scale'  # initialize to the first step
+        else:
+            last_run = 'dq_init'
+
         for step in steps:
             if not steps[step]:
                 last_run = deepcopy(step)

From 0b4791ba9024c6a9828c82edadda3f42d453dae8 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:02:05 -0400
Subject: [PATCH 31/41] avoid logging large numpy arrays

---
 jwql/instrument_monitors/common_monitors/bias_monitor.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py
index 120cfab0e..9e1f9ad32 100755
--- a/jwql/instrument_monitors/common_monitors/bias_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/bias_monitor.py
@@ -432,7 +432,13 @@ def process(self, file_list):
             # Add this new entry to the bias database table
             with engine.begin() as connection:
                 connection.execute(self.stats_table.__table__.insert(), bias_db_entry)
-            logging.info('\tNew entry added to bias database table: {}'.format(bias_db_entry))
+
+            # Don't print long arrays of numbers to the log file
+            log_dict = {}
+            for key in bias_db_entry:
+                if key not in ['collapsed_rows', 'collapsed_columns', 'counts', 'bin_centers']:
+                    log_dict[key] = bias_db_entry[key]
+            logging.info('\tNew entry added to bias database table: {}'.format(log_dict))
 
             # Remove the raw and calibrated files to save memory space
             os.remove(filename)

From 8737521c9ab8c4ea45389faf47ef4b27ef0d94e6 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:36:02 -0400
Subject: [PATCH 32/41] Make plotting functions robust against missing db
 entries

---
 .../jwql/monitor_pages/monitor_bias_bokeh.py  | 54 +++++++++++--------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 7f14c9fb2..35cd6dbe0 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -466,31 +466,39 @@ def create_plot(self, colname):
             title_text = 'Column'
             axis_text = 'Row Number'
 
-        #datestr = self.data['expstart'][0].strftime("%m/%d/%Y")
-        datestr = self.data['expstart_str'].iloc[0]
-        title_str = f'Calibrated data: Collapsed {title_text}, {datestr}'
-
-        if len(self.data[colname].iloc[0]) > 0:
-            plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save',
-                          background_fill_color="#fafafa")
-
-            # Add a column containing pixel numbers to plot against
-            pix_num = np.arange(len(self.data[colname].iloc[0]))
-            self.data['pixel'] = [pix_num]
-
-            series = self.data.iloc[0]
-            series = series[['pixel', colname]]
-            source = ColumnDataSource(dict(series))
-            plot.scatter(x='pixel', y=colname, fill_color="#C85108", line_color="#C85108",
-                         alpha=0.75, source=source)
-
-            hover_text = axis_text.split(' ')[0]
-            hover_tool = HoverTool(tooltips=f'{hover_text} @pixel: @{colname}')
-            plot.tools.append(hover_tool)
-            plot.xaxis.axis_label = axis_text
-            plot.yaxis.axis_label = 'Median Signal (DN)'
+        # Make sure there is data present
+        if len(self.data) > 0:
+            # Make sure that the colname column is not empty
+            if len(self.data[colname].iloc[0]) > 0:
+                datestr = self.data['expstart_str'].iloc[0]
+                title_str = f'Calibrated data: Collapsed {title_text}, {datestr}'
+
+                plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save',
+                              background_fill_color="#fafafa")
+
+                # Add a column containing pixel numbers to plot against
+                pix_num = np.arange(len(self.data[colname].iloc[0]))
+                self.data['pixel'] = [pix_num]
+
+                series = self.data.iloc[0]
+                series = series[['pixel', colname]]
+                source = ColumnDataSource(dict(series))
+                plot.scatter(x='pixel', y=colname, fill_color="#C85108", line_color="#C85108",
+                             alpha=0.75, source=source)
+
+                hover_text = axis_text.split(' ')[0]
+                hover_tool = HoverTool(tooltips=f'{hover_text} @pixel: @{colname}')
+                plot.tools.append(hover_tool)
+                plot.xaxis.axis_label = axis_text
+                plot.yaxis.axis_label = 'Median Signal (DN)'
+            else:
+                # If there is a latest_data entry, but the collapsed_row or collapsed_col
+                # columns are empty, then make a placeholder plot.
+                title_str = f'Calibrated data: Collapsed {title_text}'
+                plot = PlaceholderPlot(title_str, axis_text, 'Median Signal (DN)').plot
         else:
             # If there are no data, then create an empty placeholder plot
+            title_str = f'Calibrated data: Collapsed {title_text}'
             plot = PlaceholderPlot(title_str, axis_text, 'Median Signal (DN)').plot
 
         return plot

From 8dd5bbd1a57f226f5e12bfcd45431c2b234bfb94 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:43:49 -0400
Subject: [PATCH 33/41] Make other plotting functions robust against missing db
 entries

---
 .../jwql/monitor_pages/monitor_bias_bokeh.py  | 53 ++++++++++---------
 1 file changed, 29 insertions(+), 24 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 35cd6dbe0..47a85b8b8 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -382,31 +382,36 @@ def create_plot(self):
         """
         x_label = 'Signal (DN)'
         y_label = '# Pixels'
-        if len(self.data['counts'].iloc[0]) > 0:
-
-            # In order to use Bokeh's quad, we need left and right bin edges, rather than bin centers
-            bin_centers = np.array(self.data['bin_centers'][0])
-            half_widths = (bin_centers[1:] - bin_centers[0:-1]) / 2
-            half_widths = np.insert(half_widths, 0, half_widths[0])
-            self.data['bin_left'] = [bin_centers - half_widths]
-            self.data['bin_right'] = [bin_centers + half_widths]
-
-            datestr = self.data['expstart_str'].iloc[0]
-            self.plot = figure(title=f'Calibrated data: Histogram, {datestr}', tools='pan,box_zoom,reset,wheel_zoom,save',
-                               background_fill_color="#fafafa")
-
-            # Keep only the columns where the data are a list
-            series = self.data.iloc[0]
-            series = series[['counts', 'bin_left', 'bin_right', 'bin_centers']]
-            source = ColumnDataSource(dict(series))
-            self.plot.quad(top='counts', bottom=0, left='bin_left', right='bin_right',
-                           fill_color="#C85108", line_color="#C85108", alpha=0.75, source=source)
-
-            hover_tool = HoverTool(tooltips=f'@bin_centers DN: @counts')
-            self.plot.tools.append(hover_tool)
-            self.plot.xaxis.axis_label = x_label
-            self.plot.yaxis.axis_label = y_label
+        # Be sure data is not empty
+        if len(data) > 0:
+            # Be sure the array of histogram information is not empty
+            if len(self.data['counts'].iloc[0]) > 0:
+
+                # In order to use Bokeh's quad, we need left and right bin edges, rather than bin centers
+                bin_centers = np.array(self.data['bin_centers'][0])
+                half_widths = (bin_centers[1:] - bin_centers[0:-1]) / 2
+                half_widths = np.insert(half_widths, 0, half_widths[0])
+                self.data['bin_left'] = [bin_centers - half_widths]
+                self.data['bin_right'] = [bin_centers + half_widths]
 
+                datestr = self.data['expstart_str'].iloc[0]
+                self.plot = figure(title=f'Calibrated data: Histogram, {datestr}', tools='pan,box_zoom,reset,wheel_zoom,save',
+                                   background_fill_color="#fafafa")
+
+                # Keep only the columns where the data are a list
+                series = self.data.iloc[0]
+                series = series[['counts', 'bin_left', 'bin_right', 'bin_centers']]
+                source = ColumnDataSource(dict(series))
+                self.plot.quad(top='counts', bottom=0, left='bin_left', right='bin_right',
+                               fill_color="#C85108", line_color="#C85108", alpha=0.75, source=source)
+
+                hover_tool = HoverTool(tooltips=f'@bin_centers DN: @counts')
+                self.plot.tools.append(hover_tool)
+                self.plot.xaxis.axis_label = x_label
+                self.plot.yaxis.axis_label = y_label
+
+            else:
+                self.plot = PlaceholderPlot('Calibrated data: Histogram', x_label, y_label).plot
         else:
             self.plot = PlaceholderPlot('Calibrated data: Histogram', x_label, y_label).plot
 

From 40bd0dae87e1c704c87914b56ae55c81d7ded34c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:46:38 -0400
Subject: [PATCH 34/41] typo

---
 jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 47a85b8b8..4b5a6a6de 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -383,7 +383,7 @@ def create_plot(self):
         x_label = 'Signal (DN)'
         y_label = '# Pixels'
         # Be sure data is not empty
-        if len(data) > 0:
+        if len(self.data) > 0:
             # Be sure the array of histogram information is not empty
             if len(self.data['counts'].iloc[0]) > 0:
 

From 45c1f8f64ade3edb4c1818a2334dd8f1c6ee593d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 4 Aug 2023 14:02:46 -0400
Subject: [PATCH 35/41] Check fidelity of file copy from mast

---
 .../common_monitors/dark_monitor.py                 | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 75cdf1062..c1c2ecc4c 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -1022,6 +1022,19 @@ def run(self):
                             # Copy files from filesystem
                             dark_files, not_copied = copy_files(new_filenames, self.data_dir)
 
+                            # Check that there were no problems with the file copying. If any of the copied
+                            # files have different sizes between the MAST filesystem and the JWQL filesystem,
+                            # then throw them out.
+                            for dark_file in dark_files:
+                                copied_size = os.stat(dark_file).st_size
+                                orig_size = os.stat(filesystem_path(os.path.basename(dark_file))).st_size
+                                if orig_size != copied_size:
+                                    logging.info(f"\tProblem copying {os.path.basename(dark_file)} from the filesystem.")
+                                    logging.info(f"Size in filesystem: {orig_size}, size of copy: {copied_size}. Skipping file.")
+                                    not_copied.append(dark_file)
+                                    dark_files.remove(dark_file)
+                                    os.remove(dark_file)
+
                             logging.info('\tNew_filenames: {}'.format(new_filenames))
                             logging.info('\tData dir: {}'.format(self.data_dir))
                             logging.info('\tCopied to working dir: {}'.format(dark_files))

From dc7e2720f1ea7675936e76bb2bdaa6ae97702608 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 17 Aug 2023 12:01:15 -0400
Subject: [PATCH 36/41] env updates with correct nodejs

---
 environment_python_3.10.yml | 40 +++++++++++++++----------------
 environment_python_3.9.yml  | 40 +++++++++++++++----------------
 pyproject.toml              |  3 +--
 requirements.txt            | 48 ++++++++++++++++++-------------------
 rtd_requirements.txt        | 20 ++++++++--------
 5 files changed, 75 insertions(+), 76 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 32bef050d..291e94710 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -22,51 +22,51 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.1
+  - astropy=5.3.2
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
   - celery=5.3.1
-  - cryptography=41.0.2
+  - cryptography=41.0.3
   - django=4.2.3
   - inflection=0.5.1
   - ipython=8.14.0
   - jinja2=3.1.2
-  - jsonschema=4.17.3
+  - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.1
-  - numpy=1.25.1
+  - nodejs=18.16.0
+  - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.3
-  - psycopg2=2.9.6
+  - postgresql=15.2
+  - psycopg2=2.9.3
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.280
+  - ruff=0.0.284
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.19
+  - sqlalchemy=2.0.20
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery=0.4.6
-    - bandit=1.7.5
-    - jwst=1.11.3
-    - pysiaf=0.20.0
-    - pysqlite3=0.5.1
-    - pyvo=1.4.1
-    - redis=4.6.0
-    - selenium=4.10.0
-    - stdatamodels=1.7.1
-    - stsci_rtd_theme=1.0.0
-    - vine=5.0.0
+    - astroquery==0.4.6
+    - bandit==1.7.5
+    - jwst==1.11.4
+    - pysiaf==0.20.0
+    - pysqlite3==0.5.1
+    - pyvo==1.4.2
+    - redis==5.0.0
+    - selenium==4.11.2
+    - stdatamodels==1.7.2
+    - stsci_rtd_theme==1.0.0
+    - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index dd677fa50..a42b9e770 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -22,51 +22,51 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.1
+  - astropy=5.3.2
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
   - celery=5.3.1
-  - cryptography=41.0.2
+  - cryptography=41.0.3
   - django=4.2.3
   - inflection=0.5.1
   - ipython=8.14.0
   - jinja2=3.1.2
-  - jsonschema=4.17.3
+  - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.1
-  - numpy=1.25.1
+  - nodejs=18.16.0
+  - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.3
-  - psycopg2=2.9.6
+  - postgresql=15.2
+  - psycopg2=2.9.3
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.280
+  - ruff=0.0.284
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.19
+  - sqlalchemy=2.0.20
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery=0.4.6
-    - bandit=1.7.5
-    - jwst=1.11.3
-    - pysiaf=0.20.0
-    - pysqlite3=0.5.1
-    - pyvo=1.4.1
-    - redis=4.6.0
-    - selenium=4.10.0
-    - stdatamodels=1.7.1
-    - stsci_rtd_theme=1.0.0
-    - vine=5.0.0
+    - astroquery==0.4.6
+    - bandit==1.7.5
+    - jwst==1.11.4
+    - pysiaf==0.20.0
+    - pysqlite3==0.5.1
+    - pyvo==1.4.2
+    - redis==5.0.0
+    - selenium==4.11.2
+    - stdatamodels==1.7.2
+    - stsci_rtd_theme==1.0.0
+    - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package
diff --git a/pyproject.toml b/pyproject.toml
index f17cefdf9..c17d3dc28 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,7 +27,6 @@ dependencies = [
   "jwst",
   "jwst_reffiles",
   "matplotlib",
-  "nodejs",
   "numpy",
   "numpydoc",
   "pandas",
@@ -46,7 +45,7 @@ test = [
   "pytest",
   "pytest-cov",
   "pytest-mock",
-]  
+]
 docs = [
   "sphinx",
   "sphinx_rtd_theme",
diff --git a/requirements.txt b/requirements.txt
index 501641339..9571c33c9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,38 +1,38 @@
-astropy==5.2.2
+astropy==5.3.2
 astroquery==0.4.6
 bandit==1.7.5
 beautifulsoup4==4.12.2
 bokeh==2.4.3
-celery==5.2.7
-cryptography==40.0.2
-django==4.2.1
+celery==5.3.1
+cryptography==41.0.3
+django==4.2.3
 inflection==0.5.1
-ipython==8.13.2
+ipython==8.14.0
 jinja2==3.1.2
-jsonschema==4.17.3
-jwst==1.10.2
-matplotlib==3.7.0
-nodejs==0.1.1
-numpy==1.24.3
+jsonschema==4.19.0
+jwst==1.11.4
+matplotlib==3.7.2
+nodejs==18.16.0
+numpy==1.25.2
 numpydoc==1.5.0
-pandas==2.0.1
+pandas==2.0.3
 psycopg2-binary==2.9.3
-pysiaf==0.19.1
-pysqlite3==0.5.0
-pytest==7.3.1
-pytest-cov==4.0.0
-pytest-mock==3.10.0
-pyvo==1.4.1
+pysiaf==0.20.0
+pysqlite3==0.5.1
+pytest==7.4.0
+pytest-cov==4.1.0
+pytest-mock==3.11.1
+pyvo==1.4.2
 pyyaml==6.0
-redis==4.5.5
-ruff==0.0.269
+redis==5.0.0
+ruff==0.0.284
 scipy==1.9.3
-selenium==4.9.1
-setuptools==67.7.2
+selenium==4.11.2
+setuptools==68.0.0
 sphinx==6.2.1
-sphinx_rtd_theme==1.2.0
-sqlalchemy==2.0.15
-stdatamodels==1.3.1
+sphinx_rtd_theme==1.2.2
+sqlalchemy==2.0.20
+stdatamodels==1.7.2
 stsci_rtd_theme==1.0.0
 twine==4.0.2
 vine==5.0.0
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index ba1ecf587..aa1047b5e 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -1,15 +1,15 @@
-sphinx_automodapi==0.14.1
+sphinx_automodapi==0.15.0
 bokeh==2.4.3
-celery==5.2.7
-cython==0.29.33
-django==4.2.1
+celery==5.3.1
+cython==3.0.0
+django==4.2.3
 docutils==0.18.1
-jwst==1.10.2
-pygments==2.14.0
-pytest==7.3.1
-redis==4.5.5
-sphinx>=2
-sphinx_rtd_theme==1.2.0
+jwst==1.11.4
+pygments==2.16.1
+pytest==7.4.0
+redis==5.0.0
+sphinx==6.2.1
+sphinx_rtd_theme==1.2.2
 stsci_rtd_theme==1.0.0
 tomli==2.0.1
 git+https://github.com/spacetelescope/jwst_reffiles

From bc3b546ad514d5b2b73e4279ee3214a9774f2e10 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 17 Aug 2023 12:02:00 -0400
Subject: [PATCH 37/41] Return env name back to nominal

---
 environment_python_3.10.yml | 2 +-
 environment_python_3.9.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 291e94710..f554506b0 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -15,7 +15,7 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.10
 
-name: jwql-3.10-deptest
+name: jwql-3.10
 
 channels:
   - conda-forge
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index a42b9e770..3986b0829 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -15,7 +15,7 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.9
 
-name: jwql-3.9-deptest
+name: jwql-3.9
 
 channels:
   - conda-forge

From a6beb9415c8e2bf6335f24a6f47f42becc4b3078 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 18 Aug 2023 12:29:09 -0400
Subject: [PATCH 38/41] bring nodejs up to most recent version

---
 environment_python_3.10.yml | 14 +++++++-------
 environment_python_3.9.yml  | 14 +++++++-------
 requirements.txt            |  6 +++---
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index f554506b0..f61973680 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -1,10 +1,10 @@
 # This file describes a conda environment that can be to install jwql
 #
 # Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_10.yml
+# $ conda env create -f environment_python_3.10.yml
 #
 # The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_10.yml
+# $ conda env create -n <custom name> -f environment_python_3.10.yml
 #
 # Run the following command to activate the environment:
 # $ source activate jwql-3.10
@@ -23,8 +23,8 @@ channels:
 
 dependencies:
   - astropy=5.3.2
-  - bokeh=2.4.3
   - beautifulsoup4=4.12.2
+  - bokeh=2.4.3
   - celery=5.3.1
   - cryptography=41.0.3
   - django=4.2.3
@@ -33,20 +33,20 @@ dependencies:
   - jinja2=3.1.2
   - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.0
+  - nodejs=20.5.1
   - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.2
-  - psycopg2=2.9.3
+  - postgresql=15.4
+  - psycopg2=2.9.6
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.284
+  - ruff=0.0.285
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 3986b0829..d3724b1d8 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -1,10 +1,10 @@
 # This file describes a conda environment that can be to install jwql
 #
 # Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_9.yml
+# $ conda env create -f environment_python_3.9.yml
 #
 # The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_9.yml
+# $ conda env create -n <custom name> -f environment_python_3.9.yml
 #
 # Run the following command to activate the environment:
 # $ source activate jwql-3.9
@@ -23,8 +23,8 @@ channels:
 
 dependencies:
   - astropy=5.3.2
-  - bokeh=2.4.3
   - beautifulsoup4=4.12.2
+  - bokeh=2.4.3
   - celery=5.3.1
   - cryptography=41.0.3
   - django=4.2.3
@@ -33,20 +33,20 @@ dependencies:
   - jinja2=3.1.2
   - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.0
+  - nodejs=20.5.1
   - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.2
-  - psycopg2=2.9.3
+  - postgresql=15.4
+  - psycopg2=2.9.6
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.284
+  - ruff=0.0.285
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
diff --git a/requirements.txt b/requirements.txt
index 9571c33c9..4391be8b3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,11 +12,11 @@ jinja2==3.1.2
 jsonschema==4.19.0
 jwst==1.11.4
 matplotlib==3.7.2
-nodejs==18.16.0
+nodejs==20.5.1
 numpy==1.25.2
 numpydoc==1.5.0
 pandas==2.0.3
-psycopg2-binary==2.9.3
+psycopg2-binary==2.9.7
 pysiaf==0.20.0
 pysqlite3==0.5.1
 pytest==7.4.0
@@ -25,7 +25,7 @@ pytest-mock==3.11.1
 pyvo==1.4.2
 pyyaml==6.0
 redis==5.0.0
-ruff==0.0.284
+ruff==0.0.285
 scipy==1.9.3
 selenium==4.11.2
 setuptools==68.0.0

From b85a1b0f7080afc1a58a1e8070b22377cf878b58 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 21 Sep 2023 21:21:42 +0000
Subject: [PATCH 39/41] Bump cryptography from 41.0.3 to 41.0.4

Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4391be8b3..d4f3a5616 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ bandit==1.7.5
 beautifulsoup4==4.12.2
 bokeh==2.4.3
 celery==5.3.1
-cryptography==41.0.3
+cryptography==41.0.4
 django==4.2.3
 inflection==0.5.1
 ipython==8.14.0

From 7ce0d9c89e9ace520c98f2487762e5cb208156e4 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 25 Sep 2023 13:19:07 -0400
Subject: [PATCH 40/41] Fix typo in env file names in readme

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 70b4e173b..377005a11 100644
--- a/README.md
+++ b/README.md
@@ -83,16 +83,16 @@ source activate base/root
 
 **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step.
 
-Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
+Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3_9.yml
+conda env create -f environment_python_3.9.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3_10.yml
+conda env create -f environment_python_3.10.yml
 ```
 
 ### Configuration File

From ebf05a9e22ca555ee3f7d9b48fcd48dd0068de5c Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 28 Sep 2023 10:59:37 -0400
Subject: [PATCH 41/41] Include migrations in source control

---
 .../apps/jwql/migrations/0001_initial.py      | 45 +++++++++
 .../migrations/0002_auto_20220913_1525.py     | 27 ++++++
 .../migrations/0003_auto_20220921_0955.py     | 58 +++++++++++
 .../migrations/0004_auto_20220922_0911.py     | 33 +++++++
 .../migrations/0005_auto_20220922_1422.py     | 23 +++++
 .../migrations/0006_auto_20230214_1624.py     | 95 +++++++++++++++++++
 .../migrations/0007_auto_20230222_1157.py     | 22 +++++
 .../migrations/0008_rootfileinfo_exp_type.py  | 18 ++++
 .../migrations/0009_auto_20230303_0930.py     | 63 ++++++++++++
 .../migrations/0010_auto_20230313_1053.py     | 23 +++++
 jwql/website/apps/jwql/migrations/__init__.py |  0
 11 files changed, 407 insertions(+)
 create mode 100644 jwql/website/apps/jwql/migrations/0001_initial.py
 create mode 100644 jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py
 create mode 100644 jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py
 create mode 100644 jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py
 create mode 100644 jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py
 create mode 100644 jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py
 create mode 100644 jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py
 create mode 100644 jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py
 create mode 100644 jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py
 create mode 100644 jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py
 create mode 100644 jwql/website/apps/jwql/migrations/__init__.py

diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py
new file mode 100644
index 000000000..9de5ca6a9
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0001_initial.py
@@ -0,0 +1,45 @@
+# Generated by Django 3.1.7 on 2022-09-09 17:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImageData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')),
+                ('pub_date', models.DateTimeField(verbose_name='date published')),
+                ('filepath', models.FilePathField(path='/user/lchambers/jwql/')),
+            ],
+            options={
+                'verbose_name_plural': 'image data',
+                'db_table': 'imagedata',
+            },
+        ),
+        migrations.CreateModel(
+            name='InstrumentFilterHandler',
+            fields=[
+                ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ThumbnailFilterInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('observation', models.PositiveIntegerField()),
+                ('proposal', models.PositiveIntegerField()),
+                ('root_name', models.CharField(max_length=300)),
+                ('marked_viewed', models.BooleanField(default=False)),
+                ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')),
+            ],
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py
new file mode 100644
index 000000000..049901811
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py
@@ -0,0 +1,27 @@
+# Generated by Django 3.1.7 on 2022-09-13 20:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='thumbnailfilterinfo',
+            name='id',
+        ),
+        migrations.AlterField(
+            model_name='instrumentfilterhandler',
+            name='instrument',
+            field=models.TextField(max_length=10, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='thumbnailfilterinfo',
+            name='root_name',
+            field=models.TextField(max_length=300, primary_key=True, serialize=False),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py
new file mode 100644
index 000000000..df7bc6965
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py
@@ -0,0 +1,58 @@
+# Generated by Django 3.1.7 on 2022-09-21 14:55
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0002_auto_20220913_1525'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Archive',
+            fields=[
+                ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)),
+            ],
+            options={
+                'ordering': ['instrument'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Observation',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)),
+                ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')),
+                ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')),
+                ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')),
+                ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)),
+            ],
+            options={
+                'ordering': ['-obsnum'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Proposal',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)),
+                ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)),
+                ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')),
+            ],
+            options={
+                'ordering': ['-prop_id'],
+                'unique_together': {('prop_id', 'archive')},
+            },
+        ),
+        migrations.DeleteModel(
+            name='ImageData',
+        ),
+        migrations.AddField(
+            model_name='observation',
+            name='proposal',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py
new file mode 100644
index 000000000..998dd2ce4
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py
@@ -0,0 +1,33 @@
+# Generated by Django 3.1.7 on 2022-09-22 14:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0003_auto_20220921_0955'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='thumbnailfilterinfo',
+            name='obsnum',
+            field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name='RootFileInfo',
+            fields=[
+                ('instrument', models.CharField(help_text='Instrument name', max_length=7)),
+                ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)),
+                ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)),
+                ('viewed', models.BooleanField(default=False)),
+                ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')),
+            ],
+            options={
+                'ordering': ['-root_name'],
+            },
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py
new file mode 100644
index 000000000..3c51cbe23
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.1.7 on 2022-09-22 19:22
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0004_auto_20220922_0911'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='thumbnailfilterinfo',
+            name='inst_handler',
+        ),
+        migrations.DeleteModel(
+            name='InstrumentFilterHandler',
+        ),
+        migrations.DeleteModel(
+            name='ThumbnailFilterInfo',
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py
new file mode 100644
index 000000000..493608df3
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py
@@ -0,0 +1,95 @@
+# Generated by Django 3.1.7 on 2023-02-14 21:24
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0005_auto_20220922_1422'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Anomalies',
+            fields=[
+                ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')),
+                ('cosmic_ray_shower', models.BooleanField(default=False)),
+                ('diffraction_spike', models.BooleanField(default=False)),
+                ('excessive_saturation', models.BooleanField(default=False)),
+                ('guidestar_failure', models.BooleanField(default=False)),
+                ('persistence', models.BooleanField(default=False)),
+                ('crosstalk', models.BooleanField(default=False)),
+                ('data_transfer_error', models.BooleanField(default=False)),
+                ('ghost', models.BooleanField(default=False)),
+                ('snowball', models.BooleanField(default=False)),
+                ('column_pull_up', models.BooleanField(default=False)),
+                ('column_pull_down', models.BooleanField(default=False)),
+                ('dominant_msa_leakage', models.BooleanField(default=False)),
+                ('dragons_breath', models.BooleanField(default=False)),
+                ('mrs_glow', models.BooleanField(default=False)),
+                ('mrs_zipper', models.BooleanField(default=False)),
+                ('internal_reflection', models.BooleanField(default=False)),
+                ('optical_short', models.BooleanField(default=False)),
+                ('row_pull_up', models.BooleanField(default=False)),
+                ('row_pull_down', models.BooleanField(default=False)),
+                ('lrs_contamination', models.BooleanField(default=False)),
+                ('tree_rings', models.BooleanField(default=False)),
+                ('scattered_light', models.BooleanField(default=False)),
+                ('claws', models.BooleanField(default=False)),
+                ('wisps', models.BooleanField(default=False)),
+                ('tilt_event', models.BooleanField(default=False)),
+                ('light_saber', models.BooleanField(default=False)),
+                ('other', models.BooleanField(default=False)),
+            ],
+            options={
+                'ordering': ['-root_file_info'],
+            },
+        ),
+        migrations.AddField(
+            model_name='proposal',
+            name='cat_type',
+            field=models.CharField(default='', help_text='Category Type', max_length=10),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='aperature',
+            field=models.CharField(default='', help_text='Aperature', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='detector',
+            field=models.CharField(default='', help_text='Detector', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='filter',
+            field=models.CharField(default='', help_text='Instrument name', max_length=7),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='grating',
+            field=models.CharField(default='', help_text='Grating', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='pupil',
+            field=models.CharField(default='', help_text='Pupil', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='read_patt',
+            field=models.CharField(default='', help_text='Read Pattern', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='read_patt_num',
+            field=models.IntegerField(default=0, help_text='Read Pattern Number'),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='subarray',
+            field=models.CharField(default='', help_text='Subarray', max_length=40),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py
new file mode 100644
index 000000000..2d4ffdba3
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py
@@ -0,0 +1,22 @@
+# Generated by Django 3.1.7 on 2023-02-22 16:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0006_auto_20230214_1624'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='rootfileinfo',
+            name='aperature',
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='aperture',
+            field=models.CharField(default='', help_text='Aperture', max_length=40),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py
new file mode 100644
index 000000000..2c67cd955
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.1.7 on 2023-02-22 17:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0007_auto_20230222_1157'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='exp_type',
+            field=models.CharField(default='', help_text='Exposure Type', max_length=40),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py
new file mode 100644
index 000000000..d0dbafc76
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py
@@ -0,0 +1,63 @@
+# Generated by Django 3.1.7 on 2023-03-03 14:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0008_rootfileinfo_exp_type'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='proposal',
+            old_name='cat_type',
+            new_name='category',
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='expstart',
+            field=models.FloatField(default=0.0, help_text='Exposure Start Time'),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='aperture',
+            field=models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='detector',
+            field=models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='exp_type',
+            field=models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='filter',
+            field=models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='grating',
+            field=models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='pupil',
+            field=models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='read_patt',
+            field=models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='subarray',
+            field=models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py
new file mode 100644
index 000000000..3d80e95fe
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.1.7 on 2023-03-13 15:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0009_auto_20230303_0930'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='anomalies',
+            name='flag_date',
+            field=models.DateTimeField(blank=True, help_text='flag date', null=True),
+        ),
+        migrations.AddField(
+            model_name='anomalies',
+            name='user',
+            field=models.CharField(blank=True, default='', help_text='user', max_length=50, null=True),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb