First try adding diag_mode to default diagnostics

Introduce new diag_mode option to MARBL_generate_diagnostics_file.py; when
diag_mode = 'none', output diagnostic file should be 'never_{op}' for all
variables. When diag_mode = 'minimal', only variables with diag_mode =
'minimal' in diagnostics_latest.json have non-never output frequencies. When
diag_mode is 'full', variables with diag_mode = 'minimal' or 'full' will have
non-never output frequencies.

MARBL_tools/MARBL_diagnostics_file_class.py

@@ -53,10 +53,11 @@ def __init__(self, default_diagnostics_file, MARBL_settings, unit_system):
                 diags_to_delete.append(diag_name)
                 continue
 
-            #    iii. frequency and operator should always be lists
+            #    iii. frequency, operator, and diag_mode should always be lists
             if not isinstance(self.diagnostics_dict[diag_name]['frequency'], list):
                 self.diagnostics_dict[diag_name]['frequency'] = [self.diagnostics_dict[diag_name]['frequency']]
                 self.diagnostics_dict[diag_name]['operator'] = [self.diagnostics_dict[diag_name]['operator']]
+                self.diagnostics_dict[diag_name]['diag_mode'] = [self.diagnostics_dict[diag_name]['diag_mode']]
 
             #    iv. update units
             fix_units = {}

MARBL_tools/MARBL_generate_diagnostics_file.py

@@ -45,9 +45,10 @@
   -a, --append          Append to existing diagnostics file (default: False)
 """
 
+import MARBL_utils
 #######################################
 
-def generate_diagnostics_file(MARBL_diagnostics, diagnostics_file_out, append=False):
+def generate_diagnostics_file(MARBL_diagnostics, diagnostics_file_out, diag_mode="full", append=False):
     """ Produce a list of MARBL diagnostic frequencies and operators from a JSON parameter file
     """
 
@@ -84,12 +85,18 @@ def generate_diagnostics_file(MARBL_diagnostics, diagnostics_file_out, append=Fa
     # is also a dictionary containing frequency and operator information. Note that
     # string values of frequency and operator are converted to lists of len 1 when the
     # JSON file that generates this list is processed
+    diag_mode_opts = MARBL_utils.diag_mode_opts()
+    diag_mode_in = diag_mode_opts.index(diag_mode)
     for diag_name in sorted(MARBL_diagnostics.diagnostics_dict.keys()):
         frequencies = MARBL_diagnostics.diagnostics_dict[diag_name]['frequency']
         operators = MARBL_diagnostics.diagnostics_dict[diag_name]['operator']
+        diag_modes = MARBL_diagnostics.diagnostics_dict[diag_name]['diag_mode']
         freq_op = []
-        for freq, op in zip(frequencies, operators):
-            freq_op.append(freq + '_' + op)
+        for freq, op, dm in zip(frequencies, operators, diag_modes):
+            if diag_mode_in >= diag_mode_opts.index(dm):
+                freq_op.append(freq + '_' + op)
+            else:
+                freq_op.append('never_' + op)
         fout.write("%s : %s\n" % (diag_name, ", ".join(freq_op)))
     fout.close()
 
@@ -116,7 +123,7 @@ def _parse_args(marbl_root):
 
     # Is the GCM providing initial bury coefficients via saved state?
     parser.add_argument('-v', '--saved_state_vars_source', action='store', dest='saved_state_vars_source',
-                        default='settings_file', choices = set(('settings_file', 'GCM')),
+                        default='settings_file', choices=['settings_file', 'GCM'],
                         help="Source of initial value for saved state vars that can come from GCM or settings file")
 
     # Command line argument to specify resolution (default is None)
@@ -135,6 +142,11 @@ def _parse_args(marbl_root):
     parser.add_argument('-u', '--unit_system', action='store', dest='unit_system', default='cgs',
                         choices=['cgs', 'mks'], help='Unit system for parameter values')
 
+    # Diagnostic mode (level of output to include)
+    parser.add_argument('-m', '--diag-mode', action='store', dest='diag_mode', default='full',
+                        choices=MARBL_utils.diag_mode_opts(),
+                        help='Level of output to include')
+
     # Append to existing diagnostics file?
     parser.add_argument('-a', '--append', action='store_true', dest='append',
                         help='Append to existing diagnostics file')
@@ -163,7 +175,8 @@ def _parse_args(marbl_root):
                                            grid=args.grid,
                                            input_file=args.settings_file_in,
                                            unit_system=args.unit_system)
-    MARBL_diagnostics = MARBL_diagnostics_class(args.default_diagnostics_file, DefaultSettings, args.unit_system)
+    MARBL_diagnostics = MARBL_diagnostics_class(args.default_diagnostics_file, DefaultSettings,
+                                                args.unit_system)
 
     # Write the diagnostic file
-    generate_diagnostics_file(MARBL_diagnostics, args.diagnostics_file_out, args.append)
+    generate_diagnostics_file(MARBL_diagnostics, args.diagnostics_file_out, args.diag_mode, args.append)

MARBL_tools/MARBL_utils.py

@@ -8,6 +8,14 @@
 #                            PUBLIC MODULE METHODS                             #
 ################################################################################
 
+def diag_mode_opts():
+    """ Return ordered list of the valid values for diag_mode.
+        (Ordered list => selecting diag mode includes variables)
+    """
+    return ['none', 'minimal', 'full']
+
+################################################################################
+
 def settings_dictionary_is_consistent(SettingsDict):
     """ Make sure dictionary generated from JSON settings file conforms to MARBL
         parameter file standards:
@@ -82,13 +90,15 @@ def diagnostics_dictionary_is_consistent(DiagsDict):
            iii. vertical_grid (2D vars should explicitly list "none")
            iv.  frequency
            v.   operator
+           vi.  diag_mode
         3. Diagnostic variable dictionaries may contain 'dependencies' key as well,
            but it must be a dictionary itself.
-        4. Consistency between frequency and operator
-           i.   frequency and operator are both lists, or neither are
-           ii.  If they are both lists, must be same size
+        4. Consistency among frequency, operator, and diag_mode
+           i.   frequency, operator, and diag_mode are all lists, or none are
+           ii.  If they are all lists, must be same length
         5. Allowable frequencies are never, low, medium, and high
         6. Allowable operators are instantaneous, average, minimum, and maximum
+        7. Allowable diag_modes are defined in diag_mode_opts()
     """
 
     import logging
@@ -109,8 +119,8 @@ def diagnostics_dictionary_is_consistent(DiagsDict):
             continue
 
         # 2. diag_dict must have the following keys:
-        valid_keys = ["longname", "units", "vertical_grid", "frequency", "operator"]
-        for key_check in valid_keys:
+        required_keys = ["longname", "units", "vertical_grid", "frequency", "operator", "diag_mode"]
+        for key_check in required_keys:
             if key_check not in diag_dict.keys():
                 message = "Diagnostic %s is not well-defined in YAML" % diag_name
                 message = message + "\n     * Expecting %s as a key" % key_check
@@ -124,48 +134,67 @@ def diagnostics_dictionary_is_consistent(DiagsDict):
                 logger.error(message)
                 invalid_file = True
 
-        # 4. Consistency between frequency and operator
+        # 4. Consistency among frequency, operator, diag_mode
         err_prefix = "Inconsistency in DiagsDict['%s']:" % diag_name
         #    i.   frequency and operator are both lists, or neither are
-        if isinstance(diag_dict['frequency'], list) != isinstance(diag_dict['operator'], list):
-            logger.error("%s either both frequency and operator must be lists or neither can be" % err_prefix)
+        if (isinstance(diag_dict['frequency'], list) != isinstance(diag_dict['operator'], list)) or \
+           (isinstance(diag_dict['frequency'], list) != isinstance(diag_dict['diag_mode'], list)):
+            err_message = "either all of frequency, operator, and diag_mode must be lists or neither can be"
+            logger.error(f"{err_prefix} {err_message}")
             invalid_file = True
             continue
 
         #    ii.  If they are both lists, must be same size
         if isinstance(diag_dict['frequency'], list):
             freq_len = len(diag_dict['frequency'])
             op_len = len(diag_dict['operator'])
-            if freq_len != op_len:
-                logger.error("%s frequency is length %d but operator is length %d" %
-                             (err_prefix, diag_name, freq_len, op_len))
+            dm_len = len(diag_dict['diag_mode'])
+            if (freq_len != op_len) or (freq_len != op_len):
+                err_message = f"frequency, operator, diag_mode lengths are {freq_len}, {op_len}, {diag_mode}"
+                logger.error(f"{err_prefix} {err_message}")
                 invalid_file = True
                 continue
 
         # 5. Allowable frequencies are never, low, medium, and high
         # 6. Allowable operators are instantaneous, average, minimum, and maximum
+        # 7. Allowable diag_modes are defined in diag_mode_opts()
+        #    * "none" should not appear in the dictionary
         ok_freqs = ['never', 'low', 'medium', 'high']
         ok_ops = ['instantaneous', 'average', 'minimum', 'maximum']
-        invalid_freq_op = False
+        ok_dms = diag_mode_opts()[1:] # do not include 'none' in ok_dms
+        invalid_freq_op_dm = False
         if not isinstance(diag_dict['frequency'], dict):
             if isinstance(diag_dict['frequency'], list):
-                for freq, op in zip(diag_dict['frequency'], diag_dict['operator']):
+                for freq, op, dm in zip(diag_dict['frequency'], diag_dict['operator'], diag_dict['diag_mode']):
                     if freq not in ok_freqs:
-                        logger.error("%s '%s' is not a valid frequency" % (err_prefix, freq))
-                        invalid_freq_op = True
+                        err_message = f"'{freq}' is not a valid frequency"
+                        logger.error(f'{err_prefix} {err_message}')
+                        invalid_freq_op_dm = True
                     if op not in ok_ops:
-                        logger.error("%s '%s' is not a valid operator" % (err_prefix, op))
-                        invalid_freq_op = True
+                        err_message = f"'{op}' is not a valid operator"
+                        logger.error(f'{err_prefix} {err_message}')
+                        invalid_freq_op_dm = True
+                    if dm not in ok_dms:
+                        err_message = f"'{dm}' is not a valid diag_mode"
+                        logger.error(f'{err_prefix} {err_message}')
+                        invalid_freq_op_dm = True
             else:
                 freq = diag_dict['frequency']
                 op = diag_dict['operator']
+                dm = diag_dict['diag_mode']
                 if freq not in ok_freqs:
-                    logger.error("%s '%s' is not a valid frequency" % (err_prefix, freq))
-                    invalid_freq_op = True
+                    err_message = f"'{freq}' is not a valid frequency"
+                    logger.error(f'{err_prefix} {err_message}')
+                    invalid_freq_op_dm = True
                 if op not in ok_ops:
-                    logger.error("%s '%s' is not a valid operator" % (err_prefix, op))
-                    invalid_freq_op = True
-        if invalid_freq_op:
+                    err_message = f"'{op}' is not a valid operator"
+                    logger.error(f'{err_prefix} {err_message}')
+                    invalid_freq_op_dm = True
+                if dm not in ok_dms:
+                    err_message = f"'{dm}' is not a valid diag_mode"
+                    logger.error(f'{err_prefix} {err_message}')
+                    invalid_freq_op_dm = True
+        if invalid_freq_op_dm:
             invalid_file = True
 
     return (not invalid_file)