Merge branch 'main' into call_mdtf_fixed

lib/adf_variable_defaults.yaml

@@ -54,6 +54,9 @@
 # derivable_from -> If not present in the available output files, the variable can be derived from
 #                   other variables that are present (e.g. PRECT can be derived from PRECC and PRECL),
 #                   which are specified in this list
+#                   NOTE: this is not very flexible at the moment! It can only handle variables that
+#                         are sums of the constituents. Futher flexibility is being explored.
+#
 #
 # Final Note:  Please do not modify this file unless you plan to push your changes back to the ADF repo.
 #              If you would like to modify this file for your personal ADF runs then it is recommended
@@ -62,6 +65,19 @@
 #
 #+++++++++++
 
+#+++++++++++++
+# Available ADF Default Plot Types
+#+++++++++++++
+default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional",
+                  "NHPolar","SHPolar","Special"]
+
+#+++++++++++++
+# Constants
+#+++++++++++++
+
+#Dry Air Gas Constant:
+Rgas: 287.04 #[J/K/Kg]=8.314/0.028965
+
 #+++++++++++++
 # Category: Microphysics
 #+++++++++++++
@@ -181,6 +197,87 @@ SO2:
 SOAG:
   category: "Aerosols"
 
+BC:
+  colormap: "RdBu_r"
+  diff_colormap: "BrBG"
+  scale_factor: 1000000000
+  add_offset: 0
+  new_unit: '$\mu$g/m3'
+  mpl:
+    colorbar:
+      label : '$\mu$g/m3'
+  category: "Aerosols"
+  derivable_from: ["bc_a1", "bc_a4"]
+
+POM:
+  colormap: "RdBu_r"
+  diff_colormap: "BrBG"
+  scale_factor: 1000000000
+  add_offset: 0
+  new_unit: '$\mu$g/m3'
+  mpl:
+    colorbar:
+      label : '$\mu$g/m3'
+  category: "Aerosols"
+  derivable_from: ["pom_a1", "pom_a4"]
+
+SO4:
+  colormap: "RdBu_r"
+  diff_colormap: "BrBG"
+  scale_factor: 1000000000
+  add_offset: 0
+  new_unit: '$\mu$g/m3'
+  mpl:
+    colorbar:
+      label : '$\mu$g/m3'
+  category: "Aerosols"
+  derivable_from: ["so4_a1", "so4_a2", "so4_a3", "so4_a5"]
+
+SOA:
+  colormap: "RdBu_r"
+  diff_colormap: "BrBG"
+  scale_factor: 1000000000
+  add_offset: 0
+  new_unit: '$\mu$g/m3'
+  mpl:
+    colorbar:
+      label : '$\mu$g/m3'
+  category: "Aerosols"
+  derivable_from: ["soa_a1", "soa_a2"]
+
+DUST:
+  colormap: "RdBu_r"
+  contour_levels: [0,0.1,0.25,0.4,0.6,0.8,1.4,2,3,4,8,12,30,48,114,180]
+  non_linear: True
+  diff_colormap: "BrBG"
+  scale_factor: 1000000000
+  add_offset: 0
+  new_unit: '$\mu$g/m3'
+  mpl:
+    colorbar:
+      label : '$\mu$g/m3'
+  category: "Aerosols"
+  derivable_from: ["dst_a1", "dst_a2", "dst_a3"]
+
+SeaSalt:
+  colormap: "RdBu_r"
+  contour_levels: [0,0.05,0.075,0.2,0.3,0.4,0.7,1,1.5,2,4,6,15,24,57,90]
+  non_linear: True
+  diff_colormap: "BrBG"
+  scale_factor: 1000000000
+  add_offset: 0
+  new_unit: '$\mu$g/m3'
+  mpl:
+    colorbar:
+      label : '$\mu$g/m3'
+      ticks: [0.05,0.2,0.4,1,2,6,24,90]
+    diff_colorbar:
+      label : '$\mu$g/m3'
+      ticks: [-10,8,6,4,2,0,-2,-4,-6,-8,-10]
+  category: "Aerosols"
+  derivable_from: ["ncl_a1", "ncl_a2", "ncl_a3"]
+
+
 
 #+++++++++++++++++
 # Category: Budget
@@ -497,7 +594,6 @@ QFLX:
 # Category: Surface variables
 #+++++++++++++++++
 
-
 PBLH:
   category: "Surface variables"
   obs_file: "PBLH_ERA5_monthly_climo_197901-202112.nc"
@@ -1093,4 +1189,4 @@ utendwtem:
   obs_var_name: "utendwtem"
 
 #-----------
-#End of File
+#End of File

lib/adf_web.py

@@ -353,6 +353,9 @@ def jinja_list(seas_list):
             main_site_path = "" #Set main_site_path to blank value
         #End if
 
+        #Access variable defaults yaml file
+        res = self.variable_defaults
+
         #Extract needed variables from yaml file:
         case_names = self.get_cam_info('cam_case_name', required=True)
 
@@ -650,23 +653,21 @@ def jinja_list(seas_list):
                     ofil.write(rndr)
                 #End with
 
-                #Check if the mean plot type page exists for this case:
+                #Mean plot type html file name
                 mean_ptype_file = img_pages_dir / f"mean_diag_{web_data.plot_type}.html"
-                if not mean_ptype_file.exists():
-                    #Construct individual plot type mean_diag html files, if they don't
-                    #already exist:
-                    mean_tmpl = jinenv.get_template('template_mean_diag.html')
-
-                    #Remove keys from main dictionary for this html page
-                    templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}}
-                    templ_rend_kwarg_dict["list"] = jinja_list
-                    mean_rndr = mean_tmpl.render(templ_rend_kwarg_dict)
-
-                    #Write mean diagnostic plots HTML file:
-                    with open(mean_ptype_file,'w', encoding='utf-8') as ofil:
-                        ofil.write(mean_rndr)
-                    #End with
-                #End if (mean_ptype exists)
+
+                #Construct individual plot type mean_diag html files
+                mean_tmpl = jinenv.get_template('template_mean_diag.html')
+
+                #Remove keys from main dictionary for this html page
+                templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}}
+                templ_rend_kwarg_dict["list"] = jinja_list
+                mean_rndr = mean_tmpl.render(templ_rend_kwarg_dict)
+
+                #Write mean diagnostic plots HTML file:
+                with open(mean_ptype_file,'w', encoding='utf-8') as ofil:
+                    ofil.write(mean_rndr)
+                #End with
             #End if (data frame)
 
             #Also check if index page exists for this case:
@@ -681,6 +682,15 @@ def jinja_list(seas_list):
             plot_types = plot_type_html
             #End if
 
+            #List of ADF default plot types
+            avail_plot_types = res["default_ptypes"]
+
+            #Check if current plot type is in ADF default.
+            #If not, add it so the index.html file can include it
+            for ptype in plot_types.keys():
+                if ptype not in avail_plot_types:
+                    avail_plot_types.append(plot_types)
+
             #Construct index.html
             index_title = "AMP Diagnostics Prototype"
             index_tmpl = jinenv.get_template('template_index.html')
@@ -689,7 +699,8 @@ def jinja_list(seas_list):
                                             base_name=data_name,
                                             case_yrs=case_yrs,
                                             baseline_yrs=baseline_yrs,
-                                            plot_types=plot_types)
+                                            plot_types=plot_types,
+                                            avail_plot_types=avail_plot_types)
 
             #Write Mean diagnostics index HTML file:
             with open(index_html_file, 'w', encoding='utf-8') as ofil:
@@ -748,4 +759,4 @@ def jinja_list(seas_list):
         print("  ...Webpages have been generated successfully.")
 #++++++++++++++++++++
 #End Class definition
-#++++++++++++++++++++
+#++++++++++++++++++++

lib/plotting_functions.py

@@ -3,6 +3,8 @@
 
 Functions
 ---------
+load_dataset()
+    generalized load dataset method used for plotting/analysis functions
 use_this_norm()
     switches matplotlib color normalization method
 get_difference_colors(values)
@@ -96,10 +98,19 @@
 import geocat.comp as gcomp
 from mpl_toolkits.axes_grid1.inset_locator import inset_axes
 from matplotlib.lines import Line2D
+import matplotlib.cm as cm
 
 from adf_diag import AdfDiag
 from adf_base import AdfError
 
+import warnings  # use to warn user about missing files.
+
+#Format warning messages:
+def my_formatwarning(msg, *args, **kwargs):
+    """Issue `msg` as warning."""
+    return str(msg) + '\n'
+warnings.formatwarning = my_formatwarning
+
 #Set non-X-window backend for matplotlib:
 mpl.use('Agg')
 
@@ -123,6 +134,33 @@
 #HELPER FUNCTIONS
 #################
 
+def load_dataset(fils):
+    """
+    This method exists to get an xarray Dataset from input file information that can be passed into the plotting methods.
+
+    Parameters
+    ----------
+    fils : list
+        strings or paths to input file(s)
+
+    Returns
+    -------
+    xr.Dataset
+
+    Notes
+    -----
+    When just one entry is provided, use `open_dataset`, otherwise `open_mfdatset`
+    """
+    if len(fils) == 0:
+        warnings.warn(f"Input file list is empty.")
+        return None
+    elif len(fils) > 1:
+        return xr.open_mfdataset(fils, combine='by_coords')
+    else:
+        return xr.open_dataset(fils[0])
+    #End if
+#End def
+
 def use_this_norm():
     """Just use the right normalization; avoids a deprecation warning."""
 
@@ -723,20 +761,20 @@ def make_polar_plot(wks, case_nickname, base_nickname,
     levs_diff = np.unique(np.array(levelsdiff))
 
     if len(levs) < 2:
-        img1 = ax1.contourf(lons, lats, d1_cyclic, transform=ccrs.PlateCarree(), transform_first=True, colors="w", norm=norm1)
+        img1 = ax1.contourf(lons, lats, d1_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=norm1)
         ax1.text(0.4, 0.4, empty_message, transform=ax1.transAxes, bbox=props)
 
-        img2 = ax2.contourf(lons, lats, d2_cyclic, transform=ccrs.PlateCarree(), transform_first=True, colors="w", norm=norm1)
+        img2 = ax2.contourf(lons, lats, d2_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=norm1)
         ax2.text(0.4, 0.4, empty_message, transform=ax2.transAxes, bbox=props)
     else:
-        img1 = ax1.contourf(lons, lats, d1_cyclic, transform=ccrs.PlateCarree(), transform_first=True, cmap=cmap1, norm=norm1, levels=levels1)
-        img2 = ax2.contourf(lons, lats, d2_cyclic, transform=ccrs.PlateCarree(), transform_first=True, cmap=cmap1, norm=norm1, levels=levels1)
+        img1 = ax1.contourf(lons, lats, d1_cyclic, transform=ccrs.PlateCarree(), cmap=cmap1, norm=norm1, levels=levels1)
+        img2 = ax2.contourf(lons, lats, d2_cyclic, transform=ccrs.PlateCarree(), cmap=cmap1, norm=norm1, levels=levels1)
 
     if len(levs_diff) < 2:
-        img3 = ax3.contourf(lons, lats, dif_cyclic, transform=ccrs.PlateCarree(), transform_first=True, colors="w", norm=dnorm)
+        img3 = ax3.contourf(lons, lats, dif_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=dnorm)
         ax3.text(0.4, 0.4, empty_message, transform=ax3.transAxes, bbox=props)
     else:
-        img3 = ax3.contourf(lons, lats, dif_cyclic, transform=ccrs.PlateCarree(), transform_first=True, cmap=cmapdiff, norm=dnorm, levels=levelsdiff)
+        img3 = ax3.contourf(lons, lats, dif_cyclic, transform=ccrs.PlateCarree(), cmap=cmapdiff, norm=dnorm, levels=levelsdiff)
 
     #Set Main title for subplots:
     st = fig.suptitle(wks.stem[:-5].replace("_"," - "), fontsize=18)
@@ -1716,6 +1754,7 @@ def prep_contour_plot(adata, bdata, diffdata, **kwargs):
         - 'subplots_opt': mpl kwargs for subplots
         - 'contourf_opt': mpl kwargs for contourf
         - 'colorbar_opt': mpl kwargs for colorbar
+        - 'diff_colorbar_opt' : mpl kwargs for difference colorbar
         - 'normdiff': color normalization for difference panel
         - 'cmapdiff': colormap for difference panel
         - 'levelsdiff': contour levels for difference panel
@@ -1739,16 +1778,28 @@ def prep_contour_plot(adata, bdata, diffdata, **kwargs):
 
     if 'contour_levels' in kwargs:
         levels1 = kwargs['contour_levels']
-        norm1 = mpl.colors.Normalize(vmin=min(levels1), vmax=max(levels1))
+        if ('non_linear' in kwargs) and (kwargs['non_linear']):
+            cmap_obj = cm.get_cmap(cmap1)
+            norm1 = mpl.colors.BoundaryNorm(levels1, cmap_obj.N)
+        else:
+            norm1 = mpl.colors.Normalize(vmin=min(levels1), vmax=max(levels1))
     elif 'contour_levels_range' in kwargs:
         assert len(kwargs['contour_levels_range']) == 3, \
         "contour_levels_range must have exactly three entries: min, max, step"
 
         levels1 = np.arange(*kwargs['contour_levels_range'])
-        norm1 = mpl.colors.Normalize(vmin=min(levels1), vmax=max(levels1))
+        if ('non_linear' in kwargs) and (kwargs['non_linear']):
+            cmap_obj = cm.get_cmap(cmap1)
+            norm1 = mpl.colors.BoundaryNorm(levels1, cmap_obj.N)
+        else:
+            norm1 = mpl.colors.Normalize(vmin=min(levels1), vmax=max(levels1))
     else:
         levels1 = np.linspace(minval, maxval, 12)
-        norm1 = mpl.colors.Normalize(vmin=minval, vmax=maxval)
+        if ('non_linear' in kwargs) and (kwargs['non_linear']):
+            cmap_obj = cm.get_cmap(cmap1)
+            norm1 = mpl.colors.BoundaryNorm(levels1, cmap_obj.N)
+        else:
+            norm1 = mpl.colors.Normalize(vmin=minval, vmax=maxval)
     #End if
 
     #Check if the minval and maxval are actually different.  If not,
@@ -1803,16 +1854,19 @@ def prep_contour_plot(adata, bdata, diffdata, **kwargs):
     subplots_opt = {}
     contourf_opt = {}
     colorbar_opt = {}
+    diff_colorbar_opt = {}
 
     # extract any MPL kwargs that should be passed on:
     if 'mpl' in kwargs:
         subplots_opt.update(kwargs['mpl'].get('subplots',{}))
         contourf_opt.update(kwargs['mpl'].get('contourf',{}))
         colorbar_opt.update(kwargs['mpl'].get('colorbar',{}))
+        diff_colorbar_opt.update(kwargs['mpl'].get('diff_colorbar',{}))
     #End if
     return {'subplots_opt': subplots_opt,
             'contourf_opt': contourf_opt,
             'colorbar_opt': colorbar_opt,
+            'diff_colorbar_opt': diff_colorbar_opt,
             'normdiff': normdiff,
             'cmapdiff': cmapdiff,
             'levelsdiff': levelsdiff,
@@ -1899,7 +1953,6 @@ def plot_zonal_mean_and_save(wks, case_nickname, base_nickname,
 
         levs_diff = np.unique(np.array(cp_info['levelsdiff']))
 
-
         if len(levs) < 2:
             img0, ax[0] = zonal_plot(adata['lat'], azm, ax=ax[0])
             ax[0].text(0.4, 0.4, empty_message, transform=ax[0].transAxes, bbox=props)
@@ -1917,7 +1970,7 @@ def plot_zonal_mean_and_save(wks, case_nickname, base_nickname,
             ax[2].text(0.4, 0.4, empty_message, transform=ax[2].transAxes, bbox=props)
         else:
             img2, ax[2] = zonal_plot(adata['lat'], diff, ax=ax[2], norm=cp_info['normdiff'],cmap=cp_info['cmapdiff'],levels=cp_info['levelsdiff'],**cp_info['contourf_opt'])
-            fig.colorbar(img2, ax=ax[2], location='right',**cp_info['colorbar_opt'])
+            fig.colorbar(img2, ax=ax[2], location='right',**cp_info['diff_colorbar_opt'])
 
         ax[0].set_title(case_title, loc='left', fontsize=tiFontSize)
         ax[1].set_title(base_title, loc='left', fontsize=tiFontSize)

lib/test/unit_tests/test_adf_config.py

@@ -61,7 +61,7 @@ def test_AdfConfig_create(self):
 
         obs_data_loc = adf_test.read_config_var("obs_data_loc", conf_dict=basic_diag_dict)
 
-        self.assertEqual(obs_data_loc, "/glade/work/nusbaume/SE_projects/model_diagnostics/ADF_obs")
+        self.assertEqual(obs_data_loc, "/glade/campaign/cgd/amp/amwg/ADF_obs")
 
     #####