Replace CDAT

tests/integration/generated/test_bundles_chrysalis.cfg

@@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201"
 partition = "compute"
 qos = "regular"
 walltime = "07:00:00"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346"
 
 [bundle]
 

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346"
 
 [climo]
 active = True
@@ -134,7 +134,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "v2.LR.historical_0201"
   swap_test_ref = False
@@ -155,7 +155,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
   run_type = "model_vs_model"
   sets = "lat_lon_land",
   short_ref_name = "same simulation"

tests/integration/generated/test_debug_chrysalis.cfg

@@ -12,46 +12,53 @@ environment_commands = ""
 input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/test-346"
 
 [ts]
 active = True
+e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 years = "1850:1854:2",
 
-  [[ atm_monthly_180x360_aave ]]
+  [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
+  mapping_file = "glb"
+  years = "1850:1860:5",
 
-  [[ atm_daily_180x360_aave ]]
-  frequency = "daily"
-  input_files = "eam.h1"
-  input_subdir = "archive/atm/hist"
-  vars = "PRECT"
-
-  [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = ""
-  extra_vars = "landfrac"
+  [[ lnd_monthly_glb ]]
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
   vars = "LAISHA,LAISUN"
-  #vars = "FSH,RH2M"
-  ts_fmt = "cmip"
+  years = "1850:1860:5",
 
-[ilamb]
+[mpas_analysis]
 active = True
-grid = '180x360_aave'
-nodes = 8
+anomalyRefYear = 1850
+climo_years ="1850-1854", "1855-1860",
+enso_years = "1850-1854", "1855-1860",
+mesh = "EC30to60E2r2"
+parallelTaskCount = 6
 partition = "compute"
-short_name = 'v2.LR.historical_0201'
-ts_num_years = 2
-years = "1850:1854:2",
+qos = "regular"
+ts_years = "1850-1854", "1850-1860",
+walltime = "00:30:00"
 
-  [[ land_monthly ]]
+[global_time_series]
+active = True
+climo_years ="1850-1854", "1855-1860",
+experiment_name = "v2.LR.historical_0201"
+figstr = "v2_historical_0201"
+moc_file=mocTimeSeries_1850-1860.nc
+plots_lnd = "LAISHA,LAISUN"
+ts_num_years = 5
+ts_years = "1850-1854", "1850-1860",
+walltime = "00:30:00"
+years = "1850-1860",

tests/integration/generated/update_bundles_expected_files_chrysalis.sh

@@ -6,9 +6,9 @@
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
-cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
 zppy_top_level=$(pwd)
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.

tests/integration/generated/update_complete_run_expected_files_chrysalis.sh

@@ -6,7 +6,7 @@
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 zppy_top_level=$(pwd)
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.

tests/integration/template_debug.cfg

@@ -12,46 +12,53 @@ environment_commands = "#expand environment_commands#"
 input = "#expand user_input#/E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
 output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201"
 partition = "#expand partition_short#"
 qos = "#expand qos_short#"
-www = "#expand user_www#zppy_debug_www/#expand unique_id#"
+www = "#expand user_www#zppy_test_debug_www/#expand unique_id#"
 
 [ts]
 active = True
+e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
 walltime = "00:30:00"
 years = "1850:1854:2",
 
-  [[ atm_monthly_180x360_aave ]]
+  [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
+  mapping_file = "glb"
+  years = "1850:1860:5",
 
-  [[ atm_daily_180x360_aave ]]
-  frequency = "daily"
-  input_files = "eam.h1"
-  input_subdir = "archive/atm/hist"
-  vars = "PRECT"
-
-  [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
-  extra_vars = "landfrac"
+  [[ lnd_monthly_glb ]]
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
   vars = "LAISHA,LAISUN"
-  #vars = "FSH,RH2M"
-  ts_fmt = "cmip"
+  years = "1850:1860:5",
 
-[ilamb]
+[mpas_analysis]
 active = True
-grid = '180x360_aave'
-nodes = 8
+anomalyRefYear = 1850
+climo_years ="1850-1854", "1855-1860",
+enso_years = "1850-1854", "1855-1860",
+mesh = "EC30to60E2r2"
+parallelTaskCount = 6
 partition = "#expand partition_long#"
-short_name = 'v2.LR.historical_0201'
-ts_num_years = 2
-years = "1850:1854:2",
+qos = "#expand qos_long#"
+ts_years = "1850-1854", "1850-1860",
+walltime = "#expand mpas_analysis_walltime#"
 
-  [[ land_monthly ]]
+[global_time_series]
+active = True
+climo_years ="1850-1854", "1855-1860",
+experiment_name = "v2.LR.historical_0201"
+figstr = "v2_historical_0201"
+moc_file=mocTimeSeries_1850-1860.nc
+plots_lnd = "LAISHA,LAISUN"
+ts_num_years = 5
+ts_years = "1850-1854", "1850-1860",
+walltime = "00:30:00"
+years = "1850-1860",

tests/integration/utils.py

@@ -7,7 +7,7 @@
 from mache import MachineInfo
 from PIL import Image, ImageChops, ImageDraw
 
-UNIQUE_ID = "unique_id"
+UNIQUE_ID = "test-346"
 
 # Image checking ##########################################################
 

zppy/templates/coupled_global.py

@@ -3,12 +3,15 @@
 import math
 import sys
 import traceback
-from typing import Any, List, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
+import cftime
 import matplotlib as mpl
 import matplotlib.backends.backend_pdf
 import matplotlib.pyplot as plt
 import numpy as np
+import xarray
+import xcdat  # noqa: F401
 from netCDF4 import Dataset
 from readTS import TS
 
@@ -549,19 +552,28 @@ def param_get_list(param_value):
         return param_value.split(",")
 
 
-def set_var(exp, exp_key, var_list, valid_vars, invalid_vars, rgn):
+def set_var(
+    exp: Dict[str, Any],
+    exp_key: str,
+    var_list: List[str],
+    valid_vars: List[str],
+    invalid_vars: List[str],
+    rgn: str,
+) -> None:
     if exp[exp_key] is not None:
         ts = TS(exp[exp_key])
         for var in var_list:
             try:
+                v: xarray.core.dataarray.DataArray
+                units: Optional[str]
                 v, units = ts.globalAnnual(var)
                 valid_vars.append(str(var))
             except Exception as e:
                 print(e)
                 print(f"globalAnnual failed. Invalid var = {var}")
                 invalid_vars.append(str(var))
                 continue
-            if len(v.shape) > 1:
+            if v.sizes["rgn"] > 1:
                 # number of years x 3 regions = v.shape
                 # 3 regions = global, northern hemisphere, southern hemisphere
                 # We get here if we used the updated `ts` task
@@ -574,18 +586,17 @@ def set_var(exp, exp_key, var_list, valid_vars, invalid_vars, rgn):
                     n = 2
                 else:
                     raise RuntimeError(f"Invalid rgn={rgn}")
-                v = v[:, n]  # Just use nth column
+                v = v.isel(rgn=n)  # Just use nth column
             elif rgn != "glb":
                 # v only has one dimension -- glb.
                 # Therefore it is not possible to get n or s plots.
                 raise RuntimeError(
                     f"var={var} only has global data. Cannot process rgn={rgn}"
                 )
-            exp["annual"][var] = v
             exp["annual"][var] = (v, units)
             if "year" not in exp["annual"]:
-                time = v.getTime()
-                exp["annual"]["year"] = [x.year for x in time.asComponentTime()]
+                years: np.ndarray[cftime.DatetimeNoLeap] = v.coords["time"].values
+                exp["annual"]["year"] = [x.year for x in years]
         del ts
 
 
@@ -736,36 +747,26 @@ def run(parameters, rgn):  # noqa: C901
         or ("change_sea_level" in plots_original)
     )
     use_ocn = plots_ocn or (not atmosphere_only and has_original_ocn_plots)
-    exps = [
+    exps: List[Dict[str, Any]] = [
         {
             "atmos": None
             if not use_atmos
-            else "{}/post/atm/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/atm/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "ice": None
             if not plots_ice
-            else "{}/post/ice/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/ice/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "land": None
             if not plots_lnd
-            else "{}/post/lnd/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/lnd/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "ocean": None
             if not use_ocn
-            else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "moc": None
             if not use_ocn
             else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "vol": None
             if not use_ocn
-            else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "name": experiment_name,
             "yoffset": 0.0,
             "yr": ([year1, year2],),
@@ -777,7 +778,7 @@ def run(parameters, rgn):  # noqa: C901
     invalid_vars: List[str] = []
 
     # Read data
-    exp: Any
+    exp: Dict[str, Any]
     for exp in exps:
         exp["annual"] = {}
 
@@ -855,5 +856,27 @@ def run_by_region(parameters):
         run(parameters, rgn)
 
 
+def debug():
+    args = [
+        "coupled_global.py",
+        "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201",
+        "v2.LR.historical_0201",
+        "v2_historical_0201",
+        "1850",
+        "1860",
+        "Blue",
+        "5",
+        "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+        "False",
+        "None",
+        "None",
+        "LAISHA,LAISUN",
+        "None",
+        "glb,n,s",
+    ]
+    run_by_region(args)
+
+
 if __name__ == "__main__":
     run_by_region(sys.argv)
+    # debug()

zppy/templates/e3sm_diags.bash

@@ -108,7 +108,7 @@ create_links_ts()
     # xml file will cover the whole period from year1 to year2
     xml_name=${v}_${begin_year}01_${end_year}12.xml
     export CDMS_NO_MPI=true
-    cdscan -x ${xml_name} -f ${v}_files.txt
+    cdscan ${xml_name} ${v}_files.txt
     if [ $? != 0 ]; then
       cd {{ scriptDir }}
       echo "ERROR (${error_num})" > {{ prefix }}.status
@@ -129,7 +129,7 @@ create_links_ts_rof()
   cd ${ts_rof_dir_destination}
   v="RIVER_DISCHARGE_OVER_LAND_LIQ"
   xml_name=${v}_${begin_year}01_${end_year}12.xml
-  cdscan -x ${xml_name} ${ts_rof_dir_source}/${v}_*.nc
+  cdscan ${xml_name} ${ts_rof_dir_source}/${v}_*.nc
   if [ $? != 0 ]; then
     cd {{ scriptDir }}
     echo "ERROR (${error_num})" > {{ prefix }}.status