types without xml

setup.py

@@ -30,10 +30,5 @@ def package_files(directory, prefixes, extensions):
     python_requires=">=3.6",
     packages=find_packages(include=["zppy", "zppy.*"]),
     package_data={"": data_files},
-    entry_points={
-        "console_scripts": [
-            "zppy=zppy.__main__:main",
-            "zppy_cdscan_replacement=zppy.__main__:zppy_cdscan_replacement",
-        ]
-    },
+    entry_points={"console_scripts": ["zppy=zppy.__main__:main"]},
 )

tests/integration/generated/test_bundles_chrysalis.cfg

@@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201"
 partition = "compute"
 qos = "regular"
 walltime = "07:00:00"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346"
 
 [bundle]
 

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346"
 
 [climo]
 active = True
@@ -134,7 +134,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "v2.LR.historical_0201"
   swap_test_ref = False
@@ -155,7 +155,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
   run_type = "model_vs_model"
   sets = "lat_lon_land",
   short_ref_name = "same simulation"

tests/integration/generated/test_debug_chrysalis.cfg

@@ -12,46 +12,53 @@ environment_commands = ""
 input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/test-346"
 
 [ts]
 active = True
+e3sm_to_cmip_environment_commands = ""
 walltime = "00:30:00"
 years = "1850:1854:2",
 
-  [[ atm_monthly_180x360_aave ]]
+  [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
+  mapping_file = "glb"
+  years = "1850:1860:5",
 
-  [[ atm_daily_180x360_aave ]]
-  frequency = "daily"
-  input_files = "eam.h1"
-  input_subdir = "archive/atm/hist"
-  vars = "PRECT"
-
-  [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = ""
-  extra_vars = "landfrac"
+  [[ lnd_monthly_glb ]]
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
   vars = "LAISHA,LAISUN"
-  #vars = "FSH,RH2M"
-  ts_fmt = "cmip"
+  years = "1850:1860:5",
 
-[ilamb]
+[mpas_analysis]
 active = True
-grid = '180x360_aave'
-nodes = 8
+anomalyRefYear = 1850
+climo_years ="1850-1854", "1855-1860",
+enso_years = "1850-1854", "1855-1860",
+mesh = "EC30to60E2r2"
+parallelTaskCount = 6
 partition = "compute"
-short_name = 'v2.LR.historical_0201'
-ts_num_years = 2
-years = "1850:1854:2",
+qos = "regular"
+ts_years = "1850-1854", "1850-1860",
+walltime = "00:30:00"
 
-  [[ land_monthly ]]
+[global_time_series]
+active = True
+climo_years ="1850-1854", "1855-1860",
+experiment_name = "v2.LR.historical_0201"
+figstr = "v2_historical_0201"
+moc_file=mocTimeSeries_1850-1860.nc
+plots_lnd = "LAISHA,LAISUN"
+ts_num_years = 5
+ts_years = "1850-1854", "1850-1860",
+walltime = "00:30:00"
+years = "1850-1860",

tests/integration/generated/update_bundles_expected_files_chrysalis.sh

@@ -6,9 +6,9 @@
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
-cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
 zppy_top_level=$(pwd)
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.

tests/integration/generated/update_complete_run_expected_files_chrysalis.sh

@@ -6,7 +6,7 @@
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 zppy_top_level=$(pwd)
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.

tests/integration/template_debug.cfg

@@ -12,46 +12,53 @@ environment_commands = "#expand environment_commands#"
 input = "#expand user_input#/E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
 output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201"
 partition = "#expand partition_short#"
 qos = "#expand qos_short#"
-www = "#expand user_www#zppy_debug_www/#expand unique_id#"
+www = "#expand user_www#zppy_test_debug_www/#expand unique_id#"
 
 [ts]
 active = True
+e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
 walltime = "00:30:00"
 years = "1850:1854:2",
 
-  [[ atm_monthly_180x360_aave ]]
+  [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
+  mapping_file = "glb"
+  years = "1850:1860:5",
 
-  [[ atm_daily_180x360_aave ]]
-  frequency = "daily"
-  input_files = "eam.h1"
-  input_subdir = "archive/atm/hist"
-  vars = "PRECT"
-
-  [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
-  extra_vars = "landfrac"
+  [[ lnd_monthly_glb ]]
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
   vars = "LAISHA,LAISUN"
-  #vars = "FSH,RH2M"
-  ts_fmt = "cmip"
+  years = "1850:1860:5",
 
-[ilamb]
+[mpas_analysis]
 active = True
-grid = '180x360_aave'
-nodes = 8
+anomalyRefYear = 1850
+climo_years ="1850-1854", "1855-1860",
+enso_years = "1850-1854", "1855-1860",
+mesh = "EC30to60E2r2"
+parallelTaskCount = 6
 partition = "#expand partition_long#"
-short_name = 'v2.LR.historical_0201'
-ts_num_years = 2
-years = "1850:1854:2",
+qos = "#expand qos_long#"
+ts_years = "1850-1854", "1850-1860",
+walltime = "#expand mpas_analysis_walltime#"
 
-  [[ land_monthly ]]
+[global_time_series]
+active = True
+climo_years ="1850-1854", "1855-1860",
+experiment_name = "v2.LR.historical_0201"
+figstr = "v2_historical_0201"
+moc_file=mocTimeSeries_1850-1860.nc
+plots_lnd = "LAISHA,LAISUN"
+ts_num_years = 5
+ts_years = "1850-1854", "1850-1860",
+walltime = "00:30:00"
+years = "1850-1860",

tests/integration/utils.py

@@ -7,7 +7,7 @@
 from mache import MachineInfo
 from PIL import Image, ImageChops, ImageDraw
 
-UNIQUE_ID = "unique_id"
+UNIQUE_ID = "test-346"
 
 # Image checking ##########################################################
 

zppy/__main__.py

@@ -3,10 +3,8 @@
 import importlib
 import io
 import os
-import sys
 from typing import List
 
-import pandas
 from configobj import ConfigObj
 from mache import MachineInfo
 from validate import Validator
@@ -248,17 +246,3 @@ def _validate_config(config):
         )
     else:
         print("Configuration file validation passed.")
-
-
-def zppy_cdscan_replacement():
-    output_file = sys.argv[1]
-    input_files = sys.argv[2:]
-    if len(input_files) == 1:
-        files = []
-        # Convert text file of files to a list of files.
-        with open(input_files[0]) as f:
-            for line in f:
-                files.append(line.strip())
-        input_files = files
-    dataframe = pandas.read_table(input_files)
-    dataframe.to_xml(output_file)

zppy/templates/coupled_global.py

@@ -740,32 +740,22 @@ def run(parameters, rgn):  # noqa: C901
         {
             "atmos": None
             if not use_atmos
-            else "{}/post/atm/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/atm/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "ice": None
             if not plots_ice
-            else "{}/post/ice/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/ice/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "land": None
             if not plots_lnd
-            else "{}/post/lnd/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/lnd/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "ocean": None
             if not use_ocn
-            else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "moc": None
             if not use_ocn
             else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "vol": None
             if not use_ocn
-            else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format(
-                case_dir, ts_num_years
-            ),
+            else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
             "name": experiment_name,
             "yoffset": 0.0,
             "yr": ([year1, year2],),

zppy/templates/e3sm_diags.bash

@@ -108,7 +108,7 @@ create_links_ts()
     # xml file will cover the whole period from year1 to year2
     xml_name=${v}_${begin_year}01_${end_year}12.xml
     export CDMS_NO_MPI=true
-    zppy_cdscan_replacement ${xml_name} ${v}_files.txt
+    cdscan ${xml_name} ${v}_files.txt
     if [ $? != 0 ]; then
       cd {{ scriptDir }}
       echo "ERROR (${error_num})" > {{ prefix }}.status
@@ -129,7 +129,7 @@ create_links_ts_rof()
   cd ${ts_rof_dir_destination}
   v="RIVER_DISCHARGE_OVER_LAND_LIQ"
   xml_name=${v}_${begin_year}01_${end_year}12.xml
-  zppy_cdscan_replacement ${xml_name} ${ts_rof_dir_source}/${v}_*.nc
+  cdscan ${xml_name} ${ts_rof_dir_source}/${v}_*.nc
   if [ $? != 0 ]; then
     cd {{ scriptDir }}
     echo "ERROR (${error_num})" > {{ prefix }}.status

zppy/templates/global_time_series.bash

@@ -39,28 +39,7 @@ global_ts_dir={{ global_time_series_dir }}
 export CDMS_NO_MPI=true
 
 use_atm={{ use_atm }}
-if [[ ${use_atm,,} == "true" ]]; then
-    echo 'Create xml files for atm'
-    cd ${case_dir}/post/atm/glb/ts/monthly/${ts_num_years}yr
-    zppy_cdscan_replacement glb.xml *.nc
-    if [ $? != 0 ]; then
-      cd {{ scriptDir }}
-      echo 'ERROR (1)' > {{ prefix }}.status
-      exit 1
-    fi
-fi
-
 use_lnd={{ use_lnd }}
-if [[ ${use_lnd,,} == "true" ]]; then
-    echo 'Create xml files for lnd'
-    cd ${case_dir}/post/lnd/glb/ts/monthly/${ts_num_years}yr
-    zppy_cdscan_replacement -x glb.xml *.nc
-    if [ $? != 0 ]; then
-      cd {{ scriptDir }}
-      echo 'ERROR (2)' > {{ prefix }}.status
-      exit 2
-    fi
-fi
 
 use_ocn={{ use_ocn }}
 if [[ ${use_ocn,,} == "true" ]]; then
@@ -75,16 +54,6 @@ if [[ ${use_ocn,,} == "true" ]]; then
       exit 3
     fi
 
-    echo 'Create xml for for ocn'
-    export CDMS_NO_MPI=true
-    cd ${case_dir}/post/ocn/glb/ts/monthly/${ts_num_years}yr
-    zppy_cdscan_replacement glb.xml mpaso.glb*.nc
-    if [ $? != 0 ]; then
-      cd {{ scriptDir }}
-      echo 'ERROR (4)' > {{ prefix }}.status
-      exit 4
-    fi
-
     echo 'Copy moc file'
     cd ${case_dir}/post/analysis/mpas_analysis/cache/timeseries/moc
     cp ${moc_file} ../../../../../ocn/glb/ts/monthly/${ts_num_years}yr/