Testing updates (#554)

* Testing updates

* Change unique id

* Restore unique_id

tests/integration/generated/directions_chrysalis.md

@@ -12,6 +12,25 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration
 can not be done from two different branches simultaneously
 (since files in the `zppy` directory rather than an external directory get changed).
 
+## Debug your code with test_debug
+
+Follow the directions in the cfg to modify it for your testing needs.
+
+```
+rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/<UNIQUE ID>/v2.LR.historical_0201
+rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/<UNIQUE ID>/v2.LR.historical_0201/post
+# Generate cfg
+python tests/integration/utils.py
+
+# Run jobs:
+zppy -c tests/integration/generated/test_debug_chrysalis.cfg
+# After they finish, check the results:
+cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
+grep -v "OK" *status
+# Nothing should print
+cd -
+```
+
 ### test_bundles
 
 ```
@@ -87,7 +106,7 @@ mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundl
 cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
+rm -rf image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_bundles.txt
 cd <top level of zppy repo>
@@ -119,7 +138,7 @@ rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
+rm -rf image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_complete_run.txt
 cd <top level of zppy repo>

tests/integration/generated/test_debug_chrysalis.cfg

@@ -0,0 +1,56 @@
+# Use this cfg to test a pull request before merging.
+# 1. Copy over the contents of template_complete_run.cfg.
+# 2. Delete parts that aren't needed. `output` and `www` should include `zppy_test_debug` rather than `zppy_test_complete_run`.
+# 3. Iteratively debug/develop with this cfg.
+# 4. Once it's working as expected, copy important changes to template_complete_run.cfg.
+
+[default]
+case = v2.LR.historical_0201
+constraint = ""
+dry_run = "False"
+environment_commands = ""
+input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201"
+partition = "debug"
+qos = "regular"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id"
+
+[ts]
+active = True
+walltime = "00:30:00"
+years = "1850:1854:2",
+
+  [[ atm_monthly_180x360_aave ]]
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  ts_fmt = "cmip"
+
+  [[ atm_daily_180x360_aave ]]
+  frequency = "daily"
+  input_files = "eam.h1"
+  input_subdir = "archive/atm/hist"
+  vars = "PRECT"
+
+  [[ land_monthly ]]
+  extra_vars = "landfrac"
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  #vars = "LAISHA,LAISUN"
+  vars = "FSH,RH2M"
+  ts_fmt = "cmip"
+
+[ilamb]
+active = True
+grids = '180x360_aave'
+nodes = 8
+partition = "compute"
+short_name = 'v2.LR.historical_0201'
+ts_num_years = 2
+years = "1850:1854:2",
+
+  [[ land_monthly ]]

tests/integration/template_debug.cfg

@@ -0,0 +1,56 @@
+# Use this cfg to test a pull request before merging.
+# 1. Copy over the contents of template_complete_run.cfg.
+# 2. Delete parts that aren't needed. `output` and `www` should include `zppy_test_debug` rather than `zppy_test_complete_run`.
+# 3. Iteratively debug/develop with this cfg.
+# 4. Once it's working as expected, copy important changes to template_complete_run.cfg.
+
+[default]
+case = v2.LR.historical_0201
+constraint = "#expand constraint#"
+dry_run = "#expand dry_run#"
+environment_commands = "#expand environment_commands#"
+input = "#expand user_input#/E3SMv2/v2.LR.historical_0201"
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
+output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201"
+partition = "#expand partition_short#"
+qos = "#expand qos_short#"
+www = "#expand user_www#zppy_debug_www/#expand unique_id#"
+
+[ts]
+active = True
+walltime = "00:30:00"
+years = "1850:1854:2",
+
+  [[ atm_monthly_180x360_aave ]]
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  ts_fmt = "cmip"
+
+  [[ atm_daily_180x360_aave ]]
+  frequency = "daily"
+  input_files = "eam.h1"
+  input_subdir = "archive/atm/hist"
+  vars = "PRECT"
+
+  [[ land_monthly ]]
+  extra_vars = "landfrac"
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  #vars = "LAISHA,LAISUN"
+  vars = "FSH,RH2M"
+  ts_fmt = "cmip"
+
+[ilamb]
+active = True
+grids = '180x360_aave'
+nodes = 8
+partition = "#expand partition_long#"
+short_name = 'v2.LR.historical_0201'
+ts_num_years = 2
+years = "1850:1854:2",
+
+  [[ land_monthly ]]

tests/integration/template_directions.md

@@ -12,6 +12,25 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration
 can not be done from two different branches simultaneously
 (since files in the `zppy` directory rather than an external directory get changed).
 
+## Debug your code with test_debug
+
+Follow the directions in the cfg to modify it for your testing needs.
+
+```
+rm -rf #expand user_www#zppy_test_debug_www/<UNIQUE ID>/v2.LR.historical_0201
+rm -rf #expand user_output#zppy_test_debug_output/<UNIQUE ID>/v2.LR.historical_0201/post
+# Generate cfg
+python tests/integration/utils.py
+
+# Run jobs:
+zppy -c tests/integration/generated/test_debug_#expand machine#.cfg
+# After they finish, check the results:
+cd #expand user_output#zppy_test_debug_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
+grep -v "OK" *status
+# Nothing should print
+cd -
+```
+
 ### test_bundles
 
 ```
@@ -87,7 +106,7 @@ mkdir -p #expand expected_dir#expected_bundles/bundle_files
 cp -r #expand user_output#zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts/bundle*.bash #expand expected_dir#expected_bundles/bundle_files
 cd #expand expected_dir#expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf #expand user_www#forsyth/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
+rm -rf image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_bundles.txt
 cd <top level of zppy repo>
@@ -119,7 +138,7 @@ rm -rf #expand expected_dir#expected_complete_run
 cp -r #expand user_www#zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201 #expand expected_dir#expected_complete_run
 cd #expand expected_dir#expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf #expand user_www#forsyth/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
+rm -rf image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_complete_run.txt
 cd <top level of zppy repo>

tests/integration/test_bundles.py

@@ -60,19 +60,19 @@ def test_bundles_bash_file_content(self):
         # Check that bundle files are correct
         self.assertEqual(
             os.system(
-                f"diff -bu {actual_directory}/bundle1.bash {expected_directory}/bundle1.bash"
+                f"diff -bu -I 'zppy_test_bundles_output/' {actual_directory}/bundle1.bash {expected_directory}/bundle1.bash"
             ),
             0,
         )
         self.assertEqual(
             os.system(
-                f"diff -bu {actual_directory}/bundle2.bash {expected_directory}/bundle2.bash"
+                f"diff -bu -I 'zppy_test_bundles_output/' {actual_directory}/bundle2.bash {expected_directory}/bundle2.bash"
             ),
             0,
         )
         self.assertEqual(
             os.system(
-                f"diff -bu {actual_directory}/bundle3.bash {expected_directory}/bundle3.bash"
+                f"diff -bu -I 'zppy_test_bundles_output/' {actual_directory}/bundle3.bash {expected_directory}/bundle3.bash"
             ),
             0,
         )

tests/integration/utils.py

@@ -257,7 +257,7 @@ def generate_cfgs(unified_testing=False, dry_run=False):
     else:
         expansions["dry_run"] = "False"
 
-    cfg_names = ["bundles", "complete_run"]
+    cfg_names = ["bundles", "complete_run", "debug"]
     for cfg_name in cfg_names:
         cfg_template = f"{git_top_level}/tests/integration/template_{cfg_name}.cfg"
         cfg_generated = (