Add unique id to avoid overwrites (#527)

* Add unique id to avoid overwrites

* Perlmutter docs

* Compy docs

* Chrysalis docs

docs/source/dev_guide/release_testing.rst

@@ -60,6 +60,7 @@ Testing directions for making a release
     For a, b, and c, always:
 
     * Update ``diags_environment_commands`` for the machine you're currently working on to use the environment created in the above step. If this is not done, the ``atm_monthly_180x360_aave_environment_commands`` subtask will use an older dev environment for E3SM Diags than we want.
+    * Update ``UNIQUE_ID`` to be a short description of what you're testing.
 
 6. Set up your environment.
 

tests/integration/generated/directions_chrysalis.md

@@ -2,48 +2,63 @@
 
 ## Commands to run before running integration tests
 
+Replace `<UNIQUE ID>` with a short description of what you're testing.
+Usually, this will be an issue number (e.g., "issue-123").
+This allows you to simultaneously run jobs
+launched from `zppy` on different branches,
+without worrying about overwriting results.
+
+NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration/test_*.py`)
+can not be done from two different branches simultaneously
+(since files in the `zppy` directory rather than an external directory get changed).
+
 ### test_bundles
 
 ```
-rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201
-rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post
+rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201
+rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post
 # Generate cfg
 python tests/integration/utils.py
 
 # Run first set of jobs:
 zppy -c tests/integration/generated/test_bundles_chrysalis.cfg
 # bundle1 and bundle2 should run. After they finish, check the results:
-cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts
+cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
 grep -v "OK" *status
 # Nothing should print
+cd -
 
 # Now, invoke zppy again to run jobs that needed to wait for dependencies:
 zppy -c tests/integration/generated/test_bundles_chrysalis.cfg
 # bundle3 and ilamb should run. After they finish, check the results:
-cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts
+cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
 grep -v "OK" *status
 # Nothing should print
+cd -
 
-# If a final release has just been made, run:
+# If a final release has just been made,
+# run the following to move the previous expected results elsewhere.
 cp /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles expected_bundles_v<version>
 ```
 
 ### test_complete_run
 
 ```
-rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201
-rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post
+rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201
+rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/<UNIQUE ID>/v2.LR.historical_0201/post
 # Generate cfg
 python tests/integration/utils.py
 
 # Run jobs:
 zppy -c tests/integration/generated/test_complete_run_chrysalis.cfg
 # After they finish, check the results:
-cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/scripts
+cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
 grep -v "OK" *status
 # Nothing should print
+cd -
 
-# If a final release has just been made, run:
+# If a final release has just been made,
+# run the following to move the previous expected results elsewhere.
 cp /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run expected_complete_run_v<version>
 ```
 
@@ -67,12 +82,12 @@ python -u -m unittest tests/integration/test_bash_generation.py
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
-cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_bundles_www/v2.LR.historical_0201/image_check_failures
+rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_bundles.txt
 cd <top level of zppy repo>
@@ -101,10 +116,10 @@ mv test_campaign_high_res_v1_output/post/scripts/*.settings /lcrc/group/e3sm/pub
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_complete_run_www/v2.LR.historical_0201/image_check_failures
+rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_complete_run.txt
 cd <top level of zppy repo>
@@ -129,13 +144,13 @@ python -u -m unittest tests/integration/test_defaults.py
 ### test_bundles
 
 ```
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles_unified_<#>
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles_unified_<#>
 mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles_unified_<#>/bundle_files
-cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles_unified_<#>/bundle_files
+cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles_unified_<#>/bundle_files
 ```
 
 ### test_complete_run
 
 ```
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run_unified_<#>
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run_unified_<#>
 ```

tests/integration/generated/directions_compy.md

@@ -2,48 +2,63 @@
 
 ## Commands to run before running integration tests
 
+Replace `<UNIQUE ID>` with a short description of what you're testing.
+Usually, this will be an issue number (e.g., "issue-123").
+This allows you to simultaneously run jobs
+launched from `zppy` on different branches,
+without worrying about overwriting results.
+
+NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration/test_*.py`)
+can not be done from two different branches simultaneously
+(since files in the `zppy` directory rather than an external directory get changed).
+
 ### test_bundles
 
 ```
-rm -rf /compyfs/www/fors729/zppy_test_bundles_www/v2.LR.historical_0201
-rm -rf /compyfs/fors729/zppy_test_bundles_output/v2.LR.historical_0201/post
+rm -rf /compyfs/www/fors729/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201
+rm -rf /compyfs/fors729/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post
 # Generate cfg
 python tests/integration/utils.py
 
 # Run first set of jobs:
 zppy -c tests/integration/generated/test_bundles_compy.cfg
 # bundle1 and bundle2 should run. After they finish, check the results:
-cd /compyfs/fors729/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts
+cd /compyfs/fors729/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
 grep -v "OK" *status
 # Nothing should print
+cd -
 
 # Now, invoke zppy again to run jobs that needed to wait for dependencies:
 zppy -c tests/integration/generated/test_bundles_compy.cfg
 # bundle3 and ilamb should run. After they finish, check the results:
-cd /compyfs/fors729/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts
+cd /compyfs/fors729/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
 grep -v "OK" *status
 # Nothing should print
+cd -
 
-# If a final release has just been made, run:
+# If a final release has just been made,
+# run the following to move the previous expected results elsewhere.
 cp /compyfs/www/zppy_test_resources/expected_bundles expected_bundles_v<version>
 ```
 
 ### test_complete_run
 
 ```
-rm -rf /compyfs/www/fors729/zppy_test_complete_run_www/v2.LR.historical_0201
-rm -rf /compyfs/fors729/zppy_test_complete_run_output/v2.LR.historical_0201/post
+rm -rf /compyfs/www/fors729/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201
+rm -rf /compyfs/fors729/zppy_test_complete_run_output/<UNIQUE ID>/v2.LR.historical_0201/post
 # Generate cfg
 python tests/integration/utils.py
 
 # Run jobs:
 zppy -c tests/integration/generated/test_complete_run_compy.cfg
 # After they finish, check the results:
-cd /compyfs/fors729/zppy_test_complete_run_output/v2.LR.historical_0201/post/scripts
+cd /compyfs/fors729/zppy_test_complete_run_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts
 grep -v "OK" *status
 # Nothing should print
+cd -
 
-# If a final release has just been made, run:
+# If a final release has just been made,
+# run the following to move the previous expected results elsewhere.
 cp /compyfs/www/zppy_test_resources/expected_complete_run expected_complete_run_v<version>
 ```
 
@@ -67,12 +82,12 @@ python -u -m unittest tests/integration/test_bash_generation.py
 rm -rf /compyfs/www/zppy_test_resources/expected_bundles
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /compyfs/www/fors729/zppy_test_bundles_www/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_bundles
+cp -r /compyfs/www/fors729/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_bundles
 mkdir -p /compyfs/www/zppy_test_resources/expected_bundles/bundle_files
-cp -r /compyfs/fors729/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts/bundle*.bash /compyfs/www/zppy_test_resources/expected_bundles/bundle_files
+cp -r /compyfs/fors729/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts/bundle*.bash /compyfs/www/zppy_test_resources/expected_bundles/bundle_files
 cd /compyfs/www/zppy_test_resources/expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf /compyfs/www/fors729/forsyth/zppy_test_bundles_www/v2.LR.historical_0201/image_check_failures
+rm -rf /compyfs/www/fors729/forsyth/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_bundles.txt
 cd <top level of zppy repo>
@@ -101,10 +116,10 @@ mv test_campaign_high_res_v1_output/post/scripts/*.settings /compyfs/www/zppy_te
 rm -rf /compyfs/www/zppy_test_resources/expected_complete_run
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /compyfs/www/fors729/zppy_test_complete_run_www/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_complete_run
+cp -r /compyfs/www/fors729/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_complete_run
 cd /compyfs/www/zppy_test_resources/expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.
-rm -rf /compyfs/www/fors729/forsyth/zppy_test_complete_run_www/v2.LR.historical_0201/image_check_failures
+rm -rf /compyfs/www/fors729/forsyth/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201/image_check_failures
 # This file will list all the expected images.
 find . -type f -name '*.png' > ../image_list_expected_complete_run.txt
 cd <top level of zppy repo>
@@ -129,13 +144,13 @@ python -u -m unittest tests/integration/test_defaults.py
 ### test_bundles
 
 ```
-cp -r /compyfs/www/fors729/zppy_test_bundles_www/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_bundles_unified_<#>
+cp -r /compyfs/www/fors729/zppy_test_bundles_www/<UNIQUE ID>/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_bundles_unified_<#>
 mkdir -p /compyfs/www/zppy_test_resources/expected_bundles_unified_<#>/bundle_files
-cp -r /compyfs/fors729/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts/bundle*.bash /compyfs/www/zppy_test_resources/expected_bundles_unified_<#>/bundle_files
+cp -r /compyfs/fors729/zppy_test_bundles_output/<UNIQUE ID>/v2.LR.historical_0201/post/scripts/bundle*.bash /compyfs/www/zppy_test_resources/expected_bundles_unified_<#>/bundle_files
 ```
 
 ### test_complete_run
 
 ```
-cp -r /compyfs/www/fors729/zppy_test_complete_run_www/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_complete_run_unified_<#>
+cp -r /compyfs/www/fors729/zppy_test_complete_run_www/<UNIQUE ID>/v2.LR.historical_0201 /compyfs/www/zppy_test_resources/expected_complete_run_unified_<#>
 ```