Skip to content

Commit

Permalink
zppy-interfaces refactor (#642)
Browse files Browse the repository at this point in the history
Refactor zppy to use zppy-interfaces
  • Loading branch information
forsyth2 authored Nov 22, 2024
1 parent 408e9ad commit 6561a45
Show file tree
Hide file tree
Showing 62 changed files with 1,807 additions and 2,895 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
# Does not run the integration tests, which require server access
- name: Run Unit Tests
run: |
python -m unittest tests/test_*.py
pytest tests/test_*.py
# If the branch updates documentation, then the docs will need to be updated.
publish-docs:
Expand Down
1 change: 0 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,4 @@
"python.linting.flake8Enabled": true,
"python.linting.flake8Args": ["--config=setup.cfg"],
"python.linting.mypyEnabled": true,
"python.pythonPath": "/opt/miniconda3/envs/zstash_dev/bin/python"
}
35 changes: 23 additions & 12 deletions conda/dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,32 @@ channels:
- conda-forge
- defaults
dependencies:
# Base
# =================
# Build
# =======================
- python=3.9.13
- pip=22.2.2
# Base
# =================
- configobj=5.0.6
- jinja2=3.1.2
- mache>=1.5.0
- mpas_tools>=0.15.0
- pillow=9.2.0
# Developer Tools
# Testing
# =======================
- pytest
- pytest-cov
# Documentation
# If versions are updated, also update in `.github/workflows/build_workflow.yml`
# =================
- sphinx=5.2.3
- sphinx_rtd_theme=1.0.0
- sphinx-multiversion=0.2.4
# Need to pin docutils because 0.17 has a bug with unordered lists
# https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
- docutils=0.16
# Quality Assurance Tools
# =======================
# If versions are updated, also update 'rev' in `.pre-commit-config.yaml`
- black=24.10.0 # version from https://anaconda.org/conda-forge/black
- flake8=7.1.1 # version from https://anaconda.org/conda-forge/flake8
Expand All @@ -22,13 +37,9 @@ dependencies:
- mypy=1.11.2 # version from https://anaconda.org/conda-forge/mypy
- pre-commit=4.0.1 # version from https://anaconda.org/conda-forge/pre-commit
- tbump=6.9.0
# Documentation
# If versions are updated, also update in `.github/workflows/build_workflow.yml`
# =================
- sphinx=5.2.3
- sphinx-multiversion=0.2.4
- sphinx_rtd_theme=1.0.0
# Need to pin docutils because 0.17 has a bug with unordered lists
# https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
- docutils=0.16
# Developer Tools
# =======================
- tbump=6.9.0
- ipykernel

prefix: /opt/miniconda3/envs/zppy_dev
2 changes: 1 addition & 1 deletion docs/source/post.rrm_simulation.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ years = "1:60:20",
[[conus_0.23x0.31_aave]]
grid = 'conus_0.23x0.31_aave'
sets = 'lat_lon',
cfg = "e3sm_diags/lat_lon_conus.cfg"
cfg = "inclusions/e3sm_diags/lat_lon_conus.cfg" # Built-in cfg for users to use

[mpas_analysis]
active = False
Expand Down
2 changes: 2 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ ignore =
E501
# line break before operator
W503
# comparison to False should be 'if cond is False:' or 'if not cond:'
E712
# Max width of Github code review is 119 characters
max-line-length = 119
max-complexity = 18
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ years = "1985:1995:5"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ walltime = "00:30:00"

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ years = "1985:1995:5",

[global_time_series]
active = True
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
experiment_name = "v3.LR.historical_0051"
figstr = "v3.LR.historical_0051"
plots_original=""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ walltime = "00:30:00"
[global_time_series]
active = True
climo_years = "1985-1989", "1990-1995",
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
experiment_name = "v3.LR.historical_0051"
figstr = "v3.LR.historical_0051"
moc_file=mocTimeSeries_1985-1995.nc
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ walltime = "00:30:00"

[global_time_series]
active = True
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
experiment_name = "v3.LR.historical_0051"
figstr = "v3.LR.historical_0051"
plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ years = "1985:1989:2",

[e3sm_diags]
active = True
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
ref_final_yr = 1989
ref_start_yr = 1985
Expand Down Expand Up @@ -141,6 +141,7 @@ years = "1985:1989:2",
[global_time_series]
active = True
bundle = "bundle2"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
experiment_name = "v3.LR.historical_0051"
figstr = "v3.LR.historical_0051"
plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ walltime = "00:30:00"
active = True
climo_diurnal_frequency = "diurnal_8xdaily"
climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down Expand Up @@ -168,6 +168,7 @@ walltime = "00:30:00"
[global_time_series]
active = True
climo_years ="1980-1984", "1985-1990",
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
experiment_name = "v2.LR.historical_0201"
figstr = "v2.LR.historical_0201"
moc_file=mocTimeSeries_1980-1990.nc
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ active = True
climo_diurnal_frequency = "diurnal_8xdaily"
climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
climo_subsection = "atm_monthly_180x360_aave"
environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015"
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
Expand Down Expand Up @@ -196,6 +196,7 @@ walltime = "00:30:00"
[global_time_series]
active = True
climo_years = "1985-1989", "1990-1995",
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
experiment_name = "v3.LR.historical_0051"
figstr = "v3.LR.historical_0051"
moc_file=mocTimeSeries_1985-1995.nc
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ years = "1985:1995:5",

[global_time_series]
active = True
environment_commands = "#expand global_time_series_environment_commands#"
experiment_name = "#expand case_name#"
figstr = "#expand case_name#"
plots_original=""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ walltime = "#expand mpas_analysis_walltime#"
[global_time_series]
active = True
climo_years = "1985-1989", "1990-1995",
environment_commands = "#expand global_time_series_environment_commands#"
experiment_name = "#expand case_name#"
figstr = "#expand case_name#"
moc_file=mocTimeSeries_1985-1995.nc
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ walltime = "00:30:00"

[global_time_series]
active = True
environment_commands = "#expand global_time_series_environment_commands#"
experiment_name = "#expand case_name#"
figstr = "#expand case_name#"
plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
Expand Down
1 change: 1 addition & 0 deletions tests/integration/template_weekly_bundles.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ years = "1985:1989:2",
[global_time_series]
active = True
bundle = "bundle2"
environment_commands = "#expand global_time_series_environment_commands#"
experiment_name = "#expand case_name#"
figstr = "#expand case_name#"
plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
Expand Down
1 change: 1 addition & 0 deletions tests/integration/template_weekly_comprehensive_v2.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ walltime = "#expand mpas_analysis_walltime#"
[global_time_series]
active = True
climo_years ="1980-1984", "1985-1990",
environment_commands = "#expand global_time_series_environment_commands#"
experiment_name = "#expand case_name_v2#"
figstr = "#expand case_name_v2#"
moc_file=mocTimeSeries_1980-1990.nc
Expand Down
1 change: 1 addition & 0 deletions tests/integration/template_weekly_comprehensive_v3.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ walltime = "#expand mpas_analysis_walltime#"
[global_time_series]
active = True
climo_years = "1985-1989", "1990-1995",
environment_commands = "#expand global_time_series_environment_commands#"
experiment_name = "#expand case_name#"
figstr = "#expand case_name#"
moc_file=mocTimeSeries_1985-1995.nc
Expand Down
9 changes: 6 additions & 3 deletions tests/integration/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,12 @@ def get_chyrsalis_expansions(config):
"constraint": "",
# To run this test, replace conda environment with your e3sm_diags dev environment
# To use default environment_commands, set to ""
"diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015",
"diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"diags_walltime": "5:00:00",
"e3sm_to_cmip_environment_commands": "",
"environment_commands_test": "",
"expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
"global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"mpas_analysis_walltime": "00:30:00",
"partition_long": "compute",
"partition_short": "debug",
Expand All @@ -177,11 +178,12 @@ def get_compy_expansions(config):
"constraint": "",
# To run this test, replace conda environment with your e3sm_diags dev environment
# To use default environment_commands, set to ""
"diags_environment_commands": "source /qfs/people/fors729/mambaforge/etc/profile.d/conda.sh; conda activate e3sm_diags_20240328",
"diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"diags_walltime": "03:00:00",
"e3sm_to_cmip_environment_commands": "",
"environment_commands_test": "",
"expected_dir": "/compyfs/www/zppy_test_resources/",
"global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"mpas_analysis_walltime": "00:30:00",
"partition_long": "slurm",
"partition_short": "short",
Expand All @@ -206,11 +208,12 @@ def get_perlmutter_expansions(config):
"constraint": "cpu",
# To run this test, replace conda environment with your e3sm_diags dev environment
# To use default environment_commands, set to ""
"diags_environment_commands": "source /global/homes/f/forsyth/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240429",
"diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"diags_walltime": "6:00:00",
"e3sm_to_cmip_environment_commands": "",
"environment_commands_test": "",
"expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/",
"global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"mpas_analysis_walltime": "01:00:00",
"partition_long": "",
"partition_short": "",
Expand Down
Loading

0 comments on commit 6561a45

Please sign in to comment.