diff --git a/README.md b/README.md index 18d5477..c1ee190 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,13 @@ After the last step is finished, you can use Jupyter to view generated notebooks or you can copy the entire `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run/_build/html` directory to your local machine and look at `index.html` in a web browser. -For users running on the NCAR super computers (derecho or casper), you can visualize the web page in a browser with the following steps: -1. open a new browser window that points to https://fastx.ucar.edu:3300/session/ -1. open a default desktop icon -1. select the browser client -1. type `xterm` and hit enter to open a terminal -1. In the terminal, run `cd ${CUPID_ROOT}/examples/coupled_model/computed_notebooks/quick-run/_build/html` to enter the `html` directory -1. From the updated directory, run `firefox index.html &` to open a web browser pointed at the generated web page +### Looking at Output + +For users running on the NCAR super computers (derecho or casper), you can visualize the web page in a browser using the FastX service. FastX requires you to be on the internal NCAR network (either on-site or via the VPN, and can be accessed via the following steps: + +1. Open a new browser window that points to https://fastx.ucar.edu:3300/session/ +1. Open a default desktop icon. +1. Select the browser client. +1. Type `xterm` and hit enter to open a terminal. +1. In the terminal, run `cd ${CUPID_ROOT}/examples/coupled_model/computed_notebooks/quick-run/_build/html` to enter the `html` directory. +1. From the updated directory, run `firefox index.html &` to open a web browser pointed at the generated web page. \ No newline at end of file diff --git a/cupid/run.py b/cupid/run.py index d823453..b263cd5 100755 --- a/cupid/run.py +++ b/cupid/run.py @@ -71,7 +71,7 @@ def run(): ##################################################################### # Ploomber - making a DAG - dag = ploomber.DAG(executor=ploomber.executors.Parallel()) + dag = ploomber.DAG(executor=ploomber.executors.Serial()) ##################################################################### diff --git a/examples/adf-only/config-adf-only.yml b/examples/adf-only/config-adf-only.yml deleted file mode 100644 index 4bfce06..0000000 --- a/examples/adf-only/config-adf-only.yml +++ /dev/null @@ -1,118 +0,0 @@ - -################## SETUP ################## - -################ -# Data Sources # -################ -data_sources: - # sname is any string used as a nickname for this configuration. It will be - ### used as the name of the folder your computed notebooks are put in - sname: &sname - adf-quick-run - - # run_dir is the path to the folder you want - ### all the files associated with this configuration - ### to be created in - run_dir: . - - # nb_path_root is the path to the folder that cupid will - ### look for your template notebooks in. It doesn't have to - ### be inside run_dir, or be specific to this project, as - ### long as the notebooks are there - nb_path_root: ../nblibrary - -###################### -# Computation Config # -###################### - -computation_config: - - # default_kernel_name is the name of the environment that - ### the notebooks in this configuration will be run in by default. - ### It must already be installed on your machine. You can also - ### specify a different environment than the default for any - ### notebook in NOTEBOOK CONFIG - - default_kernel_name: npl-2023a - - -############# NOTEBOOK CONFIG ############# - -############################ -# Notebooks and Parameters # -############################ - -# All parameters under global_params get passed to all the notebooks - -global_params: - test_global_param: hello - -compute_notebooks: - - # This is where all the notebooks you want run and their - ### parameters are specified. Several examples of different - ### types of notebooks are provided. - - # The first key (here simple_no_params_nb) is the name of the - ### notebook from nb_path_root, minus the .ipynb - index: - parameter_groups: - none: - sname: *sname - - adf_quick_run: - parameter_groups: - none: - sname: *sname - adf_path: ../../externals/ADF - config_path: . - config_fil_str: "config_f.cam6_3_119.FLTHIST_ne30.r328_gamma0.33_soae.001.yaml" - - - - - -########### JUPYTER BOOK CONFIG ########### - -################################## -# Jupyter Book Table of Contents # -################################## -book_toc: - - # See https://jupyterbook.org/en/stable/structure/configure.html for - ## complete documentation of Jupyter book construction options - - format: jb-book - - # All filenames are notebook filename without the .ipynb, similar to above - - root: index # root is the notebook that will be the homepage for the book - parts: - - # Parts group notebooks into different sections in the Jupyter book - ### table of contents, so you can organize different parts of your project. - - - caption: ADF Quick Run - - # Each chapter is the name of one of the notebooks that you executed - ### in compute_notebooks above, also without .ipynb - chapters: - - file: adf_quick_run - - -##################################### -# Keys for Jupyter Book _config.yml # -##################################### -book_config_keys: - - title: Example project # Title of your jupyter book - - # Other keys can be added here, see https://jupyterbook.org/en/stable/customize/config.html - ### for many more options - - - - - - - diff --git a/examples/coupled_model/config.yml b/examples/coupled_model/config.yml index f160f55..b7c37d2 100644 --- a/examples/coupled_model/config.yml +++ b/examples/coupled_model/config.yml @@ -7,8 +7,7 @@ data_sources: # sname is any string used as a nickname for this configuration. It will be ### used as the name of the folder your computed notebooks are put in - sname: &sname - quick-run + sname: quick-run # run_dir is the path to the folder you want ### all the files associated with this configuration @@ -45,7 +44,7 @@ computation_config: # All parameters under global_params get passed to all the notebooks global_params: - test_global_param: hello + CESM_output_dir: /glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing compute_notebooks: @@ -57,27 +56,39 @@ compute_notebooks: ### notebook from nb_path_root, minus the .ipynb index: parameter_groups: - none: - sname: *sname + none: {} adf_quick_run: parameter_groups: none: - sname: *sname adf_path: ../../externals/ADF config_path: . config_fil_str: "config_f.cam6_3_119.FLTHIST_ne30.r328_gamma0.33_soae.001.yaml" - surface: + ocean_surface: parameter_groups: none: - sname: *sname + Case: b.e23_alpha16b.BLT1850.ne30_t232.054 + savefigs: False + mom6_tools_config: + start_date: '0091-01-01' + end_date: '0101-01-01' + Fnames: + native: 'mom6.h.native.????-??.nc' + static: 'mom6.h.static.nc' + oce_cat: /glade/u/home/gmarques/libs/oce-catalogs/reference-datasets.yml + - land_comparision: + land_comparison: parameter_groups: none: - sname: *sname + cases: + - ctsm51d159_f45_GSWP3_bgccrop_1850pAD + - ctsm51d159_f45_GSWP3_bgccrop_1850pSASU + type: + - 1850pAD + - 1850pSASU ########### JUPYTER BOOK CONFIG ########### @@ -109,11 +120,11 @@ book_toc: - caption: Ocean chapters: - - file: surface + - file: ocean_surface - caption: Land chapters: - - file: land_comparision + - file: land_comparison ##################################### # Keys for Jupyter Book _config.yml # diff --git a/examples/nblibrary/adf_quick_run.ipynb b/examples/nblibrary/adf_quick_run.ipynb index ba9816a..93c28d5 100644 --- a/examples/nblibrary/adf_quick_run.ipynb +++ b/examples/nblibrary/adf_quick_run.ipynb @@ -788,7 +788,6 @@ "import matplotlib.pyplot as plt\n", "import xarray as xr\n", "import numpy as np\n", - "import matplotlib.ticker as ticker\n", "from matplotlib.ticker import MultipleLocator\n", "from matplotlib.lines import Line2D\n", "\n", diff --git a/examples/nblibrary/diag_config.yml b/examples/nblibrary/diag_config.yml deleted file mode 100644 index 8596d44..0000000 --- a/examples/nblibrary/diag_config.yml +++ /dev/null @@ -1,50 +0,0 @@ -# case info -Case: - CASEROOT: /glade/p/cesmdata/cseg/runs/cesm2_0/b.e23_alpha16b.BLT1850.ne30_t232.054/ - CIMEROOT: /glade/work/hannay/cesm_tags/cesm2_3_alpha16g/cime/ - OCN_DIAG_ROOT: ncfiles/ - SNAME: diagtestcase - -# start and end dates to perform averages -Avg: - end_date: '0101-01-01' - start_date: '0091-01-01' - -# cesm hist fname conventions -Fnames: - rho2: '.mom6.h.rho2.????-??.nc' - z: '.mom6.h.z.????-??.nc' - native: '.mom6.h.native.????-??.nc' - sfc: '.mom6.h.sfc.????-??.nc' - static: '.mom6.h.static.nc' - -# sections where transports are computed online -# Format: file name stream (e.g., Drake_Passage), transport -# component (umo or vmo), observational estimate (optional) -Transports: - sections : {'h.Agulhas_Section': ['umo', 129.8, 143.6], - 'h.Barents_Opening': ['vmo', 2.0], - 'h.Bering_Strait': ['vmo', 0.7, 1.1], - 'h.Davis_Strait': ['vmo', -2.1,-1.1], - 'h.Denmark_Strait': ['vmo',-4.8,-2.0], - 'h.Drake_Passage': ['umo',129.8,143.6], - 'h.English_Channel': ['umo',0.01,0.1], - 'h.Fram_Strait': ['vmo',-4.7,0.7], - 'h.Florida_Bahamas': ['vmo',28.9,34.3], - 'h.Florida_Bahamas_extended':['vmo',28.9,34.3], - 'h.Florida_Cuba':['umo', None], - 'h.Gibraltar_Strait':['umo',-1.0, 1.0], - 'h.Iceland_Norway':['vmo',28.9,34.3], - 'h.Indonesian_Throughflow':['vmo',-15.,-13.], - 'h.Mozambique_Channel':['vmo',-25.6,-7.8], - 'h.Pacific_undercurrent':['umo',24.5,28.3], - 'h.Taiwan_Luzon':['umo',-3.0,-1.8], - 'h.Windward_Passage':['vmo',-15.,5.], - 'h.Robeson_Channel':['umo', None], - 'h.Yucatan_Channel':['vmo', None], - 'h.Bosporus_Strait':['vmo', None], - } - -# path to ocean-related datasets catalog -oce_cat: /glade/u/home/gmarques/libs/oce-catalogs/reference-datasets.yml - diff --git a/examples/nblibrary/land_comparision.ipynb b/examples/nblibrary/land_comparison.ipynb similarity index 79% rename from examples/nblibrary/land_comparision.ipynb rename to examples/nblibrary/land_comparison.ipynb index 9801544..af87618 100644 --- a/examples/nblibrary/land_comparision.ipynb +++ b/examples/nblibrary/land_comparison.ipynb @@ -1,19 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "e926996f", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "# add default values for parameters here" - ] - }, { "cell_type": "markdown", "id": "8c041f27-4806-4e23-8b88-8f7db7d80934", @@ -22,7 +8,6 @@ "## Simple example comparing land variables from two simulations\n", "Created by wwieder@ucar.edu, Jan 2024\n", "\n", - "- paths are harded coded for now\n", "- quickly (and inaccurately) calculates 5 year mean from raw, .h0., files.\n", "- plots global means and differences which also not very nice to look at, especially since this points to a 4x5 grid." ] @@ -37,32 +22,36 @@ "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "import os\n", - "import time\n", - "import datetime\n", - "import calendar\n", - "import cftime\n", - "\n", "from glob import glob\n", "from os.path import join\n", "\n", - "import dask\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", "import xarray as xr\n", "\n", - "import matplotlib\n", "import matplotlib.pyplot as plt\n", - "import matplotlib.dates as mdates\n", - "from matplotlib.pyplot import cm\n", - "import cartopy.crs as ccrs\n", - "\n", - "from distributed import wait\n", "\n", "%matplotlib inline" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "4f50b897-d424-4642-9885-4563bfd9ce31", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "CESM_output_dir = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing\"\n", + "type = ['1850pAD','1850pSASU']\n", + "cases = ['ctsm51d159_f45_GSWP3_bgccrop_1850pAD', 'ctsm51d159_f45_GSWP3_bgccrop_1850pSASU']" + ] + }, { "cell_type": "code", "execution_count": null, @@ -81,19 +70,6 @@ "\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "6355b17a-9c19-4427-ac56-79f75756a9f2", - "metadata": {}, - "outputs": [], - "source": [ - "case = 'ctsm51d159_f45_GSWP3_bgccrop_'\n", - "type = ['1850pAD','1850pSASU']\n", - "cases = [case+'1850pAD',case+'1850pSASU']\n", - "cases" - ] - }, { "cell_type": "code", "execution_count": null, @@ -104,15 +80,14 @@ "for c in range(len(cases)):\n", "\n", " sim_files =[]\n", - " sim_path = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing/\"\n", - " sim_path = sim_path+cases[c]+\"/lnd/hist/\"\n", - " sim_files.extend(sorted(glob(join(sim_path+cases[c]+\".clm2.h0.*.nc\"))))\n", + " sim_path = f\"{CESM_output_dir}/{cases[c]}/lnd/hist\"\n", + " sim_files.extend(sorted(glob(join(f\"{sim_path}/{cases[c]}.clm2.h0.*.nc\"))))\n", " # subset last 5 years of data \n", " sim_files = sim_files[-60:None]\n", - " print(\"All simulation files for \"+cases[c]+\": [\", len(sim_files), \"files]\")\n", + " print(f\"All simulation files for {cases[c]}: [{len(sim_files)} files]\")\n", "\n", " temp = xr.open_mfdataset(sim_files, decode_times=True, combine='by_coords',\n", - " parallel=True,preprocess=preprocess).mean('time')\n", + " parallel=False, preprocess=preprocess).mean('time')\n", " \n", " if c == 0:\n", " ds = temp\n", @@ -216,9 +191,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:miniconda3-cupid-analysis]", + "display_name": "Python [conda env:cupid-analysis]", "language": "python", - "name": "conda-env-miniconda3-cupid-analysis-py" + "name": "conda-env-cupid-analysis-py" }, "language_info": { "codemirror_mode": { diff --git a/examples/nblibrary/surface.ipynb b/examples/nblibrary/ocean_surface.ipynb similarity index 72% rename from examples/nblibrary/surface.ipynb rename to examples/nblibrary/ocean_surface.ipynb index 910303b..d90225b 100644 --- a/examples/nblibrary/surface.ipynb +++ b/examples/nblibrary/ocean_surface.ipynb @@ -38,46 +38,43 @@ "outputs": [], "source": [ "import xarray as xr\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import warnings, os, yaml, argparse\n", - "import pandas as pd\n", - "import dask, intake\n", - "from datetime import datetime, date\n", - "from ncar_jobqueue import NCARCluster\n", - "from dask.distributed import Client\n", - "from mom6_tools.DiagsCase import DiagsCase\n", - "from mom6_tools.m6toolbox import add_global_attrs\n", - "from mom6_tools.m6plot import xycompare, xyplot\n", + "import warnings, os\n", + "import intake\n", + "from datetime import datetime\n", "from mom6_tools.MOM6grid import MOM6grid\n", - "from mom6_tools.surface import get_SSH, get_MLD, get_BLD\n", + "from mom6_tools.surface import get_MLD, get_BLD\n", "\n", "warnings.filterwarnings(\"ignore\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "CESM_output_dir = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing\"\n", + "Case = \"b.e23_alpha16b.BLT1850.ne30_t232.054\"\n", + "savefigs = False\n", + "mom6_tools_config = {}" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# Read in the yaml file\n", - "diag_config_yml_path = \"diag_config.yml\"\n", - "diag_config_yml = yaml.load(open(diag_config_yml_path,'r'), Loader=yaml.Loader)\n", - "\n", - "# load avg dates\n", - "avg = diag_config_yml['Avg']\n", - "\n", - "# Create the case instance\n", - "dcase = DiagsCase(diag_config_yml['Case'])\n", - "DOUT_S = dcase.get_value('DOUT_S')\n", - "if DOUT_S:\n", - " OUTDIR = dcase.get_value('DOUT_S_ROOT')+'/ocn/hist/'\n", - "else:\n", - " OUTDIR = dcase.get_value('RUNDIR')+'/'\n", - "\n", - "print('Output directory is:', OUTDIR)\n", - "print('Casename is:', dcase.casename)" + "OUTDIR = f'{CESM_output_dir}/{Case}/ocn/hist/'\n", + "print('Output directory is:', OUTDIR)" ] }, { @@ -97,14 +94,13 @@ "class args:\n", " pass\n", "\n", - "args.start_date = avg['start_date']\n", - "args.end_date = avg['end_date']\n", - "args.casename = dcase.casename\n", - "args.native = dcase.casename+diag_config_yml['Fnames']['native']\n", - "args.static = dcase.casename+diag_config_yml['Fnames']['static']\n", + "args.start_date = mom6_tools_config['start_date']\n", + "args.end_date = mom6_tools_config['end_date']\n", + "args.casename = Case\n", + "args.native = f\"{Case}.{mom6_tools_config['Fnames']['native']}\"\n", + "args.static = f\"{Case}.{mom6_tools_config['Fnames']['static']}\"\n", "args.mld_obs = \"mld-deboyer-tx2_3v2\"\n", - "args.savefigs = False\n", - "args.nw = 6 # requesting 6 workers" + "args.savefigs = savefigs" ] }, { @@ -124,30 +120,6 @@ " os.system('mkdir ncfiles') " ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "parallel = False\n", - "if args.nw > 1:\n", - " parallel = True\n", - " cluster = NCARCluster(interface='ext')\n", - " cluster.scale(args.nw)\n", - " client = Client(cluster)\n", - " client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client" - ] - }, { "cell_type": "code", "execution_count": null, @@ -176,7 +148,7 @@ " ds[v] = xr.zeros_like(ds.SSH)\n", " return ds[variables]\n", "\n", - "ds1 = xr.open_mfdataset(OUTDIR+args.native, parallel=parallel)\n", + "ds1 = xr.open_mfdataset(OUTDIR+args.native, parallel=False)\n", "ds = preprocess(ds1)\n", "\n", "print('Time elasped: ', datetime.now() - startTime)" @@ -198,7 +170,7 @@ "metadata": {}, "outputs": [], "source": [ - "catalog = intake.open_catalog(diag_config_yml['oce_cat'])\n", + "catalog = intake.open_catalog(mom6_tools_config['oce_cat'])\n", "mld_obs = catalog[args.mld_obs].to_dask()\n", "# uncomment to list all datasets available\n", "#list(catalog)" @@ -247,17 +219,6 @@ "# SSH (not working)\n", "#get_SSH(ds, 'SSH', grd, args)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if parallel:\n", - " print('\\n Releasing workers...')\n", - " client.close(); cluster.close()" - ] } ], "metadata": {