Merge pull request #61 from mnlevy1981/add_LocalCluster

Add parallelization to notebooks

.gitignore

@@ -16,4 +16,5 @@ externals/
 
 # Documentation
 /docs/_build/
-/docs/README.md
+/docs/README.md
+/docs/NCAR_tips.md

NCAR_tips.md

@@ -0,0 +1,42 @@
+# Running CUPiD on NCAR Supercomputers
+
+A few tips and tricks tailored for the CISL's compute environment.
+
+## Running in Parallel
+
+There are two ways to request multiple cores on either casper or derecho.
+Both cases are requesting 12 cores and 120 GB of memory.
+
+
+The recommended approach releases the cores immediately after `cupid-run` finishes:
+
+```
+[login-node] $ conda activate cupid-dev
+(cupid-dev) [login-node] $ qcmd -l select=1:ncpus=12:mem=120GB -- cupid-run config.yml
+```
+
+Alternatively, you can start an interactive session and remain on the compute nodes after `cupid-run` completes:
+
+```
+[login-node] $ qinteractive -l select=1:ncpus=12:mem=120GB
+[compute-node] $ conda activate cupid-dev
+(cupid-dev) [compute-node] $ cupid-run config.yml
+```
+
+Notes:
+1. If you chose to run on derecho, specify the `develop` queue by adding the option `-q develop` to either `qcmd` or `qinteractive`
+   (the `develop` queue is a shared resource and you are charged by the core hour rather than the node hour).
+1. `cupid-build` is not computationally expensive, and can be run on a login node for either machine.
+
+## Looking at Output
+
+You can visualize the web page in a browser using the FastX service.
+FastX requires you to be on the internal NCAR network (either on-site or via the VPN),
+and can be accessed via the following steps:
+
+1. Open a new browser window that points to https://fastx.ucar.edu:3300/session/
+1. Open a default desktop icon.
+1. Select the browser client.
+1. Type `xterm` and hit enter to open a terminal.
+1. In the terminal, run `cd ${CUPID_ROOT}/examples/coupled_model/computed_notebooks/quick-run/_build/html` to enter the `html` directory.
+1. From the updated directory, run `firefox index.html &` to open a web browser pointed at the generated web page.

README.md

@@ -53,27 +53,16 @@ $ pip install -e .  # installs cupid
 
 ## Running
 
-CUPiD currently provides two examples for generating diagnostics.
+CUPiD currently provides an example for generating diagnostics.
 To test the package out, try to run `examples/coupled-model`:
 
 ``` bash
 $ conda activate cupid-dev
 $ cd examples/coupled_model
+$ # machine-dependent: request multiple compute cores
 $ cupid-run config.yml
 $ cupid-build config.yml # Will build HTML from Jupyter Book
 ```
 
 After the last step is finished, you can use Jupyter to view generated notebooks in `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run`
-or you can copy the entire `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run/_build/html`
-directory to your local machine and look at `index.html` in a web browser.
-
-### Looking at Output
-
-For users running on the NCAR super computers (derecho or casper), you can visualize the web page in a browser using the FastX service. FastX requires you to be on the internal NCAR network (either on-site or via the VPN), and can be accessed via the following steps:
-
-1. Open a new browser window that points to https://fastx.ucar.edu:3300/session/
-1. Open a default desktop icon.
-1. Select the browser client.
-1. Type `xterm` and hit enter to open a terminal.
-1. In the terminal, run `cd ${CUPID_ROOT}/examples/coupled_model/computed_notebooks/quick-run/_build/html` to enter the `html` directory.
-1. From the updated directory, run `firefox index.html &` to open a web browser pointed at the generated web page.
+or you can view `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run/_build/html/index.html` in a web browser.

docs/NCARtips.rst

@@ -0,0 +1,2 @@
+.. include:: NCAR_tips.md
+   :parser: myst

docs/conf.py

@@ -18,20 +18,21 @@
 
 print("sys.path:", sys.path)
 
-# Copy README into docs
-# This is to allow us to remove the header image from the docs copy of README
-# without affecting the original README, but still pull the source README
+# Copy README and NCAR_tips into docs
+# This block allows us to remove the header image from any md files
+# without affecting the original version, but still pull the source
 # into the docs build fresh each time.
-os.system('cp ../README.md ./README.md')
+for file in ['README.md', 'NCAR_tips.md']:
+  os.system(f'cp ../{file} ./')
 
-# Remove any images from the first line of the README
-with open('README.md', 'r') as f:
-    readme1 = f.readline()
-    readme1 = re.sub('<img.*?> ', '', readme1)
-    readme = f.read()
+  # Remove any images from the first line of the file
+  with open(file, 'r') as f:
+      file1 = f.readline()
+      file1 = re.sub('<img.*?> ', '', file1)
+      file_rest = f.read()
 
-with open('README.md', 'w') as f:
-    f.write(readme1+readme)
+  with open(file, 'w') as f:
+      f.write(file1+file_rest)
 
 # -- Project information -----------------------------------------------------
 
@@ -65,7 +66,7 @@
 
 intersphinx_mapping = {
     'dask': ('https://docs.dask.org/en/latest/', None),
-    'python': ('http://docs.python.org/3/', None),
+    'python': ('https://docs.python.org/3/', None),
     'numpy': ("https://numpy.org/doc/stable", None),
     'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None),
     'xarray': ('http://xarray.pydata.org/en/stable/', None),

docs/index.rst

@@ -6,6 +6,7 @@ CUPiD Documentation
    :hidden:
 
     Adding Notebooks <addingnotebookstocollection.md>
+    Tips and Tricks for NCAR Machines <NCARtips.rst>
 
 .. include:: README.md
    :parser: myst

examples/coupled_model/config.yml

@@ -45,6 +45,8 @@ computation_config:
 
 global_params:
   CESM_output_dir: /glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing
+  lc_kwargs:
+    threads_per_worker: 1
 
 compute_notebooks:
 

examples/nblibrary/ocean_surface.ipynb

@@ -37,12 +37,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import xarray as xr\n",
-    "import warnings, os\n",
-    "import intake\n",
     "from datetime import datetime\n",
+    "import os\n",
+    "import warnings\n",
+    "\n",
+    "from dask.distributed import Client, LocalCluster\n",
+    "import intake\n",
     "from mom6_tools.MOM6grid import MOM6grid\n",
     "from mom6_tools.surface import get_MLD, get_BLD\n",
+    "import xarray as xr\n",
     "\n",
     "warnings.filterwarnings(\"ignore\")"
    ]
@@ -64,7 +67,8 @@
     "CESM_output_dir = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing\"\n",
     "Case = \"b.e23_alpha16b.BLT1850.ne30_t232.054\"\n",
     "savefigs = False\n",
-    "mom6_tools_config = {}"
+    "mom6_tools_config = {}\n",
+    "lc_kwargs = {}"
    ]
   },
   {
@@ -120,6 +124,19 @@
     "  os.system('mkdir ncfiles')    "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Spin up cluster\n",
+    "cluster = LocalCluster(**lc_kwargs)\n",
+    "client = Client(cluster)\n",
+    "\n",
+    "client"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

examples/nblibrary/seaice.ipynb

@@ -29,9 +29,9 @@
     "from matplotlib.gridspec import GridSpec\n",
     "import cartopy.crs as ccrs\n",
     "import cartopy.feature as cfeature\n",
-    "import cftime\n",
     "import yaml\n",
-    "import pandas as pd"
+    "import pandas as pd\n",
+    "from dask.distributed import Client, LocalCluster"
    ]
   },
   {
@@ -51,6 +51,7 @@
    "source": [
     "CESM_output_dir = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing\"\n",
     "cases = [\"b.e23_alpha16g.BLT1850.ne30_t232.075\",\"b.e23_alpha16g.BLT1850.ne30_t232.078\"]\n",
+    "lc_kwargs = {}\n",
     "\n",
     "begyr1 = 1\n",
     "endyr1 = 102\n",
@@ -59,6 +60,20 @@
     "nyears = 25"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "03782fa0-3224-4c2d-8355-b89d1b5f57c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Spin up cluster\n",
+    "cluster = LocalCluster(**lc_kwargs)\n",
+    "client = Client(cluster)\n",
+    "\n",
+    "client"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -76,8 +91,10 @@
     "cbegyr2 = f\"{begyr2:04d}\"\n",
     "cendyr2 = f\"{endyr2:04d}\"\n",
     "\n",
-    "ds1 = xr.open_mfdataset(CESM_output_dir+\"/\"+case1+\"/ts/\"+case1+\".cice.h.\"+\"*.\"+cbegyr1+\"01-\"+cendyr1+\"12.nc\")\n",
-    "ds2 = xr.open_mfdataset(CESM_output_dir+\"/\"+case2+\"/ts/\"+case2+\".cice.h.\"+\"*.\"+cbegyr2+\"01-\"+cendyr2+\"12.nc\")\n",
+    "ds1 = xr.open_mfdataset(CESM_output_dir+\"/\"+case1+\"/ts/\"+case1+\".cice.h.\"+\"*.\"+cbegyr1+\"01-\"+cendyr1+\"12.nc\",\n",
+    "                       data_vars='minimal', compat='override', coords='minimal')\n",
+    "ds2 = xr.open_mfdataset(CESM_output_dir+\"/\"+case2+\"/ts/\"+case2+\".cice.h.\"+\"*.\"+cbegyr2+\"01-\"+cendyr2+\"12.nc\",\n",
+    "                       data_vars='minimal', compat='override', coords='minimal')\n",
     "\n",
     "TLAT = ds1['TLAT']\n",
     "TLON = ds1['TLON']\n",
@@ -262,7 +279,7 @@
     "                  color = 'black', scale=1.,\n",
     "                  transform=ccrs.PlateCarree())\n",
     "    units = \"cm/s\"\n",
-    "    qk = ax.quiverkey(Q,0.85,0.025,0.10,r'10 '+units,labelpos='S', coordinates='axes',color='black',zorder=2)\n",
+    "    ax.quiverkey(Q,0.85,0.025,0.10,r'10 '+units,labelpos='S', coordinates='axes',color='black',zorder=2)\n",
     "\n",
     "    if (proj == \"N\"):\n",
     "       ax = fig.add_subplot(gs[0,2:], projection=ccrs.NorthPolarStereo())\n",
@@ -293,7 +310,7 @@
     "                  color = 'black', scale=1.,\n",
     "                  transform=ccrs.PlateCarree())\n",
     "    units = \"cm/s\"\n",
-    "    qk = ax.quiverkey(Q,0.85,0.025,0.10,r'10 '+units,labelpos='S', coordinates='axes',color='black',zorder=2)\n",
+    "    ax.quiverkey(Q,0.85,0.025,0.10,r'10 '+units,labelpos='S', coordinates='axes',color='black',zorder=2)\n",
     "\n",
     "    if (proj == \"N\"):\n",
     "       ax = fig.add_subplot(gs[1,1:3], projection=ccrs.NorthPolarStereo())\n",
@@ -324,7 +341,7 @@
     "                  color = 'black', scale=1.,\n",
     "                  transform=ccrs.PlateCarree())\n",
     "    units = \"cm/s\"\n",
-    "    qk = ax.quiverkey(Q,0.85,0.025,0.10,r'10 '+units,labelpos='S', coordinates='axes',color='black',zorder=2)\n",
+    "    ax.quiverkey(Q,0.85,0.025,0.10,r'10 '+units,labelpos='S', coordinates='axes',color='black',zorder=2)\n",
     "\n",
     "    plt.suptitle(\"Velocity m/s\")"
    ]