From febf162535f76942da03fde7c2869456866ee1a9 Mon Sep 17 00:00:00 2001 From: mgrover1 Date: Tue, 19 Sep 2023 09:50:51 -0500 Subject: [PATCH] ADD: Add enhanced notebook template --- _toc.yml | 3 + notebooks/notebook-template.ipynb | 380 +++++++++++++++++++++++++++++- 2 files changed, 375 insertions(+), 8 deletions(-) diff --git a/_toc.yml b/_toc.yml index d883d361ee..53f312c49c 100644 --- a/_toc.yml +++ b/_toc.yml @@ -7,6 +7,9 @@ parts: - caption: CROCUS Level 1 Node chapters: - glob: notebooks/crocus_level1_node/* +- caption: Contributing + chapters: + - file: notebooks/notebook-template - caption: Quick Looks (Recent Data) chapters: - glob: notebooks/quicklooks/* diff --git a/notebooks/notebook-template.ipynb b/notebooks/notebook-template.ipynb index 58408f8362..d01feee119 100644 --- a/notebooks/notebook-template.ipynb +++ b/notebooks/notebook-template.ipynb @@ -10,14 +10,14 @@ "\n", "or edit this cell to see raw HTML `img` demonstration. This is preferred if you need to shrink your embedded image. **Either way be sure to include `alt` text for any embedded images to make your content more accessible.**\n", "\n", - "\"Project" + "\"Project" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# ARM Notebook Template\n", + "# CROCUS Notebook Template\n", "\n", "**Note - this is a slightly modified version of the Project Pythia notebook template**\n", "\n", @@ -58,9 +58,9 @@ "\n", "| Concepts | Importance | Notes |\n", "| --- | --- | --- |\n", - "| [Intro to Cartopy](https://foundations.projectpythia.org/core/cartopy/cartopy.html) | Necessary | |\n", - "| [Understanding of NetCDF](https://foundations.projectpythia.org/core/data-formats/netcdf-cf.html) | Helpful | Familiarity with metadata structure |\n", - "| Project management | Helpful | |\n", + "| [Intro to Waggle Sensors](https://github.com/waggle-sensor) | Helpful | Familiarity with Waggle Sensors|\n", + "| [Waggle/Sage Data Client](https://github.com/sagecontinuum/sage-data-client) | Necessary | Accessing Data |\n", + "| [Working with Pandas Dataframes](https://foundations.projectpythia.org/core/pandas.html) | Necessary | Understanding Data Object\n", "\n", "- **Time to learn**: estimate in minutes. For a rough idea, use 5 mins per subsection, 10 if longer; add these up for a total. Safer to round up and overestimate.\n", "- **System requirements**:\n", @@ -86,11 +86,375 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "import sys" + "import sage_data_client" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Access" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namevaluemeta.hostmeta.jobmeta.missingmeta.nodemeta.pluginmeta.sensormeta.taskmeta.unitsmeta.vsnmeta.zone
timestamp
2023-07-02 12:00:00.029156535+00:00wxt.env.humidity100.0000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536percentW08Dcore
2023-07-02 12:00:00.107252119+00:00wxt.env.humidity100.0000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536percentW08Dcore
2023-07-02 12:00:00.206164123+00:00wxt.env.humidity100.0000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536percentW08Dcore
2023-07-02 12:00:00.283311181+00:00wxt.env.humidity100.0000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536percentW08Dcore
2023-07-02 12:00:00.364271449+00:00wxt.env.humidity100.0000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536percentW08Dcore
.......................................
2023-07-02 12:59:59.600882716+00:00wxt.wind.speed1.5000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536meters per secondW08Dcore
2023-07-02 12:59:59.684669667+00:00wxt.wind.speed1.5000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536meters per secondW08Dcore
2023-07-02 12:59:59.760763484+00:00wxt.wind.speed1.5000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536meters per secondW08Dcore
2023-07-02 12:59:59.841247667+00:00wxt.wind.speed1.5000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536meters per secondW08Dcore
2023-07-02 12:59:59.920791677+00:00wxt.wind.speed1.5000048b02d3ae277.ws-nxcorewaggle-wxt536-9999.9000048b02d3ae277registry.sagecontinuum.org/jrobrien/waggle-wxt...vaisala-wxt536waggle-wxt536meters per secondW08Dcore
\n", + "

350864 rows × 12 columns

\n", + "
" + ], + "text/plain": [ + " name value \\\n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 wxt.env.humidity 100.0 \n", + "2023-07-02 12:00:00.107252119+00:00 wxt.env.humidity 100.0 \n", + "2023-07-02 12:00:00.206164123+00:00 wxt.env.humidity 100.0 \n", + "2023-07-02 12:00:00.283311181+00:00 wxt.env.humidity 100.0 \n", + "2023-07-02 12:00:00.364271449+00:00 wxt.env.humidity 100.0 \n", + "... ... ... \n", + "2023-07-02 12:59:59.600882716+00:00 wxt.wind.speed 1.5 \n", + "2023-07-02 12:59:59.684669667+00:00 wxt.wind.speed 1.5 \n", + "2023-07-02 12:59:59.760763484+00:00 wxt.wind.speed 1.5 \n", + "2023-07-02 12:59:59.841247667+00:00 wxt.wind.speed 1.5 \n", + "2023-07-02 12:59:59.920791677+00:00 wxt.wind.speed 1.5 \n", + "\n", + " meta.host \\\n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:00:00.107252119+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:00:00.206164123+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:00:00.283311181+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:00:00.364271449+00:00 000048b02d3ae277.ws-nxcore \n", + "... ... \n", + "2023-07-02 12:59:59.600882716+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:59:59.684669667+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:59:59.760763484+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:59:59.841247667+00:00 000048b02d3ae277.ws-nxcore \n", + "2023-07-02 12:59:59.920791677+00:00 000048b02d3ae277.ws-nxcore \n", + "\n", + " meta.job meta.missing \\\n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:00:00.107252119+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:00:00.206164123+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:00:00.283311181+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:00:00.364271449+00:00 waggle-wxt536 -9999.9 \n", + "... ... ... \n", + "2023-07-02 12:59:59.600882716+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:59:59.684669667+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:59:59.760763484+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:59:59.841247667+00:00 waggle-wxt536 -9999.9 \n", + "2023-07-02 12:59:59.920791677+00:00 waggle-wxt536 -9999.9 \n", + "\n", + " meta.node \\\n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 000048b02d3ae277 \n", + "2023-07-02 12:00:00.107252119+00:00 000048b02d3ae277 \n", + "2023-07-02 12:00:00.206164123+00:00 000048b02d3ae277 \n", + "2023-07-02 12:00:00.283311181+00:00 000048b02d3ae277 \n", + "2023-07-02 12:00:00.364271449+00:00 000048b02d3ae277 \n", + "... ... \n", + "2023-07-02 12:59:59.600882716+00:00 000048b02d3ae277 \n", + "2023-07-02 12:59:59.684669667+00:00 000048b02d3ae277 \n", + "2023-07-02 12:59:59.760763484+00:00 000048b02d3ae277 \n", + "2023-07-02 12:59:59.841247667+00:00 000048b02d3ae277 \n", + "2023-07-02 12:59:59.920791677+00:00 000048b02d3ae277 \n", + "\n", + " meta.plugin \\\n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:00:00.107252119+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:00:00.206164123+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:00:00.283311181+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:00:00.364271449+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "... ... \n", + "2023-07-02 12:59:59.600882716+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:59:59.684669667+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:59:59.760763484+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:59:59.841247667+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "2023-07-02 12:59:59.920791677+00:00 registry.sagecontinuum.org/jrobrien/waggle-wxt... \n", + "\n", + " meta.sensor meta.task \\\n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:00:00.107252119+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:00:00.206164123+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:00:00.283311181+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:00:00.364271449+00:00 vaisala-wxt536 waggle-wxt536 \n", + "... ... ... \n", + "2023-07-02 12:59:59.600882716+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:59:59.684669667+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:59:59.760763484+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:59:59.841247667+00:00 vaisala-wxt536 waggle-wxt536 \n", + "2023-07-02 12:59:59.920791677+00:00 vaisala-wxt536 waggle-wxt536 \n", + "\n", + " meta.units meta.vsn meta.zone \n", + "timestamp \n", + "2023-07-02 12:00:00.029156535+00:00 percent W08D core \n", + "2023-07-02 12:00:00.107252119+00:00 percent W08D core \n", + "2023-07-02 12:00:00.206164123+00:00 percent W08D core \n", + "2023-07-02 12:00:00.283311181+00:00 percent W08D core \n", + "2023-07-02 12:00:00.364271449+00:00 percent W08D core \n", + "... ... ... ... \n", + "2023-07-02 12:59:59.600882716+00:00 meters per second W08D core \n", + "2023-07-02 12:59:59.684669667+00:00 meters per second W08D core \n", + "2023-07-02 12:59:59.760763484+00:00 meters per second W08D core \n", + "2023-07-02 12:59:59.841247667+00:00 meters per second W08D core \n", + "2023-07-02 12:59:59.920791677+00:00 meters per second W08D core \n", + "\n", + "[350864 rows x 12 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Look at the July 2, 2023 case between 12 and 13 UTC (7 to 8 AM local time)\n", + "start = \"2023-07-02T12:00:00Z\"\n", + "end = \"2023-07-02T13:00:00Z\"\n", + "\n", + "# Filter for a single node - in this case, the Northeastern Illinois University node\n", + "node = \"W08D\"\n", + "\n", + "# Search for the weather data, collected by the wxt sensor\n", + "sensor = \"vaisala-wxt536\"\n", + "\n", + "df = sage_data_client.query(start=start,\n", + " end=end, \n", + " filter={\n", + " \"vsn\": node,\n", + " \"sensor\": sensor\n", + " }\n", + ")\n", + "\n", + "# Set the timestamp as the index\n", + "df = df.set_index(\"timestamp\")\n", + "df" ] }, { @@ -297,7 +661,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.11.4" }, "nbdime-conflicts": { "local_diff": [