From 864292da8defe995cd30c060dd2e255e245a257e Mon Sep 17 00:00:00 2001 From: Darren Chaddock Date: Mon, 9 Dec 2024 13:42:56 -0700 Subject: [PATCH] updates for crib sheets --- RELEASE_NOTES.md | 6 + .../data/download_and_read_data.ipynb | 457 +++++++++--------- .../download_skymaps_and_calibrations.ipynb | 211 ++++---- .../explore_datasets_and_observatories.ipynb | 304 ++++++------ poetry.lock | 8 +- pyaurorax/__init__.py | 2 +- pyaurorax/cli/cli.py | 2 +- pyaurorax/pyaurorax.py | 18 +- pyproject.toml | 2 +- tests/test_suite/test_version.py | 2 +- 10 files changed, 485 insertions(+), 527 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 602c507c..352d395f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,9 @@ +Version 1.6.0 +------------------- + +- added dataset_name optional parameter to the `purge_download_output_root_path()` function + + Version 1.5.0 ------------------- diff --git a/examples/notebooks/data/download_and_read_data.ipynb b/examples/notebooks/data/download_and_read_data.ipynb index 5ffa3bc2..c1b037ee 100644 --- a/examples/notebooks/data/download_and_read_data.ipynb +++ b/examples/notebooks/data/download_and_read_data.ipynb @@ -4,49 +4,10 @@ "cell_type": "code", "execution_count": 12, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%html\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, "outputs": [], "source": [ - "import datetime\n", "import pprint\n", + "import datetime\n", "import pyaurorax\n", "\n", "aurorax = pyaurorax.PyAuroraX()" @@ -56,100 +17,25 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Download data" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name Provider Level DOI Details Short Description \n", - "================================================================================================================================================================================================\n", - "REGO_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Flatfield calibration data (IDL save format) \n", - "REGO_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Rayleighs calibration data (IDL save format) \n", - "REGO_DAILY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (JPG format) \n", - "REGO_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PGM format) \n", - "REGO_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PNG format) \n", - "REGO_DAILY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (JPG format) \n", - "REGO_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PGM format) \n", - "REGO_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PNG format) \n", - "REGO_HOURLY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (JPG format) \n", - "REGO_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PGM format) \n", - "REGO_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PNG format) \n", - "REGO_HOURLY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (JPG format) \n", - "REGO_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PGM format) \n", - "REGO_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PNG format) \n", - "REGO_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers 3-sec raw data \n", - "REGO_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers skymap data (IDL save format) \n", - "THEMIS_ASI_DAILY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (JPG format) \n", - "THEMIS_ASI_DAILY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (PGM format) \n", - "THEMIS_ASI_DAILY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (JPG format) \n", - "THEMIS_ASI_DAILY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (PGM format) \n", - "THEMIS_ASI_HOURLY_AVERAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (JPG format) \n", - "THEMIS_ASI_HOURLY_AVERAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (PGM format) \n", - "THEMIS_ASI_HOURLY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (JPG format) \n", - "THEMIS_ASI_HOURLY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (PGM format) \n", - "THEMIS_ASI_HOURLY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (JPG format) \n", - "THEMIS_ASI_HOURLY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (PGM format) \n", - "THEMIS_ASI_RAW UCalgary L0 None THEMIS All Sky Imagers 3-sec raw data \n", - "THEMIS_ASI_RAW_ROW2 UCalgary L0 None THEMIS All Sky Imagers raw row2 data \n", - "THEMIS_ASI_RAW_WIDE UCalgary L0 None THEMIS All Sky Imagers raw wide data \n", - "THEMIS_ASI_SKYMAP_IDLSAV UCalgary L3 None THEMIS All Sky Imagers skymap data (IDL save format) \n", - "THEMIS_ASI_THUMB32 UCalgary L1A None THEMIS All Sky Imagers thumb32 data \n", - "THEMIS_ASI_VEC1024 UCalgary L1A None THEMIS All Sky Imagers vec1024 data \n", - "TREX_BLUE_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Flatfield calibration data (IDL save format)\n", - "TREX_BLUE_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Rayleighs calibration data (IDL save format)\n", - "TREX_BLUE_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PGM format) \n", - "TREX_BLUE_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PNG format) \n", - "TREX_BLUE_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PGM format) \n", - "TREX_BLUE_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PNG format) \n", - "TREX_BLUE_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PGM format) \n", - "TREX_BLUE_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PNG format) \n", - "TREX_BLUE_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PGM format) \n", - "TREX_BLUE_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PNG format) \n", - "TREX_BLUE_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers 3-sec raw data \n", - "TREX_BLUE_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers skymap data (IDL save format) \n", - "TREX_NIR_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Flatfield calibration data (IDL save format) \n", - "TREX_NIR_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Rayleighs calibration data (IDL save format) \n", - "TREX_NIR_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PGM format) \n", - "TREX_NIR_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PNG format) \n", - "TREX_NIR_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PGM format) \n", - "TREX_NIR_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PNG format) \n", - "TREX_NIR_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PGM format) \n", - "TREX_NIR_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PNG format) \n", - "TREX_NIR_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PGM format) \n", - "TREX_NIR_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PNG format) \n", - "TREX_NIR_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers 6-sec raw data \n", - "TREX_NIR_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers skymap data (IDL save format) \n", - "TREX_RGB_DAILY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily keogram summary product \n", - "TREX_RGB_DAILY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily montage summary product \n", - "TREX_RGB_HOURLY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly keogram summary product \n", - "TREX_RGB_HOURLY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly montage summary product \n", - "TREX_RGB_RAW_BURST UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3Hz raw burst data \n", - "TREX_RGB_RAW_NOMINAL UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3-sec raw data \n", - "TREX_RGB_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers skymap data (IDL save format) \n" - ] - } - ], - "source": [ - "# list available datasets\n", - "aurorax.data.list_datasets_in_table()" + "# Downloading data\n", + "\n", + "PyAuroraX allows you to download data for a given dataset, time frame, and optionally the site. A progress bar is shown by default, and it can be disabled or modified using the optional parameters. The output path of the downloaded data can be modified when you initialize the `pyaurorax.PyAuroraX()` object. We show an example of this near the bottom of this crib sheet.\n", + "\n", + "To figure out the dataset name that we want to download data for, we can use the `aurorax.data.list_datasets()` function, or navigate to the [Dataset Descriptions](https://data.phys.ucalgary.ca/about_datasets) page and dive into a particular instrument array page.\n", + "\n", + "Below we are going to download an hour of THEMIS ASI data from the Athabasca, AB, imager. We will use the `THEMIS_ASI_RAW` dataset name, and the `start`, `end`, and `site_uid` parameters to filter further.\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "97a70ae7468f47afaf531d64098ea611", + "model_id": "665d187fd2064f0aa3ad7b2b5edd6bed", "version_major": 2, "version_minor": 0 }, @@ -162,7 +48,7 @@ } ], "source": [ - "# download an hour of THEMIS ASI data\n", + "# download an hour of THEMIS ASI data from Athabasca\n", "dataset_name = \"THEMIS_ASI_RAW\"\n", "start_dt = datetime.datetime(2021, 11, 4, 9, 0)\n", "end_dt = datetime.datetime(2021, 11, 4, 9, 59)\n", @@ -172,99 +58,36 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "FileDownloadResult(filenames=[WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0900_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0901_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0902_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0903_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0904_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0905_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0906_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0907_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0908_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0909_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0910_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0911_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0912_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0913_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0914_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0915_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0916_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0917_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0918_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0919_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0920_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0921_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0922_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0923_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0924_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0925_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0926_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0927_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0928_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0929_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0930_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0931_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0932_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0933_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0934_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0935_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0936_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0937_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0938_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0939_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0940_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0941_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0942_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0943_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0944_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0945_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0946_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0947_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0948_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0949_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0950_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0951_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0952_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0953_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0954_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0955_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0956_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0957_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0958_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0959_atha_themis02_full.pgm.gz')], count=60, total_bytes=127920163, output_root_path='C:\\\\Users\\\\darrenc\\\\pyaurorax_data/THEMIS_ASI_RAW', dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-sec raw data', provider='UCalgary', level='L0', doi_details='None', ...))\n", - "\n", - "{'count': 60,\n", - " 'dataset': Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-sec raw data', provider='UCalgary', level='L0', doi_details='None', ...),\n", - " 'filenames': [WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0900_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0901_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0902_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0903_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0904_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0905_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0906_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0907_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0908_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0909_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0910_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0911_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0912_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0913_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0914_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0915_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0916_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0917_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0918_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0919_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0920_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0921_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0922_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0923_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0924_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0925_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0926_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0927_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0928_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0929_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0930_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0931_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0932_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0933_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0934_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0935_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0936_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0937_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0938_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0939_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0940_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0941_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0942_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0943_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0944_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0945_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0946_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0947_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0948_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0949_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0950_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0951_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0952_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0953_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0954_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0955_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0956_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0957_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0958_atha_themis02_full.pgm.gz'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0959_atha_themis02_full.pgm.gz')],\n", - " 'output_root_path': 'C:\\\\Users\\\\darrenc\\\\pyaurorax_data/THEMIS_ASI_RAW',\n", - " 'total_bytes': 127920163}\n" + "FileListingResponse:\n", + " count : 60\n", + " dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-sec raw data', provider='UCalgary', level='L0', doi_details='None', ...)\n", + " filenames : [60 filenames]\n", + " output_root_path : /home/darrenc/pyaurorax_data/THEMIS_ASI_RAW\n", + " total_bytes : 0\n" ] } ], "source": [ "# view information about the downloaded data\n", - "print(r)\n", - "print()\n", - "\n", - "pprint.pprint(r.__dict__)" + "r.pretty_print()" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7422451387c34a1498fa637b571d4065", + "model_id": "a0810307aa834155a29ecb18c4e3dfb9", "version_major": 2, "version_minor": 0 }, @@ -277,11 +100,34 @@ } ], "source": [ - "# download a couple minutes of data from all THEMIS sites\n", + "# an example of downloading several minutes of data from all\n", + "# THEMIS ASI sites (no site_uid filtering)\n", "dataset_name = \"THEMIS_ASI_RAW\"\n", "start_dt = datetime.datetime(2021, 11, 4, 9, 0)\n", "end_dt = datetime.datetime(2021, 11, 4, 9, 4)\n", - "_ = aurorax.data.ucalgary.download(dataset_name, start_dt, end_dt)" + "_ = aurorax.data.ucalgary.download(dataset_name, start_dt, end_dt) # we'll ignore the return variable since we won't use later in this crib sheet" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# to change where data is downloaded to, you can adjust an attribute in\n", + "# the PyAuroraX() class that was initialized at the beginning of the code.\n", + "#\n", + "# NOTE: this line is commented out on purpose since we just want to show \n", + "# how to do this, vs actually doing it.\n", + "#\n", + "# NOTE: the path you set can be a regular string path (nice for Linux and Mac)\n", + "# or a pathlib Path() object (nice for Windows).``\n", + "\n", + "#------------------\n", + "# aurorax.download_output_root_path = \"some path\"\n", + "# \n", + "# import pathlib\n", + "# aurorax.download_output_root_path = Path(\"some path\")\n" ] }, { @@ -290,76 +136,178 @@ "source": [ "# Read data\n", "\n", - "The data reading routines take in a list of filenames on your computer. The list of files is returned in a `download()` call, but can also be created using `glob` or similar.\n", + "Downloading data is only one part of the process. To allow you to not have to repeatedly download data, the `download()` and `read()` functions are split into two processes. \n", + "\n", + "The data reading routines are simple at the core. They take in a list of filenames on your computer, read in those files, and return the results back as an object. Be sure to pass in only one type of data at a time, otherwise the read routine will get rather confused!\n", + "\n", + "The advantage of this is that the read function just needs filenames. You can download data to any storage medium, and manually leverage `glob` like functions to get filenames. This can be beneficial if you don't have an internet connection at the time, but have already downloaded data. Or, you can simply run the `download()` function repeatedly; it will not re-download data if you already have it, unless the `overwrite` parameter is enabled.\n", "\n", "There are two methods that can be used for reading data:\n", "\n", "1) using the generic method\n", "2) using a specific dataset read function\n", "\n", - "The generic method is the recommended way, but if more control is wanted, you can use the specific read functions directly. The generic method simply uses the dataset name to determine which specific read function to use.\n" + "The generic method is the recommended way as it is simpler. However, if more control is wanted then you can use the specific read functions directly. The generic method simply uses the dataset name to determine which specific read function to use.\n" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Data(data=array(dims=(256, 256, 1200), dtype=uint16), timestamp=[1200 datetimes], metadata=[1200 dictionaries], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...))\n", + "Data(data=array([[[2540, 2602, 2635, ..., 2562, 2646, 2579],\n", + " [2503, 2533, 2604, ..., 2556, 2622, 2519],\n", + " [2600, 2537, 2600, ..., 2596, 2580, 2520],\n", + " ...,\n", + " [2557, 2567, 2589, ..., 2561, 2622, 2614],\n", + " [2542, 2575, 2536, ..., 2502, 2540, 2597],\n", + " [2589, 2550, 2568, ..., 2600, 2601, 2562]],\n", + "\n", + " [[2544, 2526, 2521, ..., 2574, 2569, 2545],\n", + " [2584, 2566, 2662, ..., 2608, 2671, 2562],\n", + " [2601, 2593, 2592, ..., 2591, 2562, 2531],\n", + " ...,\n", + " [2529, 2618, 2596, ..., 2575, 2624, 2680],\n", + " [2574, 2504, 2624, ..., 2598, 2512, 2554],\n", + " [2612, 2574, 2535, ..., 2548, 2532, 2532]],\n", + "\n", + " [[2572, 2568, 2552, ..., 2582, 2594, 2523],\n", + " [2550, 2549, 2525, ..., 2558, 2612, 2528],\n", + " [2519, 2591, 2555, ..., 2495, 2509, 2617],\n", + " ...,\n", + " [2590, 2521, 2587, ..., 2630, 2565, 2606],\n", + " [2551, 2564, 2508, ..., 2505, 2561, 2528],\n", + " [2611, 2537, 2574, ..., 2551, 2648, 2611]],\n", + "\n", + " ...,\n", + "\n", + " [[2564, 2497, 2570, ..., 2560, 2555, 2535],\n", + " [2582, 2579, 2517, ..., 2575, 2551, 2554],\n", + " [2530, 2580, 2552, ..., 2616, 2542, 2608],\n", + " ...,\n", + " [2512, 2566, 2537, ..., 2486, 2521, 2603],\n", + " [2630, 2562, 2545, ..., 2524, 2605, 2510],\n", + " [2581, 2524, 2524, ..., 2672, 2588, 2569]],\n", + "\n", + " [[2548, 2443, 2541, ..., 2583, 2569, 2603],\n", + " [2596, 2474, 2488, ..., 2563, 2605, 2573],\n", + " [2545, 2546, 2516, ..., 2589, 2606, 2665],\n", + " ...,\n", + " [2595, 2592, 2664, ..., 2539, 2560, 2591],\n", + " [2567, 2576, 2624, ..., 2579, 2508, 2559],\n", + " [2538, 2551, 2555, ..., 2550, 2595, 2564]],\n", + "\n", + " [[2525, 2560, 2524, ..., 2551, 2531, 2532],\n", + " [2527, 2590, 2562, ..., 2540, 2531, 2552],\n", + " [2604, 2515, 2581, ..., 2542, 2521, 2538],\n", + " ...,\n", + " [2560, 2582, 2608, ..., 2599, 2569, 2586],\n", + " [2506, 2475, 2556, ..., 2588, 2623, 2625],\n", + " [2497, 2549, 2548, ..., 2568, 2595, 2557]]], dtype=uint16), timestamp=[1200 datetimes], metadata=[1200 dictionaries], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...))\n", "\n", "Data:\n", - " data : array(dims=(256, 256, 1200), dtype=uint16)\n", - " timestamp : [1200 datetimes]\n", - " metadata : [1200 dictionaries]\n", - " problematic_files : []\n", - " calibrated_data : None\n", - " dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...)\n" + " data : array(dims=(256, 256, 1200), dtype=uint16)\n", + " timestamp : [1200 datetimes]\n", + " metadata : [1200 dictionaries]\n", + " problematic_files : []\n", + " calibrated_data : None\n", + " dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...)\n" ] } ], "source": [ - "# let's show the generic method first, since it is the easiest way\n", + "# we will show the generic method first, since it is the easiest way\n", + "#\n", + "# NOTE: we are reading the 1hr of data we downloaded earlier on, using 2 \n", + "# parallel processes to improve performance\n", "data = aurorax.data.ucalgary.read(r.dataset, r.filenames, n_parallel=2)\n", "\n", "print(data)\n", "print()\n", - "\n", "data.pretty_print()" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Data(data=array(dims=(256, 256, 1200), dtype=uint16), timestamp=[1200 datetimes], metadata=[1200 dictionaries], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...))\n", + "Data(data=array([[[2540, 2602, 2635, ..., 2562, 2646, 2579],\n", + " [2503, 2533, 2604, ..., 2556, 2622, 2519],\n", + " [2600, 2537, 2600, ..., 2596, 2580, 2520],\n", + " ...,\n", + " [2557, 2567, 2589, ..., 2561, 2622, 2614],\n", + " [2542, 2575, 2536, ..., 2502, 2540, 2597],\n", + " [2589, 2550, 2568, ..., 2600, 2601, 2562]],\n", + "\n", + " [[2544, 2526, 2521, ..., 2574, 2569, 2545],\n", + " [2584, 2566, 2662, ..., 2608, 2671, 2562],\n", + " [2601, 2593, 2592, ..., 2591, 2562, 2531],\n", + " ...,\n", + " [2529, 2618, 2596, ..., 2575, 2624, 2680],\n", + " [2574, 2504, 2624, ..., 2598, 2512, 2554],\n", + " [2612, 2574, 2535, ..., 2548, 2532, 2532]],\n", + "\n", + " [[2572, 2568, 2552, ..., 2582, 2594, 2523],\n", + " [2550, 2549, 2525, ..., 2558, 2612, 2528],\n", + " [2519, 2591, 2555, ..., 2495, 2509, 2617],\n", + " ...,\n", + " [2590, 2521, 2587, ..., 2630, 2565, 2606],\n", + " [2551, 2564, 2508, ..., 2505, 2561, 2528],\n", + " [2611, 2537, 2574, ..., 2551, 2648, 2611]],\n", + "\n", + " ...,\n", + "\n", + " [[2564, 2497, 2570, ..., 2560, 2555, 2535],\n", + " [2582, 2579, 2517, ..., 2575, 2551, 2554],\n", + " [2530, 2580, 2552, ..., 2616, 2542, 2608],\n", + " ...,\n", + " [2512, 2566, 2537, ..., 2486, 2521, 2603],\n", + " [2630, 2562, 2545, ..., 2524, 2605, 2510],\n", + " [2581, 2524, 2524, ..., 2672, 2588, 2569]],\n", + "\n", + " [[2548, 2443, 2541, ..., 2583, 2569, 2603],\n", + " [2596, 2474, 2488, ..., 2563, 2605, 2573],\n", + " [2545, 2546, 2516, ..., 2589, 2606, 2665],\n", + " ...,\n", + " [2595, 2592, 2664, ..., 2539, 2560, 2591],\n", + " [2567, 2576, 2624, ..., 2579, 2508, 2559],\n", + " [2538, 2551, 2555, ..., 2550, 2595, 2564]],\n", + "\n", + " [[2525, 2560, 2524, ..., 2551, 2531, 2532],\n", + " [2527, 2590, 2562, ..., 2540, 2531, 2552],\n", + " [2604, 2515, 2581, ..., 2542, 2521, 2538],\n", + " ...,\n", + " [2560, 2582, 2608, ..., 2599, 2569, 2586],\n", + " [2506, 2475, 2556, ..., 2588, 2623, 2625],\n", + " [2497, 2549, 2548, ..., 2568, 2595, 2557]]], dtype=uint16), timestamp=[1200 datetimes], metadata=[1200 dictionaries], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...))\n", "\n", "Data:\n", - " data : array(dims=(256, 256, 1200), dtype=uint16)\n", - " timestamp : [1200 datetimes]\n", - " metadata : [1200 dictionaries]\n", - " problematic_files : []\n", - " calibrated_data : None\n", - " dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...)\n" + " data : array(dims=(256, 256, 1200), dtype=uint16)\n", + " timestamp : [1200 datetimes]\n", + " metadata : [1200 dictionaries]\n", + " problematic_files : []\n", + " calibrated_data : None\n", + " dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...)\n" ] } ], "source": [ - "# since we know we're reading in THEMIS raw data, we can also use the specific read routine. In most\n", - "# circumstances, this method isn't necessary to use.\n", + "# since we know we're reading in THEMIS raw data, we can also use the \n", + "# specific read routine. Use these specific read functions if you want \n", + "# more control than the simpler read() function\n", "data = aurorax.data.ucalgary.readers.read_themis(r.filenames, n_parallel=2, dataset=r.dataset)\n", "\n", "print(data)\n", "print()\n", - "\n", "data.pretty_print()" ] }, @@ -367,23 +315,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Managing downloaded data" + "# Managing downloaded data\n", + "\n", + "Managing data is hard! For the All-sky Imager (ASI) data, the biggest concern to keep in mind is the available storage. ASI data is not small...THEMIS ASI alone is 120TB as of December 2024!\n", + "\n", + "To help with this, we have some utility functions at your fingertips. The `show_data_usage()` function can help show you how much data is on your computer in PyAuroraX's download output root path. Then `purge_download_output_root_path()` can delete all the data in that directory." ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Dataset name Size \n", - "=========================\n", - "THEMIS_ASI_RAW 253.6 MB\n", + "Dataset name Size \n", + "===============================\n", + "THEMIS_ASI_RAW 253.6 MB\n", + "TREX_RGB_RAW_NOMINAL 7.2 MB \n", "\n", - "Total size: 253.6 MB\n" + "Total size: 260.8 MB\n" ] } ], @@ -394,25 +347,51 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'THEMIS_ASI_RAW': {'path_obj': PosixPath('/home/darrenc/pyaurorax_data/THEMIS_ASI_RAW'),\n", + " 'size_bytes': 253634069,\n", + " 'size_str': '253.6 MB'},\n", + " 'TREX_RGB_RAW_NOMINAL': {'path_obj': PosixPath('/home/darrenc/pyaurorax_data/TREX_RGB_RAW_NOMINAL'),\n", + " 'size_bytes': 7187686,\n", + " 'size_str': '7.2 MB'}}\n" + ] + } + ], "source": [ "# alternatively, you can get the data usage information returned as a dictionary\n", - "data_usage_dict = aurorax.show_data_usage(return_dict=True)" + "data_usage_dict = aurorax.show_data_usage(return_dict=True)\n", + "pprint.pprint(data_usage_dict)" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# to clean up all data we've downloaded, you can do the following.\n", - "# \n", - "# alternatively, you can manually delete data yourself\n", - "aurorax.purge_download_output_root_path()" + "# to clean up all data we've downloaded, you can delete\n", + "# the data using a helper function, or manually delete \n", + "# it yourself\n", + "#\n", + "# delete all data\n", + "aurorax.purge_download_output_root_path()\n", + "\n", + "# delete data for single specific dataset\n", + "# aurorax.purge_download_output_root_path(dataset_name=\"THEMIS_ASI_RAW\")\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -431,7 +410,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.19" + "version": "3.9.20" } }, "nbformat": 4, diff --git a/examples/notebooks/data/download_skymaps_and_calibrations.ipynb b/examples/notebooks/data/download_skymaps_and_calibrations.ipynb index f4f04792..250268f7 100644 --- a/examples/notebooks/data/download_skymaps_and_calibrations.ipynb +++ b/examples/notebooks/data/download_skymaps_and_calibrations.ipynb @@ -1,44 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%html\n", - "" - ] - }, { "cell_type": "code", "execution_count": 2, @@ -56,7 +17,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Skymaps" + "# Skymaps\n", + "\n", + "Skymap files are used for projecting all-sky image data on a map. Skymaps are created for each of the ASI datasets we provide, and generate new ones every year, or when the deployed instrument is serviced. \n", + "\n", + "A detailed description of the skymaps can be found [here](https://data.phys.ucalgary.ca/sort_by_project/other/documentation/skymap_file_description.pdf).\n", + "\n", + "If you find that you are projecting data onto a map with multiple imagers and an auroral arc is not lining up between two sites, this is normally resolved by using a different skymap (a good first try is the one before or after what you are using). If you continue to have issues, reach out to the [dataset contact](https://data.phys.ucalgary.ca/about_datasets)." ] }, { @@ -74,12 +41,13 @@ "THEMIS_ASI_SKYMAP_IDLSAV UCalgary L3 None THEMIS All Sky Imagers skymap data (IDL save format) \n", "TREX_BLUE_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers skymap data (IDL save format)\n", "TREX_NIR_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers skymap data (IDL save format) \n", - "TREX_RGB_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers skymap data (IDL save format) \n" + "TREX_RGB_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers skymap data (IDL save format) \n", + "TREX_SPECT_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/2wnp-yc80 TREx Spectrograph skymap data (IDL save format) \n" ] } ], "source": [ - "# search datasets for skymaps\n", + "# list all available skymap datasets\n", "aurorax.data.list_datasets_in_table(name=\"SKYMAP\")" ] }, @@ -89,8 +57,8 @@ "metadata": {}, "outputs": [], "source": [ - "# we'll set our dataset for use later\n", - "dataset = aurorax.data.list_datasets(name=\"THEMIS_ASI_SKYMAP_IDLSAV\")[0]" + "# set the dataset that we'll use later\n", + "dataset = aurorax.data.get_dataset(name=\"THEMIS_ASI_SKYMAP_IDLSAV\")" ] }, { @@ -99,8 +67,8 @@ "source": [ "When selecting a skymap to use for projecting an image on a map, we have two methods available to us:\n", "\n", - "1) choosing manually\n", - "2) using the `find_best_skymap()` function to choose automatically\n", + "1) using the `find_best_skymap()` function to choose automatically\n", + "2) choosing manually\n", "\n", "Skymaps are generated for each site, and for a given time period. It is important to choose a skymap that is valid for the date you're looking at data for, otherwise the image may not appear accurately when projected on a map.\n" ] @@ -109,18 +77,62 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Choosing a skymap manually" + "## Automatically choosing a skymap\n", + "\n", + "The easiest way to choose a skymap is to lean on the `download_best_skymap()` function to let PyAuroraX figure out what's best to use. It takes the dataset name, site/observatory unique identifier, and a timestamp. Since skymaps change over the course of an imaging season, it is important to supply the timestamp of the data that you are plotting.\n", + "\n", + "To explore all available skymaps, you can see them in the data tree for each instrument array (ie. [THEMIS ASI](https://data.phys.ucalgary.ca/sort_by_project/THEMIS/asi/skymaps/), [TREx RGB](https://data.phys.ucalgary.ca/sort_by_project/TREx/RGB/skymaps/)).\n", + "\n", + "You can also let the library choose the skymap for you using the `download_best_skymap()` function." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[PosixPath('/home/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20210308/themis_skymap_gill_20210308-+_v02.sav')]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's say we are working on data from the Gillam THEMIS ASI on 2021-11-04\n", + "#\n", + "# set params\n", + "dataset_name = \"THEMIS_ASI_SKYMAP_IDLSAV\"\n", + "site_uid = \"gill\"\n", + "dt = datetime.datetime(2021, 11, 4)\n", + "\n", + "# download the recommended skymap\n", + "r = aurorax.data.ucalgary.download_best_skymap(dataset_name, site_uid, dt)\n", + "r.filenames" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Choosing a skymap manually\n", + "\n", + "Choosing a skymap manually" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a9c344f8ac6d46078b4087fc83291613", + "model_id": "96c1a042d8344e3bb56caff20f4554ba", "version_major": 2, "version_minor": 0 }, @@ -134,20 +146,18 @@ { "data": { "text/plain": [ - "[WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20210308/themis_skymap_gill_20210308-+_v02.sav'),\n", - " WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20220301/themis_skymap_gill_20220301-+_v02.sav')]" + "[PosixPath('/home/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20210308/themis_skymap_gill_20210308-+_v02.sav'),\n", + " PosixPath('/home/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20220301/themis_skymap_gill_20220301-+_v02.sav')]" ] }, - "execution_count": 5, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# First, let's choose the skymap we want manually. Let's assume we are working on data\n", - "# from the Gillam THEMIS ASI on 2021-11-04.\n", - "#\n", - "# We'll download the skymaps for a few years around that time, and then we'll choose which one we want after\n", + "# Next, let's see how you'd manually choose the skymap. We'll download the skymaps \n", + "# for a few years around that time, and then we'll go and decide which one we want\n", "r = aurorax.data.ucalgary.download(\n", " \"THEMIS_ASI_SKYMAP_IDLSAV\",\n", " datetime.datetime(2021, 1, 1, 0, 0),\n", @@ -163,12 +173,12 @@ "source": [ "Looks like we have a couple skymaps to choose from around 2021-11-04. We'll choose the first one since the date for it is before, and the second one's date is after. \n", "\n", - "The date indicates the first date it is valid for. There are some cases where a later or earlier skymap can be used. That is a situation where you can play around and try different skymaps, looking for which one works best for you. Most skymaps have small differences, but some have large ones that you'll notice very quickly when working with the projected data on a map.\n" + "The date indicates the first date it is valid for. There are some cases where a later or earlier skymap can be used. That is a situation where you can play around and try different skymaps. Most skymaps have small differences, but some have large ones that you'll notice very quickly when working with the projected data on a map (usually happens after the instrument has been serviced).\n" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -178,35 +188,34 @@ "Data(data=[1 Skymap object], timestamp=[], metadata=[], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_SKYMAP_IDLSAV, short_description='THEMIS All Sky Im...))\n", "\n", "Data:\n", - " data : [1 Skymap object]\n", - " timestamp : []\n", - " metadata : []\n", - " problematic_files : []\n", - " calibrated_data : None\n", - " dataset : Dataset(name=THEMIS_ASI_SKYMAP_IDLSAV, short_description='THEMIS All Sky Im...)\n", + " data : [1 Skymap object]\n", + " timestamp : []\n", + " metadata : []\n", + " problematic_files : []\n", + " calibrated_data : None\n", + " dataset : Dataset(name=THEMIS_ASI_SKYMAP_IDLSAV, short_description='THEMIS All Sky Im...)\n", "\n", "Skymap:\n", - " filename : C:\\Users\\darrenc\\pyaurorax_data\\THEMIS_ASI_SKYMAP_IDLSAV\\gill\\gill_20210308\\themis_skymap_gill_20210308-+_v02.sav\n", - " full_azimuth : array(dims=(256, 256), dtype=>f4)\n", - " full_elevation : array(dims=(256, 256), dtype=>f4)\n", - " full_map_altitude : array(dims=(3,), dtype=>f4)\n", - " full_map_latitude : array(dims=(3, 257, 257), dtype=>f4)\n", - " full_map_longitude : array(dims=(3, 257, 257), dtype=>f4)\n", - " generation_info : SkymapGenerationInfo(...)\n", - " imager_uid : themis19\n", - " project_uid : themis\n", - " site_map_altitude : 0.0\n", - " site_map_latitude : 56.376724\n", - " site_map_longitude : 265.35632\n", - " site_uid : gill\n", - " version : v02\n" + " filename : /home/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20210308/themis_skymap_gill_20210308-+_v02.sav\n", + " full_azimuth : array(dims=(256, 256), dtype=>f4)\n", + " full_elevation : array(dims=(256, 256), dtype=>f4)\n", + " full_map_altitude : array(dims=(3,), dtype=>f4)\n", + " full_map_latitude : array(dims=(3, 257, 257), dtype=>f4)\n", + " full_map_longitude : array(dims=(3, 257, 257), dtype=>f4)\n", + " generation_info : SkymapGenerationInfo(...)\n", + " imager_uid : themis19\n", + " project_uid : themis\n", + " site_map_altitude : 0.0\n", + " site_map_latitude : 56.376724\n", + " site_map_longitude : 265.35632\n", + " site_uid : gill\n", + " version : v02\n" ] } ], "source": [ - "# Now that we know which one we'll use, we can read it in.\n", - "#\n", - "# You can also read in all of them and choose later using the resulting Data object.\n", + "# Now that we know which one we'll use, we can read it in. You can also read \n", + "# in all of them and choose later using the resulting Data object.\n", "skymap_data = aurorax.data.ucalgary.read(dataset, r.filenames[0])\n", "\n", "print(skymap_data)\n", @@ -217,45 +226,9 @@ "skymap_data.data[0].pretty_print()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Automatically choosing a skymap\n", - "\n", - "You can also let the library choose the skymap for you using the `download_best_skymap()` function." - ] - }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_SKYMAP_IDLSAV/gill/gill_20210308/themis_skymap_gill_20210308-+_v02.sav')]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set params\n", - "dataset_name = \"THEMIS_ASI_SKYMAP_IDLSAV\"\n", - "site_uid = \"gill\"\n", - "dt = datetime.datetime(2021, 11, 4)\n", - "\n", - "# get the recommendation\n", - "r = aurorax.data.ucalgary.download_best_skymap(dataset_name, site_uid, dt)\n", - "r.filenames\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -573,7 +546,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.19" + "version": "3.9.20" } }, "nbformat": 4, diff --git a/examples/notebooks/data/explore_datasets_and_observatories.ipynb b/examples/notebooks/data/explore_datasets_and_observatories.ipynb index 06e71219..34cd91f8 100644 --- a/examples/notebooks/data/explore_datasets_and_observatories.ipynb +++ b/examples/notebooks/data/explore_datasets_and_observatories.ipynb @@ -7,7 +7,6 @@ "outputs": [], "source": [ "import pyaurorax\n", - "import pprint\n", "\n", "aurorax = pyaurorax.PyAuroraX()" ] @@ -16,7 +15,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Explore datasets" + "# Explore datasets\n", + "\n", + "All data available is organized by unique 'dataset' identifier strings, similar to CDAWeb. For example, `THEMIS_ASI_RAW` is the dataset name for the raw THEMIS all-sky imager data, one of the common datasets for that instrument array.\n", + "\n", + "There are a few functions to help explore the available datasets and information about them. There is `list_datasets()` to retrieve any datasets matching optional filters, `list_datasets_in_table()` which does the same thing but displays the results in a nice table to help with readability, and `get_dataset()` which retrieves a specific single dataset.\n", + "\n", + "You can also navigate to the [Dataset Descriptions](https://data.phys.ucalgary.ca/about_datasets) page and navigate to a particular instrument page. There, you will find a listing of all available (and commonly utilized) datasets for an instrument, along with instrument location/field-of-view maps, and observatory locations.\n", + "\n", + "Each dataset has a few attributes. For example, DOI and citation information, data tree location, and provider." ] }, { @@ -28,7 +35,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found 63 datasets\n" + "Found 73 datasets\n" ] } ], @@ -50,33 +57,33 @@ "text": [ "Dataset(name=REGO_CALIBRATION_FLATFIELD_IDLSAV, short_description='REGO All Sky Imagers Flatfield calibration data (IDL save format)', provider='UCalgary', level='L3', doi_details='https://commons.datacite.org/doi.org/10.11575/z7x6-5c42', ...)\n", "\n", - "{'citation': 'Spanswick, E., & Donovan, E. (2014). Redline Geospace '\n", - " 'Observatory (REGO) dataset [Data set]. University of Calgary. '\n", - " 'https://doi.org/10.11575/Z7X6-5C42',\n", - " 'data_tree_url': 'https://data.phys.ucalgary.ca/sort_by_project/GO-Canada/REGO/calibration',\n", - " 'doi': 'https://doi.org/10.11575/Z7X6-5C42',\n", - " 'doi_details': 'https://commons.datacite.org/doi.org/10.11575/z7x6-5c42',\n", - " 'file_listing_supported': True,\n", - " 'file_reading_supported': True,\n", - " 'level': 'L3',\n", - " 'long_description': 'Redline Geospace Observatory (REGO) All Sky Imager '\n", - " 'array. More information can be found at '\n", - " 'https://data.phys.ucalgary.ca',\n", - " 'name': 'REGO_CALIBRATION_FLATFIELD_IDLSAV',\n", - " 'provider': 'UCalgary',\n", - " 'short_description': 'REGO All Sky Imagers Flatfield calibration data (IDL '\n", - " 'save format)'}\n" + "Dataset:\n", + " citation : Spanswick, E., & Donovan, E. (2014). Redline Geospace Observatory (REGO) dataset [Data set]. University of Calgary. https://doi.org/10.11575/Z7X6-5C42\n", + " data_tree_url : https://data.phys.ucalgary.ca/sort_by_project/GO-Canada/REGO/calibration\n", + " doi : https://doi.org/10.11575/Z7X6-5C42\n", + " doi_details : https://commons.datacite.org/doi.org/10.11575/z7x6-5c42\n", + " file_listing_supported : True\n", + " file_reading_supported : True\n", + " file_time_resolution : not_applicable\n", + " level : L3\n", + " long_description : Redline Geospace Observatory (REGO) All Sky Imager array. More information can be found at https://data.phys.ucalgary.ca\n", + " name : REGO_CALIBRATION_FLATFIELD_IDLSAV\n", + " provider : UCalgary\n", + " short_description : REGO All Sky Imagers Flatfield calibration data (IDL save format)\n", + " supported_libraries : ['idl-aurorax', 'pyaurorax', 'pyucalgarysrs']\n" ] } ], "source": [ "# view the first dataset\n", "print(datasets[0])\n", - "\n", "print()\n", "\n", - "# show all values in the Dataset class\n", - "pprint.pprint(datasets[0].__dict__)" + "# show a nice printout of the dataset information\n", + "#\n", + "# NOTE: for most objects in PyAuroraX, we provide a 'pretty_print()' method which\n", + "# displays class variables in a similar more legible way.\n", + "datasets[0].pretty_print()\n" ] }, { @@ -88,105 +95,106 @@ "name": "stdout", "output_type": "stream", "text": [ - "Dataset:\n", - " citation : Spanswick, E., & Donovan, E. (2014). Redline Geospace Observatory (REGO) dataset [Data set]. University of Calgary. https://doi.org/10.11575/Z7X6-5C42\n", - " data_tree_url : https://data.phys.ucalgary.ca/sort_by_project/GO-Canada/REGO/calibration\n", - " doi : https://doi.org/10.11575/Z7X6-5C42\n", - " doi_details : https://commons.datacite.org/doi.org/10.11575/z7x6-5c42\n", - " file_listing_supported : True\n", - " file_reading_supported : True\n", - " level : L3\n", - " long_description : Redline Geospace Observatory (REGO) All Sky Imager array. More information can be found at https://data.phys.ucalgary.ca\n", - " name : REGO_CALIBRATION_FLATFIELD_IDLSAV\n", - " provider : UCalgary\n", - " short_description : REGO All Sky Imagers Flatfield calibration data (IDL save format)\n" - ] - } - ], - "source": [ - "# there is also a pretty_print function to show all information in a different way\n", - "datasets[0].pretty_print()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name Provider Level DOI Details Short Description \n", - "================================================================================================================================================================================================\n", - "REGO_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Flatfield calibration data (IDL save format) \n", - "REGO_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Rayleighs calibration data (IDL save format) \n", - "REGO_DAILY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (JPG format) \n", - "REGO_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PGM format) \n", - "REGO_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PNG format) \n", - "REGO_DAILY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (JPG format) \n", - "REGO_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PGM format) \n", - "REGO_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PNG format) \n", - "REGO_HOURLY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (JPG format) \n", - "REGO_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PGM format) \n", - "REGO_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PNG format) \n", - "REGO_HOURLY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (JPG format) \n", - "REGO_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PGM format) \n", - "REGO_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PNG format) \n", - "REGO_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers 3-sec raw data \n", - "REGO_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers skymap data (IDL save format) \n", - "THEMIS_ASI_DAILY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (JPG format) \n", - "THEMIS_ASI_DAILY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (PGM format) \n", - "THEMIS_ASI_DAILY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (JPG format) \n", - "THEMIS_ASI_DAILY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (PGM format) \n", - "THEMIS_ASI_HOURLY_AVERAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (JPG format) \n", - "THEMIS_ASI_HOURLY_AVERAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (PGM format) \n", - "THEMIS_ASI_HOURLY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (JPG format) \n", - "THEMIS_ASI_HOURLY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (PGM format) \n", - "THEMIS_ASI_HOURLY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (JPG format) \n", - "THEMIS_ASI_HOURLY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (PGM format) \n", - "THEMIS_ASI_RAW UCalgary L0 None THEMIS All Sky Imagers 3-sec raw data \n", - "THEMIS_ASI_RAW_ROW2 UCalgary L0 None THEMIS All Sky Imagers raw row2 data \n", - "THEMIS_ASI_RAW_WIDE UCalgary L0 None THEMIS All Sky Imagers raw wide data \n", - "THEMIS_ASI_SKYMAP_IDLSAV UCalgary L3 None THEMIS All Sky Imagers skymap data (IDL save format) \n", - "THEMIS_ASI_THUMB32 UCalgary L1A None THEMIS All Sky Imagers thumb32 data \n", - "THEMIS_ASI_VEC1024 UCalgary L1A None THEMIS All Sky Imagers vec1024 data \n", - "TREX_BLUE_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Flatfield calibration data (IDL save format)\n", - "TREX_BLUE_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Rayleighs calibration data (IDL save format)\n", - "TREX_BLUE_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PGM format) \n", - "TREX_BLUE_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PNG format) \n", - "TREX_BLUE_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PGM format) \n", - "TREX_BLUE_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PNG format) \n", - "TREX_BLUE_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PGM format) \n", - "TREX_BLUE_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PNG format) \n", - "TREX_BLUE_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PGM format) \n", - "TREX_BLUE_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PNG format) \n", - "TREX_BLUE_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers 3-sec raw data \n", - "TREX_BLUE_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers skymap data (IDL save format) \n", - "TREX_NIR_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Flatfield calibration data (IDL save format) \n", - "TREX_NIR_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Rayleighs calibration data (IDL save format) \n", - "TREX_NIR_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PGM format) \n", - "TREX_NIR_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PNG format) \n", - "TREX_NIR_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PGM format) \n", - "TREX_NIR_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PNG format) \n", - "TREX_NIR_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PGM format) \n", - "TREX_NIR_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PNG format) \n", - "TREX_NIR_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PGM format) \n", - "TREX_NIR_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PNG format) \n", - "TREX_NIR_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers 6-sec raw data \n", - "TREX_NIR_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers skymap data (IDL save format) \n", - "TREX_RGB_DAILY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily keogram summary product \n", - "TREX_RGB_DAILY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily montage summary product \n", - "TREX_RGB_HOURLY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly keogram summary product \n", - "TREX_RGB_HOURLY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly montage summary product \n", - "TREX_RGB_RAW_BURST UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3Hz raw burst data \n", - "TREX_RGB_RAW_NOMINAL UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3-sec raw data \n", - "TREX_RGB_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers skymap data (IDL save format) \n" + "Name Provider Level DOI Details Short Description \n", + "========================================================================================================================================================================================================\n", + "REGO_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Flatfield calibration data (IDL save format) \n", + "REGO_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Rayleighs calibration data (IDL save format) \n", + "REGO_DAILY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (JPG format) \n", + "REGO_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PGM format) \n", + "REGO_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PNG format) \n", + "REGO_DAILY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (JPG format) \n", + "REGO_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PGM format) \n", + "REGO_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PNG format) \n", + "REGO_GRID_MOSV001 UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers MOSv001 verified grid files (corner subtracted,\n", + " flatfield corrected, rayleighs converted, intensity grid priority) \n", + "REGO_HOURLY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (JPG format) \n", + "REGO_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PGM format) \n", + "REGO_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PNG format) \n", + "REGO_HOURLY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (JPG format) \n", + "REGO_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PGM format) \n", + "REGO_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PNG format) \n", + "REGO_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers 3-sec raw data \n", + "REGO_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers skymap data (IDL save format) \n", + "THEMIS_ASI_DAILY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (JPG format) \n", + "THEMIS_ASI_DAILY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (PGM format) \n", + "THEMIS_ASI_DAILY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (JPG format) \n", + "THEMIS_ASI_DAILY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (PGM format) \n", + "THEMIS_ASI_GRID_MOSU001 UCalgary L3 None THEMIS All Sky Imagers MOSu001 unverified grid files (corner \n", + " subtracted, intensity grid priority, ML cloud detection) \n", + "THEMIS_ASI_GRID_MOSV001 UCalgary L3 None THEMIS All Sky Imagers MOSv001 verified grid files (corner \n", + " subtracted, intensity grid priority) \n", + "THEMIS_ASI_HOURLY_AVERAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (JPG format) \n", + "THEMIS_ASI_HOURLY_AVERAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (PGM format) \n", + "THEMIS_ASI_HOURLY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (JPG format) \n", + "THEMIS_ASI_HOURLY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (PGM format) \n", + "THEMIS_ASI_HOURLY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (JPG format) \n", + "THEMIS_ASI_HOURLY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (PGM format) \n", + "THEMIS_ASI_RAW UCalgary L0 None THEMIS All Sky Imagers 3-sec raw data \n", + "THEMIS_ASI_RAW_ROW2 UCalgary L0 None THEMIS All Sky Imagers raw row2 data \n", + "THEMIS_ASI_RAW_WIDE UCalgary L0 None THEMIS All Sky Imagers raw wide data \n", + "THEMIS_ASI_SKYMAP_IDLSAV UCalgary L3 None THEMIS All Sky Imagers skymap data (IDL save format) \n", + "THEMIS_ASI_THUMB32 UCalgary L1A None THEMIS All Sky Imagers thumb32 data \n", + "THEMIS_ASI_VEC1024 UCalgary L1A None THEMIS All Sky Imagers vec1024 data \n", + "TREX_BLUE_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Flatfield calibration data (IDL save \n", + " format) \n", + "TREX_BLUE_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Rayleighs calibration data (IDL save \n", + " format) \n", + "TREX_BLUE_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PGM \n", + " format) \n", + "TREX_BLUE_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PNG \n", + " format) \n", + "TREX_BLUE_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PGM \n", + " format) \n", + "TREX_BLUE_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PNG \n", + " format) \n", + "TREX_BLUE_GRID_MOSV001 UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers MOSv001 verified grid files (raw data,\n", + " intensity grid priority) \n", + "TREX_BLUE_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PGM \n", + " format) \n", + "TREX_BLUE_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PNG \n", + " format) \n", + "TREX_BLUE_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PGM \n", + " format) \n", + "TREX_BLUE_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PNG \n", + " format) \n", + "TREX_BLUE_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers 3-sec raw data \n", + "TREX_BLUE_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers skymap data (IDL save format) \n", + "TREX_NIR_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Flatfield calibration data (IDL save \n", + " format) \n", + "TREX_NIR_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Rayleighs calibration data (IDL save \n", + " format) \n", + "TREX_NIR_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PGM format) \n", + "TREX_NIR_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PNG format) \n", + "TREX_NIR_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PGM format) \n", + "TREX_NIR_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PNG format) \n", + "TREX_NIR_GRID_MOSV001 UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers MOSv001 verified grid files (raw data, \n", + " intensity grid priority) \n", + "TREX_NIR_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PGM format)\n", + "TREX_NIR_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PNG format)\n", + "TREX_NIR_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PGM format)\n", + "TREX_NIR_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PNG format)\n", + "TREX_NIR_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers 6-sec raw data \n", + "TREX_NIR_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers skymap data (IDL save format) \n", + "TREX_RGB5577_GRID_MOSV001 UCalgary L3 https://doi.org/10.26464/epp2023063, TREx RGB All Sky Imager derived 557.7nm emission MOSv001 verified \n", + " https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 grid files (intensity grid priority) \n", + "TREX_RGB_DAILY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily keogram summary product \n", + "TREX_RGB_DAILY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily montage summary product \n", + "TREX_RGB_GRID_MOSV001 UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers MOSv001 verified grid files (raw data, \n", + " intensity grid priority) \n", + "TREX_RGB_HOURLY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly keogram summary product \n", + "TREX_RGB_HOURLY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly montage summary product \n", + "TREX_RGB_RAW_BURST UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3Hz raw burst data \n", + "TREX_RGB_RAW_NOMINAL UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3-sec raw data \n", + "TREX_RGB_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers skymap data (IDL save format) \n", + "TREX_SPECT_PROCESSED_V1 UCalgary L1 https://commons.datacite.org/doi.org/10.11575/2wnp-yc80 TREx Spectrograph 15-sec processed data, version 1 (calibrated, de- \n", + " smiled) \n", + "TREX_SPECT_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/2wnp-yc80 TREx Spectrograph 15-sec raw data \n", + "TREX_SPECT_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/2wnp-yc80 TREx Spectrograph skymap data (IDL save format) \n" ] } ], "source": [ - "# you can also print all datasets in a table\n", + "# print all datasets in a tabular format\n", "aurorax.data.list_datasets_in_table()" ] }, @@ -194,12 +202,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Explore observatories" + "# Explore observatories\n", + "\n", + "A set of observatories are available for each instrument array. These observatories provide information about the sites where data was produced during the array operations. Each observatory object provdes site code and full names, along with their geodetic latitude and longitude.\n", + "\n", + "You can use the `list_observatories()` function to retrieve all, and `list_observatories_in_table()` to again show the information in a tabular form. To determine the valid 'instrument_array' values, please refer to the PyAuroraX API reference, or utilize the type hinting built into common editors such as VSCode." ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -211,7 +223,7 @@ } ], "source": [ - "# list all observatories\n", + "# list all observatories for THEMIS ASI\n", "observatories = aurorax.data.list_observatories(\"themis_asi\")\n", "\n", "print(\"Found %d observatories\" % (len(observatories)))" @@ -219,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -228,50 +240,27 @@ "text": [ "Observatory(uid=atha, full_name='Athabasca, AB, Canada', geodetic_latitude=54.6, geodetic_longitude=-113.64, provider='UCalgary')\n", "\n", - "{'full_name': 'Athabasca, AB, Canada',\n", - " 'geodetic_latitude': 54.6,\n", - " 'geodetic_longitude': -113.64,\n", - " 'provider': 'UCalgary',\n", - " 'uid': 'atha'}\n" + "Observatory:\n", + " full_name : Athabasca, AB, Canada\n", + " geodetic_latitude : 54.6\n", + " geodetic_longitude : -113.64\n", + " provider : UCalgary\n", + " uid : atha\n" ] } ], "source": [ "# view the first dataset\n", "print(observatories[0])\n", - "\n", "print()\n", "\n", - "# show all values in the Dataset class\n", - "pprint.pprint(observatories[0].__dict__)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observatory:\n", - " full_name : Athabasca, AB, Canada\n", - " geodetic_latitude : 54.6\n", - " geodetic_longitude : -113.64\n", - " provider : UCalgary\n", - " uid : atha\n" - ] - } - ], - "source": [ - "# there is also a pretty_print function to show all information in a different way\n", + "# show a nice printout of the observatory information\n", "observatories[0].pretty_print()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -308,9 +297,16 @@ } ], "source": [ - "# you can also print all observatories in a table\n", + "# print all THEMIS ASI observatories in a tabular format\n", "aurorax.data.list_observatories_in_table(\"themis_asi\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -329,7 +325,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.19" + "version": "3.9.20" } }, "nbformat": 4, diff --git a/poetry.lock b/poetry.lock index 09c2fe81..9566478d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aacgmv2" @@ -769,13 +769,13 @@ files = [ [[package]] name = "mako" -version = "1.3.7" +version = "1.3.8" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = false python-versions = ">=3.8" files = [ - {file = "Mako-1.3.7-py3-none-any.whl", hash = "sha256:d18f990ad57f800ce8e76cbfb0b74afe471c293517e9f5003ace6dad5aa72c36"}, - {file = "mako-1.3.7.tar.gz", hash = "sha256:20405b1232e0759f0e7d87b01f6bb94fce0761747f1cb876ecf90bd512d0b639"}, + {file = "Mako-1.3.8-py3-none-any.whl", hash = "sha256:42f48953c7eb91332040ff567eb7eea69b22e7a4affbc5ba8e845e8f730f6627"}, + {file = "mako-1.3.8.tar.gz", hash = "sha256:577b97e414580d3e088d47c2dbbe9594aa7a5146ed2875d4dfa9075af2dd3cc8"}, ] [package.dependencies] diff --git a/pyaurorax/__init__.py b/pyaurorax/__init__.py index 023e527b..be34e193 100644 --- a/pyaurorax/__init__.py +++ b/pyaurorax/__init__.py @@ -32,7 +32,7 @@ """ # versioning info -__version__ = "1.5.0" +__version__ = "1.6.0" # documentation excludes __pdoc__ = {"cli": False, "pyaurorax": False} diff --git a/pyaurorax/cli/cli.py b/pyaurorax/cli/cli.py index 22e4e1d9..e99e12b1 100644 --- a/pyaurorax/cli/cli.py +++ b/pyaurorax/cli/cli.py @@ -52,7 +52,7 @@ def __test_connectivity(aurorax, quiet=False, return_json=False): @click.group(invoke_without_command=True) -@click.version_option(version="1.5.0") +@click.version_option(version="1.6.0") @click.option("--api-key", type=str, help="Specify an API key") @click.option("--api-base-url", type=str, help="Set the AuroraX API base URL") @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") diff --git a/pyaurorax/pyaurorax.py b/pyaurorax/pyaurorax.py index 2707dcae..a2f25c76 100644 --- a/pyaurorax/pyaurorax.py +++ b/pyaurorax/pyaurorax.py @@ -313,7 +313,7 @@ def __initialize_paths(self): # ----------------------------- # public methods # ----------------------------- - def purge_download_output_root_path(self): + def purge_download_output_root_path(self, dataset_name: Optional[str] = None): """ Delete all files in the `download_output_root_path` directory. Since the library downloads data to this directory, over time it can grow too large @@ -324,6 +324,10 @@ def purge_download_output_root_path(self): download_output_root_path path as well. Normally, these two paths are the same, but it can be different if the user specifically changes it. + Args: + dataset_name (str): + Delete only files for a specific dataset name. This parameter is optional. + Raises: pyaurorax.exceptions.AuroraXPurgeError: an error was encountered during the purge operation """ @@ -331,13 +335,13 @@ def purge_download_output_root_path(self): # purge pyaurorax path for item in os.listdir(self.download_output_root_path): item = Path(self.download_output_root_path) / item - if (os.path.isdir(item) is True and self.read_tar_temp_path not in str(item)): - shutil.rmtree(item) - elif (os.path.isfile(item) is True): - os.remove(item) - # purge pyucalgarysrs path - self.__srs_obj.purge_download_output_root_path() + # check if this is the dataset we want to delete + if (dataset_name is None or item.name == dataset_name.upper()): + if (os.path.isdir(item) is True and self.read_tar_temp_path not in str(item)): + shutil.rmtree(item) + elif (os.path.isfile(item) is True): + os.remove(item) except Exception as e: # pragma: nocover raise AuroraXPurgeError("Error while purging download output root path: %s" % (str(e))) from e diff --git a/pyproject.toml b/pyproject.toml index 45035b24..01fa6486 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api" [tool.poetry] name = "pyaurorax" -version = "1.5.0" +version = "1.6.0" description = "Python library for interacting with the AuroraX platform" readme = "README.md" homepage = "https://github.com/aurorax-space/pyaurorax" diff --git a/tests/test_suite/test_version.py b/tests/test_suite/test_version.py index 13b71bc9..335a0181 100644 --- a/tests/test_suite/test_version.py +++ b/tests/test_suite/test_version.py @@ -16,4 +16,4 @@ def test_version(): - assert __version__ == "1.5.0" + assert __version__ == "1.6.0"