Add Gridded VIIRS from the NOAA AWS bucket and more

monetio/__init__.py

@@ -39,6 +39,10 @@
     #
     # satellite obs
     "goes",
+    "nesdis_eps_viirs_aod_nrt",
+    "nesdis_viirs_aod_aws_gridded",
+    "nesdis_avhrr_aot_aws_gridded",
+    "nesdis_viirs_ndvi_aws_gridded",
     #
     # models
     "camx",

monetio/sat/__init__.py

@@ -1,4 +1,4 @@
-from . import (
+from . import (  # nesdis_avhrr_aot_aws_gridded,
     _gridded_eos_mm,
     _modis_l2_mm,
     _mopitt_l3_mm,
@@ -7,9 +7,10 @@
     _tropomi_l2_no2_mm,
     goes,
     modis_ornl,
-    nesdis_edr_viirs,
-    nesdis_eps_viirs,
+    nesdis_eps_viirs_aod_nrt,
     nesdis_frp,
+    nesdis_viirs_aod_aws_gridded,
+    nesdis_viirs_ndvi_aws_gridded,
 )
 
 __all__ = [
@@ -18,6 +19,10 @@
     "_mopitt_l3_mm",
     "_omps_l3_mm",
     "_omps_nadir_mm",
+    "nesdis_viirs_aod_aws_gridded",
+    "nesdis_avhrr_aot_aws_gridded",
+    "nesdis_viirs_ndvi_aws_gridded",
+    "nesdis_eps_viirs_aod_nrt",
     "_tropomi_l2_no2_mm",
     "goes",
     "modis_ornl",

monetio/sat/nesdis_avhrr_aot_aws_gridded.py

@@ -0,0 +1,169 @@
+def create_daily_aod_list(date_generated, fs, warning=False):
+    """
+    Creates a list of daily AOD (Aerosol Optical Depth) files and calculates the total size of the files.
+
+    Parameters:
+        date_generated (list): A list of dates for which to check the existence of AOD files.
+        fs (FileSystem): The file system object used to check file existence and size.
+
+    Returns:
+        tuple: A tuple containing the list of file paths and the total size of the files.
+    """
+    import warnings
+
+    # Loop through observation dates & check for files
+    nodd_file_list = []
+    nodd_total_size = 0
+    for date in date_generated:
+        file_date = date.strftime("%Y%m%d")
+        year = file_date[:4]
+        prod_path = "noaa-cdr-aerosol-optical-thickness-pds/data/daily/" + year + "/"
+        patt = "AOT_AVHRR_*_daily-avg_"
+        file_names = fs.glob(prod_path + patt + file_date + "_*.nc")
+        # If file exists, add path to list and add file size to total
+        if file_names:
+            nodd_file_list.extend(file_names)
+            nodd_total_size = nodd_total_size + sum(fs.size(f) for f in file_names)
+        else:
+            msg = "File does not exist on AWS: " + prod_path + patt + file_date + "_*.nc"
+            if warning:
+                warnings.warn(msg)
+                nodd_file_list.append(None)
+            else:
+                raise ValueError(msg)
+
+    return nodd_file_list, nodd_total_size
+
+
+def create_monthly_aod_list(date_generated, fs, warning=False):
+    """
+    Creates a list of daily AOD (Aerosol Optical Depth) files and calculates the total size of the files.
+
+    Parameters:
+        date_generated (list): A list of dates for which to check the existence of AOD files.
+        fs (FileSystem): The file system object used to check file existence and size.
+
+    Returns:
+        tuple: A tuple containing the list of file paths and the total size of the files.
+    """
+    import warnings
+
+    # Loop through observation dates & check for files
+    nodd_file_list = []
+    nodd_total_size = 0
+    for date in date_generated:
+        file_date = date.strftime("%Y%m%d")
+        year = file_date[:4]
+        prod_path = "noaa-cdr-aerosol-optical-thickness-pds/data/monthly/" + year + "/"
+        patt = "AOT_AVHRR_*_daily-avg_"
+        file_names = fs.glob(prod_path + patt + file_date + "_*.nc")
+        # If file exists, add path to list and add file size to total
+        if file_names:
+            nodd_file_list.extend(file_names)
+            nodd_total_size = nodd_total_size + sum(fs.size(f) for f in file_names)
+        else:
+            msg = "File does not exist on AWS: " + prod_path + patt + file_date + "_*.nc"
+            if warning:
+                warnings.warn(msg)
+                nodd_file_list.append(None)
+            else:
+                raise ValueError(msg)
+
+    return nodd_file_list, nodd_total_size
+
+
+def open_dataset(date, averaging_time="daily"):
+    """
+    Opens a dataset for the given date, satellite, data resolution, and averaging time.
+
+    Parameters:
+        date (str or datetime.datetime): The date for which to open the dataset.
+        averaging_time (str, optional): The averaging time.
+            Valid values are 'daily', or 'monthly'. Defaults to 'daily'.
+
+    Returns:
+        xarray.Dataset: The opened dataset.
+
+    Raises:
+        ValueError: If the input values are invalid.
+    """
+    import pandas as pd
+    import s3fs
+    import xarray as xr
+
+    if isinstance(date, str):
+        date_generated = [pd.Timestamp(date)]
+    else:
+        date_generated = [date]
+
+    # Access AWS using anonymous credentials
+    fs = s3fs.S3FileSystem(anon=True)
+
+    if averaging_time.lower() == "monthly":
+        file_list, _ = create_monthly_aod_list(date_generated, fs)
+    elif averaging_time.lower() == "daily":
+        file_list, _ = create_daily_aod_list(date_generated, fs)
+    else:
+        raise ValueError(
+            f"Invalid input for 'averaging_time' {averaging_time!r}: "
+            "Valid values are 'daily' or 'monthly'"
+        )
+
+    if len(file_list) == 0 or all(f is None for f in file_list):
+        raise ValueError(f"Files not available for product and date: {date_generated[0]}")
+
+    aws_file = fs.open(file_list[0])
+
+    dset = xr.open_dataset(aws_file)
+
+    return dset
+
+
+def open_mfdataset(dates, averaging_time="daily", error_missing=False):
+    """
+    Opens and combines multiple NetCDF files into a single xarray dataset.
+
+    Parameters:
+        dates (pandas.DatetimeIndex): The dates for which to retrieve the data.
+        averaging_time (str, optional): The averaging time.
+            Valid values are 'daily', 'weekly', or 'monthly'. Defaults to 'daily'.
+
+    Returns:
+        xarray.Dataset: The combined dataset containing the data for the specified dates.
+
+    Raises:
+        ValueError: If the input parameters are invalid.
+
+    """
+    from collections.abc import Iterable
+
+    import pandas as pd
+    import s3fs
+    import xarray as xr
+
+    if isinstance(dates, Iterable) and not isinstance(dates, str):
+        dates = pd.DatetimeIndex(dates)
+    else:
+        dates = pd.DatetimeIndex([dates])
+
+    # Access AWS using anonymous credentials
+    fs = s3fs.S3FileSystem(anon=True)
+
+    if averaging_time.lower() == "monthly":
+        file_list, _ = create_monthly_aod_list(dates, fs, warning=not error_missing)
+    elif averaging_time.lower() == "daily":
+        file_list, _ = create_daily_aod_list(dates, fs, warning=not error_missing)
+    else:
+        raise ValueError(
+            f"Invalid input for 'averaging_time' {averaging_time!r}: "
+            "Valid values are 'daily' or 'monthly'"
+        )
+
+    if len(file_list) == 0 or all(f is None for f in file_list):
+        raise ValueError(f"Files not available for product and dates: {dates}")
+
+    aws_files = [fs.open(f) for f in file_list if f is not None]
+
+    dset = xr.open_mfdataset(aws_files, concat_dim="time", combine="nested")
+
+    return dset