Added ncei_server for older data not available at SPDF

pyspedas/projects/poes/README.md

@@ -2,9 +2,17 @@
 ## Polar Orbiting Environmental Satellites (POES) Mission
 The routines in this module can be used to load data from the POES mission. 
 
+
 ### Instruments
 - Space Environment Monitor (SEM)
 
+
+### Data location
+
+Data is loaded from the SPDF server. However, some older data is only available at NOAA 
+and in this case, the user must set ncei_server=True. 
+
+
 ### Examples
 Get started by importing pyspedas and tplot; these are required to load and plot the data:
 

pyspedas/projects/poes/config.py

@@ -1,11 +1,13 @@
 import os
 
-CONFIG = {'local_data_dir': 'poes_data/',
-          'remote_data_dir': 'https://spdf.gsfc.nasa.gov/pub/data/noaa/'}
+CONFIG = {
+    "local_data_dir": "poes_data/",
+    "remote_data_dir": "https://spdf.gsfc.nasa.gov/pub/data/noaa/",
+    "ncei_remote_data_dir": "https://www.ncei.noaa.gov/data/poes-metop-space-environment-monitor/access/l2/v01r00/cdf/",
+}
 
 # override local data directory with environment variables
-if os.environ.get('SPEDAS_DATA_DIR'):
-    CONFIG['local_data_dir'] = os.sep.join([os.environ['SPEDAS_DATA_DIR'], 'poes'])
-
-if os.environ.get('POES_DATA_DIR'):
-    CONFIG['local_data_dir'] = os.environ['POES_DATA_DIR']
+if os.environ.get("POES_DATA_DIR"):
+    CONFIG["local_data_dir"] = os.environ["POES_DATA_DIR"]
+elif os.environ.get("SPEDAS_DATA_DIR"):
+    CONFIG["local_data_dir"] = os.sep.join([os.environ["SPEDAS_DATA_DIR"], "poes"])

pyspedas/projects/poes/load.py

@@ -5,79 +5,68 @@
 
 from .config import CONFIG
 
-def load(trange=['2018-11-5', '2018-11-6'], 
-         probe=['noaa19'],
-         instrument='sem', 
-         datatype='*', 
-         prefix='',
-         suffix='',
-         get_support_data=False, 
-         varformat=None,
-         varnames=[],
-         downloadonly=False,
-         notplot=False,
-         no_update=False,
-         time_clip=False,
-         force_download=False):
+
+def load(
+    trange=["2018-11-5", "2018-11-6"],
+    probe=["noaa19"],
+    instrument="sem",
+    datatype="*",
+    prefix="",
+    suffix="",
+    get_support_data=False,
+    ncei_server=False,
+    varformat=None,
+    varnames=[],
+    downloadonly=False,
+    notplot=False,
+    no_update=False,
+    time_clip=False,
+    force_download=False,
+):
     """
-    This function loads POES Space Environment Monitor data. This function is
-    not meant to be called directly; instead, see the wrapper:
-        pyspedas.poes.sem
+    Load POES Space Environment Monitor data.
 
     Parameters
     ----------
-        trange : list of str, default=['2018-11-5', '2018-11-6']
-            time range of interest [starttime, endtime] with the format
-            'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
-            ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
-
-        probe: str or list of str, default=['noaa19']
-            POES spacecraft name(s); e.g., metop1, metop2, noaa15, noaa16,
-            noaa18, noaa19
-
-        instrument: str, default='sem'
-             Name of the instrument.
-
-        datatype: str, default='*'
-            This variable is unused. It is reserved for the future use.
-
-        prefix: str
-            The tplot variable names will be given this prefix.  By default, no prefix is added.
-            Default: ''
-
-        suffix: str, optional
-            The tplot variable names will be given this suffix.  By default,
-            no suffix is added.
-
-        get_support_data: bool, default=False
-            Data with an attribute "VAR_TYPE" with a value of "support_data"
-            will be loaded into tplot.  By default, only loads in data with a
-            "VAR_TYPE" attribute of "data".
-
-        varformat: str, default=False
-            The file variable formats to load into tplot.  Wildcard character
-            "*" is accepted.  By default, all variables are loaded in.
-
-        varnames: list of str, optional
-            List of variable names to load (if not specified,
-            all data variables are loaded)
-
-        downloadonly: bool, default=False
-            Set this flag to download the CDF files, but not load them into
-            tplot variables
-
-        notplot: bool, default=False
-            Return the data in hash tables instead of creating tplot variables
-
-        no_update: bool, default=False
-            If set, only load data from your local cache
-
-        time_clip: bool, default=False
-            Time clip the variables to exactly the range specified in the trange keyword
-
-        force_download: bool
-            Download file even if local version is more recent than server version
-            Default: False
+    trange : list of str, optional
+        Time range of interest [starttime, endtime] with the format
+        ['YYYY-MM-DD', 'YYYY-MM-DD'] or to specify more or less than a day
+        ['YYYY-MM-DD/hh:mm:ss', 'YYYY-MM-DD/hh:mm:ss'].
+        Default is ['2018-11-5', '2018-11-6'].
+    probe : str or list of str, optional
+        POES spacecraft name(s); e.g., metop1, metop2, noaa15, noaa16,
+        noaa18, noaa19. Default is ['noaa19'].
+    instrument : str, optional
+        Name of the instrument. Default is 'sem'.
+    datatype : str, optional
+        This variable is unused. It is reserved for future use. Default is '*'.
+    prefix : str, optional
+        The tplot variable names will be given this prefix. By default, no prefix is added.
+        Default is ''.
+    suffix : str, optional
+        The tplot variable names will be given this suffix. By default, no suffix is added.
+    get_support_data : bool, optional
+        Data with an attribute "VAR_TYPE" with a value of "support_data"
+        will be loaded into tplot. By default, only loads in data with a
+        "VAR_TYPE" attribute of "data". Default is False.
+    ncei_server : bool, optional
+        Set this flag to download data from the NCEI NOAA server instead of the default SPDF server.
+        The NCEI server contains older data which is not available on the SPDF server. Default is False.
+    varformat : str, optional
+        The file variable formats to load into tplot. Wildcard character
+        "*" is accepted. By default, all variables are loaded in. Default is False.
+    varnames : list of str, optional
+        List of variable names to load (if not specified, all data variables are loaded).
+    downloadonly : bool, optional
+        Set this flag to download the CDF files, but not load them into tplot variables. Default is False.
+    notplot : bool, optional
+        Return the data in hash tables instead of creating tplot variables. Default is False.
+    no_update : bool, optional
+        If set, only load data from your local cache. Default is False.
+    time_clip : bool, optional
+        Time clip the variables to exactly the range specified in the trange keyword. Default is False.
+    force_download : bool, optional
+        Download file even if local version is more recent than server version. Default is False.
 
     Returns
     -------
@@ -86,22 +75,42 @@ def load(trange=['2018-11-5', '2018-11-6'],
 
     Examples
     --------
-    This function is not intended to be called directly.
+    >>> from pyspedas.poes.load import load
+    >>> vars = load(trange=['2013-11-5', '2013-11-6'])
+    >>> print(vars)
     """
 
     if not isinstance(probe, list):
         probe = [probe]
 
     out_files = []
+    tvars = []
 
     for prb in probe:
-        if instrument == 'sem':
-            pathformat = prb + '/sem2_fluxes-2sec/%Y/' + prb + '_poes-sem2_fluxes-2sec_%Y%m%d_v??.cdf'
+        if ncei_server:
+            remote_path = CONFIG["ncei_remote_data_dir"]
+            num = prb[-2:]
+            pathformat = "%Y/" + prb + "/poes_n" + num + "_%Y%m%d.cdf"
+        else:
+            remote_path = CONFIG["remote_data_dir"]
+            if instrument == "sem":
+                pathformat = (
+                    prb
+                    + "/sem2_fluxes-2sec/%Y/"
+                    + prb
+                    + "_poes-sem2_fluxes-2sec_%Y%m%d_v??.cdf"
+                )
 
         # find the full remote path names using the trange
         remote_names = dailynames(file_format=pathformat, trange=trange)
 
-        files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update, force_download=force_download)
+        files = download(
+            remote_file=remote_names,
+            remote_path=remote_path,
+            local_path=CONFIG["local_data_dir"],
+            no_download=no_update,
+            force_download=force_download,
+        )
         if files is not None:
             for file in files:
                 out_files.append(file)
@@ -111,13 +120,20 @@ def load(trange=['2018-11-5', '2018-11-6'],
     if downloadonly:
         return out_files
 
-    tvars = cdf_to_tplot(out_files, prefix=prefix, suffix=suffix, get_support_data=get_support_data, varformat=varformat, varnames=varnames, notplot=notplot)
+    tvars = cdf_to_tplot(
+        out_files,
+        prefix=prefix,
+        suffix=suffix,
+        get_support_data=get_support_data,
+        varformat=varformat,
+        varnames=varnames,
+        notplot=notplot,
+    )
 
     if notplot:
         return tvars
 
     if time_clip:
-        for new_var in tvars:
-            tclip(new_var, trange[0], trange[1], suffix='')
+        tclip(tvars, trange[0], trange[1], suffix="", overwrite=True)
 
     return tvars

pyspedas/projects/poes/sem.py

@@ -1,82 +1,6 @@
+from functools import update_wrapper
+from pyspedas.utilities.pyspedas_functools import better_partial
 from .load import load
 
-# This routine was originally in poes/__init__.py.
-def sem(trange=['2018-11-5', '2018-11-6'],
-        probe=['noaa19'],
-        datatype=None,
-        prefix='',
-        suffix='',
-        get_support_data=False,
-        varformat=None,
-        varnames=[],
-        downloadonly=False,
-        notplot=False,
-        no_update=False,
-        time_clip=False,
-        force_download=False):
-    """
-    This function loads POES Space Environment Monitor (SEM) data
-
-    Parameters
-    ----------
-        trange : list of str
-            time range of interest [starttime, endtime] with the format
-            ['YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
-            ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
-            Default: ['2018-11-5', '2018-11-6']
-
-        probe: str or list of str
-            POES spacecraft name(s); e.g., metop1, metop2, noaa15, noaa16,
-            noaa18, noaa19. Default: noaa19
-
-        datatype: str, optional
-            This variable is unused. It is reserved for the future use.
-
-        prefix: str
-            The tplot variable names will be given this prefix.  By default, no prefix is added.
-            Default: ''
-
-        suffix: str, optional
-            The tplot variable names will be given this suffix.  By default,
-            no suffix is added.
-
-        get_support_data: bool, default=False
-            If True, data with an attribute "VAR_TYPE" with a value of "support_data"
-            will be loaded into tplot. Default: False
-
-        varformat: str
-            If specified,  file variable formats to load into tplot.  Wildcard characters
-            `*` and `?` are accepted.  By default, all variables are loaded in.
-
-        varnames: list of str, optional
-            List of variable names to load (if not specified,
-            all data variables are loaded)
-
-        downloadonly: bool
-            Set this flag to download the CDF files, but not load them into
-            tplot variables. Default: False
-
-        notplot: bool
-            Return the data in hash tables instead of creating tplot variables. Default: False
-
-        no_update: bool
-            If set, only load data from your local cache. Default: False
-
-        time_clip: bool
-            Time clip the variables to exactly the range specified in the trange keyword. Default: False
-
-        force_download: bool
-            Download file even if local version is more recent than server version
-            Default: False
-
-    Returns
-    -------
-    dict or list
-        List of tplot variables created.
-
-    Examples
-    --------
-    >>> sem_vars = pyspedas.poes.sem(trange=['2013-11-5', '2013-11-6'])
-    >>> tplot('ted_ele_tel30_low_eflux')
-    """
-    return load(instrument='sem', probe=probe, trange=trange, datatype=datatype, prefix=prefix, suffix=suffix, get_support_data=get_support_data, varformat=varformat, varnames=varnames, downloadonly=downloadonly, notplot=notplot, time_clip=time_clip, no_update=no_update, force_download=force_download)
+sem = better_partial(load, instrument="sem")
+update_wrapper(sem, load)

pyspedas/projects/poes/tests/tests.py

@@ -3,6 +3,7 @@
 from pytplot import data_exists
 from pytplot import get_data
 import pyspedas
+from pyspedas.projects.poes.load import load
 
 
 class LoadTestCases(unittest.TestCase):
@@ -35,6 +36,10 @@ def test_downloadonly(self):
         files = pyspedas.poes.sem(downloadonly=True, probe='noaa19')
         self.assertTrue(os.path.exists(files[0]))
 
+    def test_ncei_server(self):
+        vars = load(trange=['1999-01-03', '1999-01-04'], probe=['noaa15'], ncei_server=True, time_clip=True)
+        self.assertTrue('geogLL' in vars)
+
 
 if __name__ == '__main__':
     unittest.main()