Cleaned and extended function that extracts datetimes from paths (#2181)

Co-authored-by: Valeriu Predoi <[email protected]>

esmvalcore/dataset.py

@@ -857,7 +857,7 @@ def _update_timerange(self):
             dataset.facets.pop('timerange')
             dataset.supplementaries = []
             check.data_availability(dataset)
-            intervals = [_get_start_end_date(f.name) for f in dataset.files]
+            intervals = [_get_start_end_date(f) for f in dataset.files]
 
             min_date = min(interval[0] for interval in intervals)
             max_date = max(interval[1] for interval in intervals)

esmvalcore/esgf/_search.py

@@ -168,7 +168,7 @@ def select_by_time(files, timerange):
     for file in files:
         start_date, end_date = _parse_period(timerange)
         try:
-            start, end = _get_start_end_date(file.name)
+            start, end = _get_start_end_date(file)
         except ValueError:
             # If start and end year cannot be read from the filename
             # just select everything.

esmvalcore/local.py

@@ -7,7 +7,7 @@
 import re
 from glob import glob
 from pathlib import Path
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, Union
 
 import iris
 import isodate
@@ -17,17 +17,19 @@
 from .exceptions import RecipeError
 from .typing import Facets, FacetValue
 
+if TYPE_CHECKING:
+    from .esgf import ESGFFile
+
 logger = logging.getLogger(__name__)
 
 
 def _get_from_pattern(pattern, date_range_pattern, stem, group):
     """Get time, date or datetime from date range patterns in file names."""
-    #
     # Next string allows to test that there is an allowed delimiter (or
     # string start or end) close to date range (or to single date)
     start_point = end_point = None
     context = r"(?:^|[-_]|$)"
-    #
+
     # First check for a block of two potential dates
     date_range_pattern_with_context = context + date_range_pattern + context
     daterange = re.search(date_range_pattern_with_context, stem)
@@ -37,6 +39,7 @@ def _get_from_pattern(pattern, date_range_pattern, stem, group):
         date_range_pattern_with_context = (context + date_range_pattern +
                                            context)
         daterange = re.search(date_range_pattern_with_context, stem)
+
     if daterange:
         start_point = daterange.group(group)
         end_group = '_'.join([group, 'end'])
@@ -59,41 +62,72 @@ def _get_from_pattern(pattern, date_range_pattern, stem, group):
     return start_point, end_point
 
 
-def _get_start_end_date(filename):
+def _get_start_end_date(
+    file: str | Path | LocalFile | ESGFFile
+) -> tuple[str, str]:
     """Get the start and end dates as a string from a file name.
 
-    Examples of allowed dates : 1980, 198001, 19801231,
-    1980123123, 19801231T23, 19801231T2359, 19801231T235959,
-    19801231T235959Z (ISO 8601).
+    Examples of allowed dates: 1980, 198001, 1980-01, 19801231, 1980-12-31,
+    1980123123, 19801231T23, 19801231T2359, 19801231T235959, 19801231T235959Z
+    (ISO 8601).
+
+    Dates must be surrounded by '-', '_' or '.' (the latter is used by CMIP3
+    data), or string start or string end (after removing filename suffix).
+
+    Look first for two dates separated by '-', '_' or '_cat_' (the latter is
+    used by CMIP3 data), then for one single date, and if there are multiple,
+    for one date at start or end.
+
+    Parameters
+    ----------
+    file:
+        The file to read the start and end data from.
+
+    Returns
+    -------
+    tuple[str, str]
+        The start and end date.
 
-    Dates must be surrounded by - or _ or string start or string end
-    (after removing filename suffix).
+    Raises
+    ------
+    ValueError
+        Start or end date cannot be determined.
 
-    Look first for two dates separated by - or _, then for one single
-    date, and if they are multiple, for one date at start or end.
     """
-    stem = Path(filename).stem
+    if hasattr(file, 'name'):  # Path, LocalFile, ESGFFile
+        stem = Path(file.name).stem
+    else:  # str
+        stem = Path(file).stem
+
     start_date = end_date = None
-    #
+
+    # Build regex
     time_pattern = (r"(?P<hour>[0-2][0-9]"
                     r"(?P<minute>[0-5][0-9]"
                     r"(?P<second>[0-5][0-9])?)?Z?)")
     date_pattern = (r"(?P<year>[0-9]{4})"
-                    r"(?P<month>[01][0-9]"
-                    r"(?P<day>[0-3][0-9]"
+                    r"(?P<month>-?[01][0-9]"
+                    r"(?P<day>-?[0-3][0-9]"
                     rf"(T?{time_pattern})?)?)?")
     datetime_pattern = (rf"(?P<datetime>{date_pattern})")
-    #
     end_datetime_pattern = datetime_pattern.replace(">", "_end>")
-    date_range_pattern = datetime_pattern + r"[-_]" + end_datetime_pattern
+
+    # Dates can either be delimited by '-', '_', or '_cat_' (the latter for
+    # CMIP3)
+    date_range_pattern = (
+        datetime_pattern + r"[-_](?:cat_)?" + end_datetime_pattern
+    )
+
+    # Find dates using the regex
     start_date, end_date = _get_from_pattern(datetime_pattern,
                                              date_range_pattern, stem,
                                              'datetime')
 
     # As final resort, try to get the dates from the file contents
-    if (start_date is None or end_date is None) and Path(filename).exists():
-        logger.debug("Must load file %s for daterange ", filename)
-        cubes = iris.load(filename)
+    if ((start_date is None or end_date is None) and
+            isinstance(file, (str, Path)) and Path(file).exists()):
+        logger.debug("Must load file %s for daterange ", file)
+        cubes = iris.load(file)
 
         for cube in cubes:
             logger.debug(cube)
@@ -109,12 +143,30 @@ def _get_start_end_date(filename):
             break
 
     if start_date is None or end_date is None:
-        raise ValueError(f'File {filename} dates do not match a recognized '
-                         'pattern and time can not be read from the file')
+        raise ValueError(
+            f"File {file} datetimes do not match a recognized pattern and "
+            f"time coordinate can not be read from the file"
+        )
+
+    # Remove potential '-' characters from datetimes
+    start_date = start_date.replace('-', '')
+    end_date = end_date.replace('-', '')
 
     return start_date, end_date
 
 
+def _get_start_end_year(
+    file: str | Path | LocalFile | ESGFFile
+) -> tuple[int, int]:
+    """Get the start and end year as int from a file name.
+
+    See :func:`_get_start_end_date`.
+
+    """
+    (start_date, end_date) = _get_start_end_date(file)
+    return (int(start_date[:4]), int(end_date[:4]))
+
+
 def _dates_to_timerange(start_date, end_date):
     """Convert ``start_date`` and ``end_date`` to ``timerange``.
 
@@ -162,72 +214,6 @@ def _replace_years_with_timerange(variable):
     variable.pop('end_year', None)
 
 
-def _get_start_end_year(file):
-    """Get the start and end year from a file name.
-
-    Examples of allowed dates : 1980, 198001, 19801231,
-    1980123123, 19801231T23, 19801231T2359, 19801231T235959,
-    19801231T235959Z (ISO 8601).
-
-    Dates must be surrounded by - or _ or string start or string end
-    (after removing filename suffix).
-
-    Look first for two dates separated by - or _, then for one single
-    date, and if they are multiple, for one date at start or end.
-
-    Parameters
-    ----------
-    file: LocalFile or esmvalcore.esgf.ESGFFile
-        The file to read the start and end year from.
-
-    Returns
-    -------
-    tuple[int, int]
-        The start and end year.
-
-    Raises
-    ------
-    ValueError
-        When start or end year cannot be determined.
-
-    """
-    start_year = end_year = None
-
-    time_pattern = (r"(?P<hour>[0-2][0-9]"
-                    r"(?P<minute>[0-5][0-9]"
-                    r"(?P<second>[0-5][0-9])?)?Z?)")
-    date_pattern = (r"(?P<year>[0-9]{4})"
-                    r"(?P<month>[01][0-9]"
-                    r"(?P<day>[0-3][0-9]"
-                    rf"(T?{time_pattern})?)?)?")
-
-    end_date_pattern = date_pattern.replace(">", "_end>")
-    date_range_pattern = date_pattern + r"[-_]" + end_date_pattern
-    start_year, end_year = _get_from_pattern(date_pattern, date_range_pattern,
-                                             Path(file.name).stem, 'year')
-    # As final resort, try to get the dates from the file contents
-    if ((start_year is None or end_year is None) and isinstance(file, Path)
-            and file.exists()):
-        logger.debug("Must load file %s for daterange ", file)
-        cubes = iris.load(file)
-
-        for cube in cubes:
-            logger.debug(cube)
-            try:
-                time = cube.coord('time')
-            except iris.exceptions.CoordinateNotFoundError:
-                continue
-            start_year = time.cell(0).point.year
-            end_year = time.cell(-1).point.year
-            break
-
-    if start_year is None or end_year is None:
-        raise ValueError(f'File {file} dates do not match a recognized '
-                         'pattern and time can not be read from the file')
-
-    return int(start_year), int(end_year)
-
-
 def _parse_period(timerange):
     """Parse `timerange` values given as duration periods.