29 clip osm

.github/workflows/python-package.yml

@@ -40,13 +40,21 @@ jobs:
     - name: Check Java Install
       run: |
         java --version
+    - name: Install Osmosis
+      uses: ConorMacBride/install-package@v1
+      with:
+        brew: osmosis
+        apt: osmosis
+    - name: Run Integration Tests
+      run: |
+          pytest -m runinteg --runinteg # run only those tests marked runinteg
     - name: pre-commit
       run: |
         pre-commit install
         pre-commit run --all-files
     - name: Test with pytest
       run: |
-        pytest --runsetup
+        pytest
     - name: Generate Report
       run: |
         coverage run -m pytest

pipeline/gtfs/01-validate-gtfs.py

@@ -16,7 +16,7 @@
 import subprocess
 
 from transport_performance.gtfs.validation import GtfsInstance
-from transport_performance.utils.defence import _is_gtfs_pth
+from transport_performance.utils.defence import _is_expected_filetype
 
 CONFIG = toml.load(here("pipeline/gtfs/config/01-validate-gtfs.toml"))
 GTFS_PTH = here(CONFIG["GTFS"]["PATH"])
@@ -26,7 +26,7 @@
 HULL_MAP_PATH = CONFIG["MAPS"]["STOP_HULL_PTH"]
 PROFILING = CONFIG["UTILS"]["PROFILING"]
 # check GTFS Path exists
-_is_gtfs_pth(pth=GTFS_PTH, param_nm="GTFS_PTH", check_existing=True)
+_is_expected_filetype(pth=GTFS_PTH, param_nm="GTFS_PTH", check_existing=True)
 # Get the disk usage of the GTFS file.
 gtfs_du = (
     subprocess.check_output(["du", "-sh", GTFS_PTH]).split()[0].decode("utf-8")

pipeline/gtfs/config/01-validate-gtfs.toml

@@ -1,7 +1,7 @@
 title = "Config for GTFS Validation Pipeline"
 
 [GTFS]
-PATH = "data/external/croppednewport-bus-07-07-2022_gtfs.zip"
+PATH = "tests/data/newport-20230613_gtfs.zip"
 UNITS = "m"
 GEOMETRIC_CRS = 27700 # used for area calculations only
 

requirements.txt

@@ -9,8 +9,8 @@ beautifulsoup4
 requests
 pytest-mock
 toml
-rasterio
 pyprojroot
+rasterio
 matplotlib
 plotly
 nbformat>=4.2.0

src/transport_performance/gtfs/utils.py → src/transport_performance/gtfs/gtfs_utils.py

@@ -4,7 +4,10 @@
 from shapely.geometry import box
 from pyprojroot import here
 
-from transport_performance.utils.defence import _is_gtfs_pth, _check_list
+from transport_performance.utils.defence import (
+    _is_expected_filetype,
+    _check_list,
+)
 
 
 def bbox_filter_gtfs(
@@ -38,8 +41,10 @@ def bbox_filter_gtfs(
     None
 
     """
-    _is_gtfs_pth(pth=in_pth, param_nm="in_pth")
-    _is_gtfs_pth(pth=out_pth, param_nm="out_pth", check_existing=False)
+    _is_expected_filetype(pth=in_pth, param_nm="in_pth")
+    _is_expected_filetype(
+        pth=out_pth, param_nm="out_pth", check_existing=False
+    )
     _check_list(ls=bbox_list, param_nm="bbox_list", exp_type=float)
     for param in [units, crs]:
         if not isinstance(param, str):

src/transport_performance/gtfs/routes.py

@@ -98,7 +98,7 @@ def scrape_route_type_lookup(
     for url in [gtfs_url, ext_spec_url]:
         _url_defence(url)
 
-    _bool_defence(extended_schema)
+    _bool_defence(extended_schema, "extended_schema")
     # Get the basic scheme lookup
     resp_txt = _get_response_text(gtfs_url)
     soup = BeautifulSoup(resp_txt, "html.parser")

src/transport_performance/gtfs/validation.py

@@ -11,7 +11,7 @@
 
 from transport_performance.gtfs.routes import scrape_route_type_lookup
 from transport_performance.utils.defence import (
-    _is_gtfs_pth,
+    _is_expected_filetype,
     _check_namespace_export,
     _check_parent_dir_exists,
 )
@@ -91,7 +91,7 @@ class GtfsInstance:
     def __init__(
         self, gtfs_pth=here("tests/data/newport-20230613_gtfs.zip"), units="m"
     ):
-        _is_gtfs_pth(pth=gtfs_pth, param_nm="gtfs_pth")
+        _is_expected_filetype(pth=gtfs_pth, param_nm="gtfs_pth")
 
         # validate units param
         if not isinstance(units, str):

src/transport_performance/osm/osm_utils.py

@@ -0,0 +1,108 @@
+"""Utility functions for OSM files."""
+import subprocess
+from pyprojroot import here
+
+from transport_performance.utils.defence import (
+    _bool_defence,
+    _check_list,
+    _check_parent_dir_exists,
+    _is_expected_filetype,
+)
+
+
+def filter_osm(
+    pbf_pth=here("tests/data/newport-2023-06-13.osm.pbf"),
+    out_pth="filtered.osm.pbf",
+    bbox=[-3.01, 51.58, -2.99, 51.59],
+    tag_filter=True,
+    install_osmosis=False,
+):
+    """Filter an osm.pbf file to a bbox. Relies on homebrew with osmosis.
+
+    Parameters
+    ----------
+    pbf_pth: ((str, pathlib.PosixPath), optional)
+        Path to the open street map pbf to be filtered. Defaults to
+        here("tests/data/newport-2023-06-13.osm.pbf").
+    out_pth: ((str, pathlib.PosixPath), optional)
+        Path to write to. Defaults to "filtered.osm.pbf".
+    bbox:  (list, optional)
+        Bounding box used to perform the filter, in left, bottom, right top
+        order. Defaults to [-3.01, 51.58, -2.99, 51.59].
+    tag_filter: (bool, optional)
+        Should non-highway ways be filtered? Excludes waterway, landuse &
+        natural. Defaults to True.
+    install_osmosis: (bool, optional)
+        Should brew be used to install osmosis if not found. Defaults to False.
+
+    Raises
+    ------
+        Exception: Subprocess error.
+        Exception: Osmosis not found. Will not raise if install_osmosis=True.
+
+    Returns
+    -------
+        None
+
+    """
+    # defence
+    _is_expected_filetype(pbf_pth, param_nm="pbf_pth", exp_ext=".pbf")
+    _is_expected_filetype(
+        out_pth, param_nm="out_pth", exp_ext=".pbf", check_existing=False
+    )
+    for nm, val in {
+        "tag_filter": tag_filter,
+        "install_osmosis": install_osmosis,
+    }.items():
+        _bool_defence(val, param_nm=nm)
+    # check bbox values makes sense, else osmosis will error
+    if not bbox[0] < bbox[2]:
+        raise ValueError(
+            (
+                f"Bounding box longitude West {bbox[0]}"
+                f" is not smaller than East {bbox[2]}"
+            )
+        )
+    elif not bbox[1] < bbox[3]:
+        raise ValueError(
+            (
+                f"Bounding box latitude South {bbox[1]}"
+                f" is not smaller than North {bbox[3]}"
+            )
+        )
+
+    _check_list(bbox, param_nm="bbox", check_elements=True, exp_type=float)
+    _check_parent_dir_exists(out_pth, param_nm="out_pth", create=True)
+    # Compile the osmosis command
+    cmd = [
+        "osmosis",
+        "--read-pbf",
+        pbf_pth.as_posix(),
+        "--bounding-box",
+        f"left={bbox[0]}",
+        f"bottom={bbox[1]}",
+        f"right={bbox[2]}",
+        f"top={bbox[3]}",
+    ]
+    if tag_filter:  # optionaly filter ways
+        print("Rejecting ways:  waterway, landuse & natural.")
+        cmd.extend(["--tf", "reject-ways", "waterway=* landuse=* natural=*"])
+
+    cmd.extend(["--used-node", "--write-pbf", out_pth])
+
+    try:
+        subprocess.run(cmd, check=True)
+        print(f"Filter completed. Written to {out_pth}")
+    except subprocess.CalledProcessError as e1:
+        raise Exception(f"Error executing osmosis command: {e1}")
+    except FileNotFoundError as e2:
+        if install_osmosis:
+            print(f"osmosis command was not recognised: {e2}. Trying install.")
+            subprocess.run(["brew", "install", "osmosis"])
+            print("Installation of `osmosis successful.`")
+            subprocess.run(cmd, check=True)
+            print(f"Retry filter pbf completed. Written to {out_pth}")
+        else:
+            raise Exception("`osmosis` is not found. Please install.")
+
+    return None

src/transport_performance/utils/defence.py

@@ -30,6 +30,8 @@ def _is_path_like(pth, param_nm):
 
 def _check_parent_dir_exists(pth, param_nm, create=False):
     _is_path_like(pth, param_nm)
+    # realpath helps to catch cases where relative paths are passed in main
+    pth = os.path.realpath(pth)
     parent = os.path.dirname(pth)
     if not os.path.exists(parent):
         if create:
@@ -43,8 +45,8 @@ def _check_parent_dir_exists(pth, param_nm, create=False):
     return None
 
 
-def _is_gtfs_pth(pth, param_nm, check_existing=True):
-    """Handle file paths that should be existing GTFS feeds.
+def _is_expected_filetype(pth, param_nm, check_existing=True, exp_ext=".zip"):
+    """Handle file paths that should be existing filetypes.
 
     Parameters
     ----------
@@ -53,13 +55,15 @@ def _is_gtfs_pth(pth, param_nm, check_existing=True):
     param_nm : str
         The name of the parameter being tested. Helps with debugging.
     check_existing : bool
-        Whether to check if the GTFS file already exists. Defaults to True.
+        Whether to check if the filetype file already exists. Defaults to True.
+    exp_ext: str
+        The expected filetype.
 
     Raises
     ------
     TypeError: `pth` is not either of string or pathlib.PosixPath.
     FileExistsError: `pth` does not exist on disk.
-    ValueError: `pth` does not have a `.zip` file extension.
+    ValueError: `pth` does not have the expected file extension.
 
     Returns
     -------
@@ -71,9 +75,9 @@ def _is_gtfs_pth(pth, param_nm, check_existing=True):
     _, ext = os.path.splitext(pth)
     if check_existing and not os.path.exists(pth):
         raise FileExistsError(f"{pth} not found on file.")
-    if ext != ".zip":
+    if ext != exp_ext:
         raise ValueError(
-            f"`gtfs_pth` expected a zip file extension. Found {ext}"
+            f"`{param_nm}` expected file extension {exp_ext}. Found {ext}"
         )
 
     return None
@@ -109,11 +113,11 @@ def _url_defence(url):
     return None
 
 
-def _bool_defence(some_bool):
+def _bool_defence(some_bool, param_nm):
     """Defence checking. Not exported."""
     if not isinstance(some_bool, bool):
         raise TypeError(
-            f"`extended_schema` expected boolean. Got {type(some_bool)}"
+            f"`{param_nm}` expected boolean. Got {type(some_bool)}"
         )
 
     return None

tests/gtfs/test_utils.py → tests/gtfs/test_gtfs_utils.py

@@ -4,7 +4,7 @@
 import os
 import pytest
 
-from transport_performance.gtfs.utils import bbox_filter_gtfs
+from transport_performance.gtfs.gtfs_utils import bbox_filter_gtfs
 from transport_performance.gtfs.validation import GtfsInstance
 
 

tests/gtfs/test_validation.py

@@ -40,7 +40,7 @@ def test_init_defensive_behaviours(self):
         #  a case where file is found but not a zip directory
         with pytest.raises(
             ValueError,
-            match=r"`gtfs_pth` expected a zip file extension. Found .pbf",
+            match=r"`gtfs_pth` expected file extension .zip. Found .pbf",
         ):
             GtfsInstance(
                 gtfs_pth=here("tests/data/newport-2023-06-13.osm.pbf")