New "Geo" optional dependencies

#424

.github/workflows/test.yml

@@ -15,15 +15,15 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64
-    - uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_dev.txt') }}-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - run: pip install --upgrade -r requirements_dev.txt
+    - name: Install normal dependencies
+      run: pip install -r requirements_dev.txt
     - run: pip install 'jsonref${{ matrix.jsonref-version }}'
-    - run: py.test --cov .
+    - name: Test with normal dependencies
+      run: py.test -m "not geo" --cov .
+    - name: Install geo dependencies
+      run: pip install -e .[Geo]
+    - name: Test with geo dependencies
+      run: py.test -m "geo" --cov .
     - env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: coveralls --service=github

CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Changed
+
+- New "Geo" optional python dependency and some existing python dependencies moved to it.
+  If you were using this functionality before, you'll need to start installing "Geo"  to get same behaviour.
+  https://github.com/OpenDataServices/flatten-tool/issues/424
+
 ## [0.23.0] - 2023-08-30
 
 ### Changed

flattentool/input.py

@@ -13,10 +13,17 @@
 from decimal import Decimal, InvalidOperation
 from warnings import warn
 
-import geojson
 import openpyxl
 import pytz
-import shapely.wkt
+
+try:
+    import geojson
+    import shapely.wkt
+
+    SHAPELY_AND_GEOJSON_LIBRARIES_AVAILABLE = True
+except ImportError:
+    SHAPELY_AND_GEOJSON_LIBRARIES_AVAILABLE = False
+
 from openpyxl.utils.cell import _get_column_letter
 
 from flattentool.exceptions import DataErrorWarning
@@ -105,7 +112,11 @@ def convert_type(type_string, value, timezone=pytz.timezone("UTC"), convert_flag
         if type(value) == datetime.datetime:
             return value.date().isoformat()
         return str(value)
-    elif convert_flags.get("wkt") and type_string == "geojson":
+    elif (
+        convert_flags.get("wkt")
+        and type_string == "geojson"
+        and SHAPELY_AND_GEOJSON_LIBRARIES_AVAILABLE
+    ):
         try:
             geom = shapely.wkt.loads(value)
         except shapely.errors.GEOSException as e:

flattentool/json_input.py

@@ -16,7 +16,13 @@
 
 import BTrees.OOBTree
 import ijson
-import shapely.geometry
+
+try:
+    import shapely.wkt
+
+    SHAPELY_LIBRARY_AVAILABLE = True
+except ImportError:
+    SHAPELY_LIBRARY_AVAILABLE = False
 import transaction
 import xmltodict
 import zc.zlibstorage
@@ -362,6 +368,7 @@ def parse_json_dict(
             self.convert_flags.get("wkt")
             and "type" in json_dict
             and "coordinates" in json_dict
+            and SHAPELY_LIBRARY_AVAILABLE
         ):
             _sheet_key = sheet_key(sheet, parent_name.strip("/"))
             try:

flattentool/tests/test_docs.py

@@ -8,11 +8,13 @@
 import pytest
 
 examples_in_docs_data = []
+examples_in_docs_data_geo = []
 
 
 def _get_examples_in_docs_data():
-    global examples_in_docs_data
+    global examples_in_docs_data, examples_in_docs_data_geo
     examples_in_docs_data = []
+    examples_in_docs_data_geo = []
     for root, dirs, files in os.walk("examples"):
         for filename in files:
             if root.startswith("examples/help/") and sys.version_info[:2] != (3, 8):
@@ -21,7 +23,10 @@ def _get_examples_in_docs_data():
                 # (Same as we lint code with, so dev's can have one virtual env)
                 continue
             if "cmd.txt" in filename:
-                examples_in_docs_data.append((root, filename))
+                if root.startswith("examples/wkt"):
+                    examples_in_docs_data_geo.append((root, filename))
+                else:
+                    examples_in_docs_data.append((root, filename))
 
 
 _get_examples_in_docs_data()
@@ -45,6 +50,17 @@ def test_examples_receipt():
 
 @pytest.mark.parametrize("root, filename", examples_in_docs_data)
 def test_example_in_doc(root, filename):
+    _test_example_in_doc_worker(root, filename)
+
+
+@pytest.mark.parametrize("root, filename", examples_in_docs_data_geo)
+@pytest.mark.geo
+def test_example_in_doc_geo(root, filename):
+    _test_example_in_doc_worker(root, filename)
+
+
+@pytest.mark.parametrize("root, filename", examples_in_docs_data)
+def _test_example_in_doc_worker(root, filename):
     if os.path.exists(join(root, "actual")) and os.path.isdir(join(root, "actual")):
         os.rename(join(root, "actual"), join(root, "actual." + str(uuid.uuid4())))
     os.mkdir(join(root, "actual"))
@@ -141,7 +157,7 @@ def test_expected_number_of_examples_in_docs_data():
     if sys.version_info[:2] != (3, 8):
         expected -= 3
         # number of help tests
-    assert len(examples_in_docs_data) == expected
+    assert len(examples_in_docs_data) + len(examples_in_docs_data_geo) == expected
 
 
 def _simplify_warnings(lines):

flattentool/tests/test_input_SpreadsheetInput.py

@@ -533,6 +533,7 @@ def __str__(self):
     assert len(recwarn) == 0
 
 
+@pytest.mark.geo
 def test_convert_type_geojson(recwarn):
     assert convert_type(
         "geojson", "POINT (53.486434 -2.239353)", convert_flags={"wkt": True}

flattentool/tests/test_input_SpreadsheetInput_unflatten.py

@@ -527,6 +527,9 @@ def inject_root_id(root_id, d):
         False,
         True,
     ),
+]
+
+testdata_geo = [
     (
         "WKT->geojson conversion",
         [
@@ -1095,6 +1098,7 @@ def create_schema(root_id):
     ),
 ]
 
+
 ROOT_ID_PARAMS = [
     ("ocid", {}),  # If not root_id kwarg is passed, then a root_id of ocid is assumed
     ("ocid", {"root_id": "ocid"}),
@@ -1124,6 +1128,70 @@ def test_unflatten(
     warning_messages,
     reversible,
     works_without_schema,
+):
+    _test_unflatten_worker(
+        convert_titles,
+        use_schema,
+        root_id,
+        root_id_kwargs,
+        input_list,
+        expected_output_list,
+        recwarn,
+        comment,
+        warning_messages,
+        reversible,
+        works_without_schema,
+    )
+
+
+@pytest.mark.parametrize("convert_titles", [True, False])
+@pytest.mark.parametrize("use_schema", [True, False])
+@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)
+@pytest.mark.parametrize(
+    "comment,input_list,expected_output_list,warning_messages,reversible,works_without_schema",
+    testdata_geo,
+)
+@pytest.mark.geo
+def test_unflatten_geo(
+    convert_titles,
+    use_schema,
+    root_id,
+    root_id_kwargs,
+    input_list,
+    expected_output_list,
+    recwarn,
+    comment,
+    warning_messages,
+    reversible,
+    works_without_schema,
+):
+    _test_unflatten_worker(
+        convert_titles,
+        use_schema,
+        root_id,
+        root_id_kwargs,
+        input_list,
+        expected_output_list,
+        recwarn,
+        comment,
+        warning_messages,
+        reversible,
+        works_without_schema,
+    )
+
+
+def _test_unflatten_worker(
+    convert_titles,
+    use_schema,
+    root_id,
+    root_id_kwargs,
+    input_list,
+    expected_output_list,
+    recwarn,
+    comment,
+    warning_messages,
+    reversible,
+    works_without_schema,
 ):
     if not use_schema and not works_without_schema:
         pytest.skip()

flattentool/tests/test_json_input.py

@@ -852,6 +852,7 @@ def test_parse_ids_nested(self):
         ]
 
 
+@pytest.mark.geo
 def test_parse_geojson():
     parser = JSONParser(
         root_json_dict=[
@@ -900,6 +901,7 @@ def test_parse_geojson():
     ]
 
 
+@pytest.mark.geo
 def test_parse_bad_geojson(recwarn):
     parser = JSONParser(
         root_json_dict=[

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.pytest.ini_options]
+markers = [
+    "geo: tests that need extra geo dependencies (deselect with '-m \"not geo\"')",
+]

setup.py

@@ -39,8 +39,6 @@ def run(self):
     "zodb",
     "zc.zlibstorage",
     "ijson",
-    "shapely",
-    "geojson",
 ]
 
 setup(
@@ -54,7 +52,7 @@ def run(self):
     license="MIT",
     description="Tools for generating CSV and other flat versions of the structured data",
     install_requires=install_requires,
-    extras_require={"HTTP": ["requests"]},
+    extras_require={"HTTP": ["requests"], "Geo": ["shapely", "geojson"]},
     cmdclass={
         "install": InstallWithCompile,
         "develop": DevelopWithCompile,