Extend use of custom result formatters to CLI tool

docs/customize/Result-Formatting.md

@@ -66,6 +66,8 @@ For example, let us extend the result for the status call in text format
 and add the server URL. Such a formatter would look like this:
 
 ``` python
+from nominatim_api import StatusResult
+
 @dispatch.format_func(StatusResult, 'text')
 def _format_status_text(result, _):
     header = 'Status for server nominatim.openstreetmap.org'
@@ -86,19 +88,39 @@ as adding formatting functions for all result types using the custom
 format name:
 
 ``` python
+from nominatim_api import StatusResult
+
 @dispatch.format_func(StatusResult, 'chatty')
 def _format_status_text(result, _):
     if result.status:
         return f"The server is currently not running. {result.message}"
 
-    return f"Good news! The server is running just fine."
+    return "Good news! The server is running just fine."
 ```
 
 That's all. Nominatim will automatically pick up the new format name and
-will allow the user to use it. Make sure to really define formatters for
-**all** result types. If they are for endpoints that you do not intend to
-use, you can simply return some static string but the function needs to be
-there.
+will allow the user to use it. There is no need to implement formatter
+functions for all the result types, when you invent a new one. The
+available formats will be determined for each API endpoint separately.
+To find out which formats are available, you can use the `--list-formats`
+option of the CLI tool:
+
+```
+me@machine:planet-project$ nominatim status --list-formats
+2024-08-16 19:54:00: Using project directory: /home/nominatim/planet-project
+text
+json
+chatty
+debug
+me@machine:planet-project$
+```
+
+The `debug` format listed in the last line will always appear. It is a
+special format that enables debug output via the command line (the same
+as the `debug=1` parameter enables for the server API). To not clash
+with this built-in function, you shouldn't name your own format 'debug'.
+
+### Content type of new formats
 
 All responses will be returned with the content type application/json by
 default. If your format produces a different content type, you need
@@ -117,6 +139,67 @@ The `content_types` module used above provides constants for the most
 frequent content types. You set the content type to an arbitrary string,
 if the content type you need is not available.
 
+## Formatting error messages
+
+Any exception thrown during processing of a request is given to
+a special error formatting function. It takes the requested content type,
+the status code and the error message. It should return the error message
+in a form appropriate for the given content type.
+
+You can overwrite the default formatting function with the decorator
+`error_format_func`:
+
+``` python
+import nominatim_api.server.content_types as ct
+
+@dispatch.error_format_func
+def _format_error(content_type: str, msg: str, status: int) -> str:
+    if content_type == ct.CONTENT_XML:
+        return f"""<?xml version="1.0" encoding="UTF-8" ?>
+                     <message>{msg}</message>
+                """
+    if content_type == ct.CONTENT_JSON:
+        return f'"{msg}"'
+
+    return f"ERROR: {msg}"
+```
+
+
+## Debugging custom formatters
+
+The easiest way to try out your custom formatter is by using the Nominatim
+CLI commands. Custom formats can be chosen with the `--format` parameter:
+
+```
+me@machine:planet-project$ nominatim status --format chatty
+2024-08-16 19:54:00: Using project directory: /home/nominatim/planet-project
+Good news! The server is running just fine.
+me@machine:planet-project$
+```
+
+They will also emit full error messages when there is a problem with the
+code you need to debug.
+
+!!! danger
+    In some cases, when you make an error with your import statement, the
+    CLI will not give you an error but instead tell you, that the API
+    commands are no longer available:
+
+        me@machine: nominatim status
+        usage: nominatim [-h] [--version] {import,freeze,replication,special-phrases,add-data,index,refresh,admin} ...
+        nominatim: error: argument subcommand: invalid choice: 'status'
+
+    This happens because the CLI tool is meant to still work when the
+    nominatim-api package is not installed. Import errors involving
+    `nominatim_api` are interpreted as "package not installed".
+
+    Use the help command to find out which is the offending import that
+    could not be found:
+
+        me@machine: nominatim -h
+        ... [other help text] ...
+        Nominatim API package not found (was looking for module: nominatim_api.xxx).
+
 ## Reference
 
 ### FormatDispatcher

src/nominatim_api/__init__.py

@@ -39,6 +39,7 @@
                       SearchResult as SearchResult,
                       SearchResults as SearchResults)
 from .localization import (Locales as Locales)
-from .result_formatting import (FormatDispatcher as FormatDispatcher)
+from .result_formatting import (FormatDispatcher as FormatDispatcher,
+                                load_format_dispatcher as load_format_dispatcher)
 
 from .version import NOMINATIM_API_VERSION as __version__

src/nominatim_api/v1/__init__.py

@@ -11,9 +11,3 @@
 #pylint: disable=useless-import-alias
 
 from .server_glue import ROUTES as ROUTES
-
-from . import format as _format
-
-list_formats = _format.dispatch.list_formats
-supports_format = _format.dispatch.supports_format
-format_result = _format.dispatch.format_result

src/nominatim_db/cli.py

@@ -243,7 +243,8 @@ def get_set_parser() -> CommandlineParser:
             raise ex
 
         parser.parser.epilog = \
-            '\n\nNominatim API package not found. The following commands are not available:'\
+            f'\n\nNominatim API package not found (was looking for module: {ex.name}).'\
+            '\nThe following commands are not available:'\
             '\n    export, convert, serve, search, reverse, lookup, details, status'\
             "\n\nRun 'pip install nominatim-api' to install the package."
 

src/nominatim_db/clicmd/args.py

@@ -137,6 +137,7 @@ class NominatimArgs:
 
     # Arguments to all query functions
     format: str
+    list_formats: bool
     addressdetails: bool
     extratags: bool
     namedetails: bool

test/python/api/test_result_formatting_v1.py

@@ -15,38 +15,38 @@
 
 import pytest
 
-import nominatim_api.v1 as api_impl
+from nominatim_api.v1.format import dispatch as v1_format
 import nominatim_api as napi
 
 STATUS_FORMATS = {'text', 'json'}
 
 # StatusResult
 
 def test_status_format_list():
-    assert set(api_impl.list_formats(napi.StatusResult)) == STATUS_FORMATS
+    assert set(v1_format.list_formats(napi.StatusResult)) == STATUS_FORMATS
 
 
 @pytest.mark.parametrize('fmt', list(STATUS_FORMATS))
 def test_status_supported(fmt):
-    assert api_impl.supports_format(napi.StatusResult, fmt)
+    assert v1_format.supports_format(napi.StatusResult, fmt)
 
 
 def test_status_unsupported():
-    assert not api_impl.supports_format(napi.StatusResult, 'gagaga')
+    assert not v1_format.supports_format(napi.StatusResult, 'gagaga')
 
 
 def test_status_format_text():
-    assert api_impl.format_result(napi.StatusResult(0, 'message here'), 'text', {}) == 'OK'
+    assert v1_format.format_result(napi.StatusResult(0, 'message here'), 'text', {}) == 'OK'
 
 
 def test_status_format_text():
-    assert api_impl.format_result(napi.StatusResult(500, 'message here'), 'text', {}) == 'ERROR: message here'
+    assert v1_format.format_result(napi.StatusResult(500, 'message here'), 'text', {}) == 'ERROR: message here'
 
 
 def test_status_format_json_minimal():
     status = napi.StatusResult(700, 'Bad format.')
 
-    result = api_impl.format_result(status, 'json', {})
+    result = v1_format.format_result(status, 'json', {})
 
     assert result == \
            f'{{"status":700,"message":"Bad format.","software_version":"{napi.__version__}"}}'
@@ -57,7 +57,7 @@ def test_status_format_json_full():
     status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc)
     status.database_version = '5.6'
 
-    result = api_impl.format_result(status, 'json', {})
+    result = v1_format.format_result(status, 'json', {})
 
     assert result == \
            f'{{"status":0,"message":"OK","data_updated":"2010-02-07T20:20:03+00:00","software_version":"{napi.__version__}","database_version":"5.6"}}'
@@ -70,7 +70,7 @@ def test_search_details_minimal():
                                  ('place', 'thing'),
                                  napi.Point(1.0, 2.0))
 
-    result = api_impl.format_result(search, 'json', {})
+    result = v1_format.format_result(search, 'json', {})
 
     assert json.loads(result) == \
            {'category': 'place',
@@ -114,7 +114,7 @@ def test_search_details_full():
                   )
     search.localize(napi.Locales())
 
-    result = api_impl.format_result(search, 'json', {})
+    result = v1_format.format_result(search, 'json', {})
 
     assert json.loads(result) == \
            {'place_id': 37563,
@@ -153,7 +153,7 @@ def test_search_details_no_geometry(gtype, isarea):
                                napi.Point(1.0, 2.0),
                                geometry={'type': gtype})
 
-    result = api_impl.format_result(search, 'json', {})
+    result = v1_format.format_result(search, 'json', {})
     js = json.loads(result)
 
     assert js['geometry'] == {'type': 'Point', 'coordinates': [1.0, 2.0]}
@@ -166,7 +166,7 @@ def test_search_details_with_geometry():
                                  napi.Point(1.0, 2.0),
                                  geometry={'geojson': '{"type":"Point","coordinates":[56.947,-87.44]}'})
 
-    result = api_impl.format_result(search, 'json', {})
+    result = v1_format.format_result(search, 'json', {})
     js = json.loads(result)
 
     assert js['geometry'] == {'type': 'Point', 'coordinates': [56.947, -87.44]}
@@ -178,7 +178,7 @@ def test_search_details_with_icon_available():
                                  ('amenity', 'restaurant'),
                                  napi.Point(1.0, 2.0))
 
-    result = api_impl.format_result(search, 'json', {'icon_base_url': 'foo'})
+    result = v1_format.format_result(search, 'json', {'icon_base_url': 'foo'})
     js = json.loads(result)
 
     assert js['icon'] == 'foo/food_restaurant.p.20.png'
@@ -189,7 +189,7 @@ def test_search_details_with_icon_not_available():
                                  ('amenity', 'tree'),
                                  napi.Point(1.0, 2.0))
 
-    result = api_impl.format_result(search, 'json', {'icon_base_url': 'foo'})
+    result = v1_format.format_result(search, 'json', {'icon_base_url': 'foo'})
     js = json.loads(result)
 
     assert 'icon' not in js
@@ -212,7 +212,7 @@ def test_search_details_with_address_minimal():
                                                     distance=0.0)
                                  ])
 
-    result = api_impl.format_result(search, 'json', {})
+    result = v1_format.format_result(search, 'json', {})
     js = json.loads(result)
 
     assert js['address'] == [{'localname': '',
@@ -245,7 +245,7 @@ def test_search_details_with_further_infos(field, outfield):
                                              distance=0.034)
                             ])
 
-    result = api_impl.format_result(search, 'json', {})
+    result = v1_format.format_result(search, 'json', {})
     js = json.loads(result)
 
     assert js[outfield] == [{'localname': 'Trespass',
@@ -279,7 +279,7 @@ def test_search_details_grouped_hierarchy():
                                              distance=0.034)
                                      ])
 
-    result = api_impl.format_result(search, 'json', {'group_hierarchy': True})
+    result = v1_format.format_result(search, 'json', {'group_hierarchy': True})
     js = json.loads(result)
 
     assert js['hierarchy'] == {'note': [{'localname': 'Trespass',
@@ -303,7 +303,7 @@ def test_search_details_keywords_name():
                                      napi.WordInfo(23, 'foo', 'mefoo'),
                                      napi.WordInfo(24, 'foo', 'bafoo')])
 
-    result = api_impl.format_result(search, 'json', {'keywords': True})
+    result = v1_format.format_result(search, 'json', {'keywords': True})
     js = json.loads(result)
 
     assert js['keywords'] == {'name': [{'id': 23, 'token': 'foo'},
@@ -319,7 +319,7 @@ def test_search_details_keywords_address():
                                      napi.WordInfo(23, 'foo', 'mefoo'),
                                      napi.WordInfo(24, 'foo', 'bafoo')])
 
-    result = api_impl.format_result(search, 'json', {'keywords': True})
+    result = v1_format.format_result(search, 'json', {'keywords': True})
     js = json.loads(result)
 
     assert js['keywords'] == {'address': [{'id': 23, 'token': 'foo'},

test/python/api/test_result_formatting_v1_reverse.py

@@ -15,7 +15,7 @@
 
 import pytest
 
-import nominatim_api.v1 as api_impl
+from nominatim_api.v1.format import dispatch as v1_format
 import nominatim_api as napi
 
 FORMATS = ['json', 'jsonv2', 'geojson', 'geocodejson', 'xml']
@@ -26,7 +26,7 @@ def test_format_reverse_minimal(fmt):
                                  ('amenity', 'post_box'),
                                  napi.Point(0.3, -8.9))
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt, {})
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt, {})
 
     if fmt == 'xml':
         root = ET.fromstring(raw)
@@ -38,7 +38,7 @@ def test_format_reverse_minimal(fmt):
 
 @pytest.mark.parametrize('fmt', FORMATS)
 def test_format_reverse_no_result(fmt):
-    raw = api_impl.format_result(napi.ReverseResults(), fmt, {})
+    raw = v1_format.format_result(napi.ReverseResults(), fmt, {})
 
     if fmt == 'xml':
         root = ET.fromstring(raw)
@@ -55,7 +55,7 @@ def test_format_reverse_with_osm_id(fmt):
                                  place_id=5564,
                                  osm_object=('N', 23))
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt, {})
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt, {})
 
     if fmt == 'xml':
         root = ET.fromstring(raw).find('result')
@@ -103,7 +103,7 @@ def test_format_reverse_with_address(fmt):
                                  ]))
     reverse.localize(napi.Locales())
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                  {'addressdetails': True})
 
 
@@ -167,7 +167,7 @@ def test_format_reverse_geocodejson_special_parts():
 
     reverse.localize(napi.Locales())
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), 'geocodejson',
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), 'geocodejson',
                                  {'addressdetails': True})
 
     props = json.loads(raw)['features'][0]['properties']['geocoding']
@@ -183,7 +183,7 @@ def test_format_reverse_with_address_none(fmt):
                                  napi.Point(1.0, 2.0),
                                  address_rows=napi.AddressLines())
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                  {'addressdetails': True})
 
 
@@ -213,7 +213,7 @@ def test_format_reverse_with_extratags(fmt):
                                  napi.Point(1.0, 2.0),
                                  extratags={'one': 'A', 'two':'B'})
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                  {'extratags': True})
 
     if fmt == 'xml':
@@ -235,7 +235,7 @@ def test_format_reverse_with_extratags_none(fmt):
                                  ('place', 'thing'),
                                  napi.Point(1.0, 2.0))
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                  {'extratags': True})
 
     if fmt == 'xml':
@@ -258,7 +258,7 @@ def test_format_reverse_with_namedetails_with_name(fmt):
                                  napi.Point(1.0, 2.0),
                                  names={'name': 'A', 'ref':'1'})
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                  {'namedetails': True})
 
     if fmt == 'xml':
@@ -280,7 +280,7 @@ def test_format_reverse_with_namedetails_without_name(fmt):
                                  ('place', 'thing'),
                                  napi.Point(1.0, 2.0))
 
-    raw = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    raw = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                  {'namedetails': True})
 
     if fmt == 'xml':
@@ -302,7 +302,7 @@ def test_search_details_with_icon_available(fmt):
                                  ('amenity', 'restaurant'),
                                  napi.Point(1.0, 2.0))
 
-    result = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    result = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                     {'icon_base_url': 'foo'})
 
     js = json.loads(result)
@@ -316,7 +316,7 @@ def test_search_details_with_icon_not_available(fmt):
                                  ('amenity', 'tree'),
                                  napi.Point(1.0, 2.0))
 
-    result = api_impl.format_result(napi.ReverseResults([reverse]), fmt,
+    result = v1_format.format_result(napi.ReverseResults([reverse]), fmt,
                                     {'icon_base_url': 'foo'})
 
     assert 'icon' not in json.loads(result)

test/python/cli/test_cmd_api.py

@@ -13,6 +13,15 @@
 import nominatim_db.clicmd.api
 import nominatim_api as napi
 
+@pytest.mark.parametrize('call', ['search', 'reverse', 'lookup', 'details', 'status'])
+def test_list_format(cli_call, call):
+    assert 0 == cli_call(call, '--list-formats')
+
+
+@pytest.mark.parametrize('call', ['search', 'reverse', 'lookup', 'details', 'status'])
+def test_bad_format(cli_call, call):
+    assert 1 == cli_call(call, '--format', 'rsdfsdfsdfsaefsdfsd')
+
 
 class TestCliStatusCall: