(breaking) Future-proof source catalog (#754)

Modify `/catalog` endpoint to return an object instead of a list. This
allows future expansion of the catalog schema, e.g. adding new types of
data.

The new schema:

```yaml
{
  "tiles" {
    "function_zxy_query": {
      "name": "public.function_zxy_query",
      "content_type": "application/x-protobuf"
    },
    "points1": {
      "name": "public.points1.geom",
      "content_type": "image/webp"
    },
    ...
  },
}
```

docs/src/using.md

@@ -31,17 +31,19 @@ curl localhost:3000/catalog | jq
 ```
 
 ```yaml
-[
-  {
-    "id": "function_zxy_query",
-    "name": "public.function_zxy_query"
+{
+  "tiles" {
+    "function_zxy_query": {
+      "name": "public.function_zxy_query",
+      "content_type": "application/x-protobuf"
+    },
+    "points1": {
+      "name": "public.points1.geom",
+      "content_type": "image/webp"
+    },
+    ...
   },
-  {
-    "id": "points1",
-    "name": "public.points1.geom"
-  },
-  ...
-]
+}
 ```
 
 ## Source TileJSON

martin/src/config.rs

@@ -110,14 +110,18 @@ impl Config {
             sources.push(Box::pin(val));
         }
 
+        // Minor in-efficiency:
+        // Sources are added to a BTreeMap, then iterated over into a sort structure and convert back to a BTreeMap.
+        // Ideally there should be a vector of values, which is then sorted (in-place?) and converted to a BTreeMap.
         Ok(AllSources {
-            sources: try_join_all(sources).await?.into_iter().fold(
-                Sources::default(),
-                |mut acc, hashmap| {
+            sources: try_join_all(sources)
+                .await?
+                .into_iter()
+                .fold(Sources::default(), |mut acc, hashmap| {
                     acc.extend(hashmap);
                     acc
-                },
-            ),
+                })
+                .sort(),
             sprites: resolve_sprites(&mut self.sprites)?,
         })
     }

martin/src/source.rs

@@ -1,5 +1,4 @@
-use std::cmp::Ordering;
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt::{Debug, Display, Formatter};
 
 use actix_web::error::ErrorNotFound;
@@ -33,40 +32,60 @@ pub type Tile = Vec<u8>;
 pub type UrlQuery = HashMap<String, String>;
 
 #[derive(Default, Clone)]
-pub struct Sources(HashMap<String, Box<dyn Source>>);
+pub struct Sources {
+    tiles: HashMap<String, Box<dyn Source>>,
+    catalog: SourceCatalog,
+}
+
+impl Sources {
+    #[must_use]
+    pub fn sort(self) -> Self {
+        Self {
+            tiles: self.tiles,
+            catalog: SourceCatalog {
+                tiles: self
+                    .catalog
+                    .tiles
+                    .into_iter()
+                    .sorted_by(|a, b| a.0.cmp(&b.0))
+                    .collect(),
+            },
+        }
+    }
+}
 
 impl Sources {
     pub fn insert(&mut self, id: String, source: Box<dyn Source>) {
-        self.0.insert(id, source);
+        let tilejson = source.get_tilejson();
+        let info = source.get_tile_info();
+        self.catalog.tiles.insert(
+            id.clone(),
+            SourceEntry {
+                content_type: info.format.content_type().to_string(),
+                content_encoding: info.encoding.content_encoding().map(ToString::to_string),
+                name: tilejson.name.filter(|v| v != &id),
+                description: tilejson.description,
+                attribution: tilejson.attribution,
+            },
+        );
+        self.tiles.insert(id, source);
     }
 
     pub fn extend(&mut self, other: Sources) {
-        self.0.extend(other.0);
+        for (k, v) in other.catalog.tiles {
+            self.catalog.tiles.insert(k, v);
+        }
+        self.tiles.extend(other.tiles);
     }
 
     #[must_use]
-    pub fn get_catalog(&self) -> Vec<IndexEntry> {
-        self.0
-            .iter()
-            .map(|(id, src)| {
-                let tilejson = src.get_tilejson();
-                let info = src.get_tile_info();
-                IndexEntry {
-                    id: id.clone(),
-                    content_type: info.format.content_type().to_string(),
-                    content_encoding: info.encoding.content_encoding().map(ToString::to_string),
-                    name: tilejson.name.filter(|v| v != id),
-                    description: tilejson.description,
-                    attribution: tilejson.attribution,
-                }
-            })
-            .sorted()
-            .collect()
+    pub fn get_catalog(&self) -> &SourceCatalog {
+        &self.catalog
     }
 
     pub fn get_source(&self, id: &str) -> actix_web::Result<&dyn Source> {
         Ok(self
-            .0
+            .tiles
             .get(id)
             .ok_or_else(|| ErrorNotFound(format!("Source {id} does not exist")))?
             .as_ref())
@@ -138,9 +157,13 @@ impl Clone for Box<dyn Source> {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct IndexEntry {
-    pub id: String,
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SourceCatalog {
+    tiles: BTreeMap<String, SourceEntry>,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SourceEntry {
     pub content_type: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub content_encoding: Option<String>,
@@ -152,18 +175,6 @@ pub struct IndexEntry {
     pub attribution: Option<String>,
 }
 
-impl PartialOrd<Self> for IndexEntry {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for IndexEntry {
-    fn cmp(&self, other: &Self) -> Ordering {
-        (&self.id, &self.name).cmp(&(&other.id, &other.name))
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

martin/src/srv/mod.rs

@@ -4,4 +4,4 @@ mod server;
 pub use config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT};
 pub use server::{new_server, router, RESERVED_KEYWORDS};
 
-pub use crate::source::IndexEntry;
+pub use crate::source::SourceEntry;

martin/src/srv/server.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 use actix_cors::Cors;
 use actix_http::ContentEncoding;
 use actix_web::dev::Server;
-use actix_web::error::ErrorBadRequest;
+use actix_web::error::{ErrorBadRequest, ErrorInternalServerError, ErrorNotFound};
 use actix_web::http::header::{
     AcceptEncoding, ContentType, Encoding as HeaderEnc, HeaderValue, Preference, CACHE_CONTROL,
     CONTENT_ENCODING,
@@ -13,8 +13,8 @@ use actix_web::http::Uri;
 use actix_web::middleware::TrailingSlash;
 use actix_web::web::{Data, Path, Query};
 use actix_web::{
-    error, middleware, route, web, App, HttpMessage, HttpRequest, HttpResponse, HttpServer,
-    Responder, Result,
+    middleware, route, web, App, HttpMessage, HttpRequest, HttpResponse, HttpServer, Responder,
+    Result,
 };
 use futures::future::try_join_all;
 use log::error;
@@ -58,21 +58,22 @@ struct TileRequest {
 
 pub fn map_internal_error<T: std::fmt::Display>(e: T) -> actix_web::Error {
     error!("{e}");
-    error::ErrorInternalServerError(e.to_string())
+    ErrorInternalServerError(e.to_string())
 }
 
 pub fn map_sprite_error(e: SpriteError) -> actix_web::Error {
-    if let SpriteError::SpriteNotFound(_) = e {
-        error::ErrorNotFound(e.to_string())
-    } else {
-        map_internal_error(e)
+    use SpriteError::SpriteNotFound;
+    match e {
+        SpriteNotFound(_) => ErrorNotFound(e.to_string()),
+        _ => map_internal_error(e),
     }
 }
 
 /// Root path will eventually have a web front. For now, just a stub.
 #[route("/", method = "GET", method = "HEAD")]
 #[allow(clippy::unused_async)]
 async fn get_index() -> &'static str {
+    // todo: once this becomes more substantial, add wrap = "middleware::Compress::default()"
     "Martin server is running. Eventually this will be a nice web front.\n\n\
     A list of all available sources is at /catalog\n\n\
     See documentation https://github.com/maplibre/martin"
@@ -112,7 +113,12 @@ async fn get_sprite_png(
         .body(sheet.encode_png().map_err(map_internal_error)?))
 }
 
-#[route("/sprite/{source_ids}.json", method = "GET", method = "HEAD")]
+#[route(
+    "/sprite/{source_ids}.json",
+    method = "GET",
+    method = "HEAD",
+    wrap = "middleware::Compress::default()"
+)]
 async fn get_sprite_json(
     path: Path<TileJsonRequest>,
     sprites: Data<SpriteSources>,
@@ -274,7 +280,7 @@ async fn get_tile(
     let (tile, info) = if path.source_ids.contains(',') {
         let (sources, use_url_query, info) = sources.get_sources(&path.source_ids, Some(path.z))?;
         if sources.is_empty() {
-            return Err(error::ErrorNotFound("No valid sources found"));
+            return Err(ErrorNotFound("No valid sources found"));
         }
         let query = if use_url_query {
             Some(Query::<UrlQuery>::from_query(req.query_string())?.into_inner())
@@ -290,7 +296,7 @@ async fn get_tile(
         let can_join = info.format == Format::Mvt
             && (info.encoding == Encoding::Uncompressed || info.encoding == Encoding::Gzip);
         if !can_join && tiles.iter().filter(|v| !v.is_empty()).count() > 1 {
-            return Err(error::ErrorBadRequest(format!(
+            return Err(ErrorBadRequest(format!(
                 "Can't merge {info} tiles. Make sure there is only one non-empty tile source at zoom level {}",
                 xyz.z
             )))?;
@@ -301,7 +307,7 @@ async fn get_tile(
         let zoom = xyz.z;
         let src = sources.get_source(id)?;
         if !Sources::check_zoom(src, id, zoom) {
-            return Err(error::ErrorNotFound(format!(
+            return Err(ErrorNotFound(format!(
                 "Zoom {zoom} is not valid for source {id}",
             )));
         }

martin/src/utils/utilities.rs

@@ -1,10 +1,8 @@
-use std::cmp::Ordering::Equal;
 use std::collections::{BTreeMap, HashMap};
 use std::io::{Read as _, Write as _};
 
 use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
-use itertools::Itertools;
 use serde::{Deserialize, Serialize, Serializer};
 
 #[must_use]
@@ -26,20 +24,13 @@ pub fn sorted_opt_map<S: Serializer, T: Serialize>(
     value: &Option<HashMap<String, T>>,
     serializer: S,
 ) -> Result<S::Ok, S::Error> {
-    value
-        .as_ref()
-        .map(|v| {
-            v.iter()
-                .sorted_by(|a, b| {
-                    let lower = a.0.to_lowercase().cmp(&b.0.to_lowercase());
-                    match lower {
-                        Equal => a.0.cmp(b.0),
-                        other => other,
-                    }
-                })
-                .collect::<BTreeMap<_, _>>()
-        })
-        .serialize(serializer)
+    value.as_ref().map(sorted_btree_map).serialize(serializer)
+}
+
+pub fn sorted_btree_map<K: Serialize + Ord, V>(value: &HashMap<K, V>) -> BTreeMap<&K, &V> {
+    let mut items: Vec<(_, _)> = value.iter().collect();
+    items.sort_by(|a, b| a.0.cmp(b.0));
+    BTreeMap::from_iter(items)
 }
 
 pub fn decode_gzip(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {

tests/debug.html

@@ -229,53 +229,53 @@
         ).then((r) => r.json()));
 
         // Set up the corresponding toggle button for each layer.
-        for (const source of sources) {
+        for (const [id, source] of Object.entries(sources.tiles)) {
           // Skip layers that already have a button set up.
-          if (document.getElementById(source.id)) {
+          if (document.getElementById(id)) {
             continue;
           }
 
           const layerType = geometryTypeToLayerType(source.geometry_type);
 
           map.addLayer({
-            id: source.id,
+            id: id,
             type: layerType,
             source: {
               type: 'vector',
-              url: `http://0.0.0.0:3000/${source.id}`
+              url: `http://0.0.0.0:3000/${id}`
             },
-            'source-layer': source.id,
+            'source-layer': id,
             layout: {
               visibility: 'none'
             },
             paint: {
-              [`${layerType}-color`]: stringToColour(source.id)
+              [`${layerType}-color`]: stringToColour(id)
             }
           });
 
-          map.on('click', source.id, (e) => {
+          map.on('click', id, (e) => {
             console.log(e.features);
           });
 
           const colorbox = document.createElement("input");
           colorbox.type = "color";
           colorbox.id = "colorbox";
-          colorbox.value = stringToColour(source.id);
-          colorbox.style.backgroundColor = stringToColour(source.id);
+          colorbox.value = stringToColour(id);
+          colorbox.style.backgroundColor = stringToColour(id);
           colorbox.onclick = function(e){
             e.stopPropagation();
           }
           colorbox.onchange = function(e){
-            map.setPaintProperty(source.id, `${layerType}-color`, e.target.value);
+            map.setPaintProperty(id, `${layerType}-color`, e.target.value);
             colorbox.style.backgroundColor = e.target.value;
           }
 
           // Create a link.
           const link = document.createElement('a');
-          link.id = source.id;
+          link.id = id;
           link.href = '#';
-          link.textContent = source.id;
-          link.title = source.id;
+          link.textContent = id;
+          link.title = id;
           link.appendChild(colorbox);
 
           // Show or hide layer when the toggle is clicked.