From 79125aebf516c9b61f0fd536f82b0bd8d8b43d5e Mon Sep 17 00:00:00 2001 From: Tomas Burleigh Date: Thu, 23 Mar 2023 20:46:17 +1300 Subject: [PATCH 1/3] Configurable tile systems to allow for projections other than web mercator. To help address #343 --- src/config.rs | 6 ++- src/lib.rs | 1 + src/pg/config.rs | 14 +++++-- src/pg/configurator.rs | 79 +++++++++++++++++++++++++++------------ src/pg/table_source.rs | 27 ++++++++++--- src/tilesystems/config.rs | 53 ++++++++++++++++++++++++++ src/tilesystems/mod.rs | 3 ++ 7 files changed, 151 insertions(+), 32 deletions(-) create mode 100644 src/tilesystems/config.rs create mode 100644 src/tilesystems/mod.rs diff --git a/src/config.rs b/src/config.rs index 314814156..6943d4e73 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,6 +16,7 @@ use crate::pg::PgConfig; use crate::pmtiles::PmtSource; use crate::source::{IdResolver, Sources}; use crate::srv::SrvConfig; +use crate::tilesystems::TileSystemsConfig; use crate::utils::{OneOrMany, Result}; use crate::Error::{ConfigLoadError, ConfigParseError, NoSources}; @@ -33,6 +34,9 @@ pub struct Config { #[serde(skip_serializing_if = "Option::is_none")] pub mbtiles: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tile_systems: Option, + #[serde(flatten)] pub unrecognized: HashMap, } @@ -80,7 +84,7 @@ impl Config { let mut sources: Vec>>>> = Vec::new(); if let Some(v) = self.postgres.as_mut() { for s in v.iter_mut() { - sources.push(Box::pin(s.resolve(idr.clone()))); + sources.push(Box::pin(s.resolve(idr.clone(), &self.tile_systems))); } } if let Some(v) = self.pmtiles.as_mut() { diff --git a/src/lib.rs b/src/lib.rs index 00623bcb7..ee677e0ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ pub mod pg; pub mod pmtiles; mod source; pub mod srv; +mod tilesystems; mod utils; #[cfg(test)] diff --git a/src/pg/config.rs b/src/pg/config.rs index dc4472c2c..ac7fc808e 100644 --- a/src/pg/config.rs +++ b/src/pg/config.rs @@ -8,6 +8,7 @@ use crate::pg::config_table::TableInfoSources; use crate::pg::configurator::PgBuilder; use crate::pg::utils::Result; use crate::source::{IdResolver, Sources}; +use crate::tilesystems::TileSystemsConfig; use crate::utils::{sorted_opt_map, BoolOrObject, OneOrMany}; pub trait PgInfo { @@ -91,10 +92,17 @@ impl PgConfig { Ok(res) } - pub async fn resolve(&mut self, id_resolver: IdResolver) -> crate::Result { + pub async fn resolve( + &mut self, + id_resolver: IdResolver, + tile_systems: &Option, + ) -> crate::Result { let pg = PgBuilder::new(self, id_resolver).await?; - let ((mut tables, tbl_info), (funcs, func_info)) = - try_join(pg.instantiate_tables(), pg.instantiate_functions()).await?; + let ((mut tables, tbl_info), (funcs, func_info)) = try_join( + pg.instantiate_tables(tile_systems), + pg.instantiate_functions(), + ) + .await?; self.tables = Some(tbl_info); self.functions = Some(func_info); diff --git a/src/pg/configurator.rs b/src/pg/configurator.rs index 3f16ee9d5..07b6234c3 100755 --- a/src/pg/configurator.rs +++ b/src/pg/configurator.rs @@ -15,6 +15,7 @@ use crate::pg::table_source::{calc_srid, get_table_sources, merge_table_info, ta use crate::pg::utils::PgError::InvalidTableExtent; use crate::pg::utils::Result; use crate::source::{IdResolver, Sources}; +use crate::tilesystems::{TileSystemConfig, TileSystemsConfig}; use crate::utils::{find_info, normalize_key, BoolOrObject, InfoMap, OneOrMany}; pub type SqlFuncInfoMapMap = InfoMap>; @@ -69,12 +70,25 @@ impl PgBuilder { }) } - pub async fn instantiate_tables(&self) -> Result<(Sources, TableInfoSources)> { + pub async fn instantiate_tables( + &self, + tile_systems: &Option, + ) -> Result<(Sources, TableInfoSources)> { let mut all_tables = get_table_sources(&self.pool).await?; // Match configured sources with the discovered ones and add them to the pending list. let mut used = HashSet::<(&str, &str, &str)>::new(); let mut pending = Vec::new(); + let mut all_tile_systems: Vec> = vec![None]; + + all_tile_systems.extend( + tile_systems + .as_ref() + .iter() + .flat_map(|map| map.iter()) + .map(|(id, ts)| Some((id.as_str(), ts))), + ); + for (id, cfg_inf) in &self.tables { // TODO: move this validation to serde somehow? if let Some(extent) = cfg_inf.extent { @@ -90,17 +104,26 @@ impl PgBuilder { let dup = !used.insert((&cfg_inf.schema, &cfg_inf.table, &cfg_inf.geometry_column)); let dup = if dup { "duplicate " } else { "" }; - let id2 = self.resolve_id(id, cfg_inf); - let Some(cfg_inf) = merge_table_info(self.default_srid, &id2, cfg_inf, src_inf) else { continue }; - warn_on_rename(id, &id2, "Table"); - info!("Configured {dup}source {id2} from {}", summary(&cfg_inf)); - pending.push(table_to_query( - id2, - cfg_inf, - self.pool.clone(), - self.disable_bounds, - self.max_feature_count, - )); + for tile_system in &all_tile_systems { + let mut id2 = self.resolve_id(id, cfg_inf); + + if let Some((ts_id, _)) = tile_system { + id2 += format!(":{ts_id}").as_str(); + } + + let Some(cfg_inf) = merge_table_info(self.default_srid, &id2, cfg_inf, src_inf) else { continue }; + warn_on_rename(id, &id2, "Table"); + info!("Configured {dup}source {id2} from {}", summary(&cfg_inf)); + + pending.push(table_to_query( + id2, + cfg_inf, + self.pool.clone(), + self.disable_bounds, + self.max_feature_count, + tile_system.map(|(_, ts)| ts), + )); + } } // Sort the discovered sources by schema, table and geometry column to ensure a consistent behavior @@ -123,17 +146,27 @@ impl PgBuilder { .replace("{schema}", &schema) .replace("{table}", &table) .replace("{column}", &column); - let id2 = self.resolve_id(&source_id, &src_inf); - let Some(srid) = calc_srid(&src_inf.format_id(), &id2, src_inf.srid, 0, self.default_srid) else { continue }; - src_inf.srid = srid; - info!("Discovered source {id2} from {}", summary(&src_inf)); - pending.push(table_to_query( - id2, - src_inf, - self.pool.clone(), - self.disable_bounds, - self.max_feature_count, - )); + + for tile_system in &all_tile_systems { + let mut id2 = self.resolve_id(&source_id, &src_inf); + + if let Some((ts_id, _)) = tile_system { + id2 += format!(":{ts_id}").as_str(); + } + + let Some(srid) = calc_srid(&src_inf.format_id(), &id2, src_inf.srid, 0, self.default_srid) else { continue }; + src_inf.srid = srid; + info!("Discovered source {id2} from {}", summary(&src_inf)); + + pending.push(table_to_query( + id2, + src_inf.clone(), + self.pool.clone(), + self.disable_bounds, + self.max_feature_count, + tile_system.map(|x| x.1), + )); + } } } } diff --git a/src/pg/table_source.rs b/src/pg/table_source.rs index 2014ff99d..9e444164a 100644 --- a/src/pg/table_source.rs +++ b/src/pg/table_source.rs @@ -12,6 +12,7 @@ use crate::pg::pg_source::PgSqlInfo; use crate::pg::pool::PgPool; use crate::pg::utils::PgError::PostgresError; use crate::pg::utils::{json_to_hashmap, polygon_to_bbox, Result}; +use crate::tilesystems::TileSystemConfig; use crate::utils::normalize_key; static DEFAULT_EXTENT: u32 = 4096; @@ -90,6 +91,7 @@ pub async fn table_to_query( pool: PgPool, disable_bounds: bool, max_feature_count: Option, + tile_system: Option<&TileSystemConfig>, ) -> Result<(String, PgSqlInfo, TableInfo)> { let schema = escape_identifier(&info.schema); let table = escape_identifier(&info.table); @@ -121,11 +123,24 @@ pub async fn table_to_query( let extent = info.extent.unwrap_or(DEFAULT_EXTENT); let buffer = info.buffer.unwrap_or(DEFAULT_BUFFER); + let tile_system_bounds = match tile_system { + Some(ts) => { + let left = ts.bounds.left; + let top = ts.bounds.top; + let right = ts.bounds.right; + let bottom = ts.bounds.bottom; + let ts_srid = ts.srid; + format!(", bounds => ST_MakeEnvelope({left}, {bottom}, {right}, {top}, {ts_srid})") + } + None => String::new(), + }; + let bbox_search = if buffer == 0 { - "ST_TileEnvelope($1::integer, $2::integer, $3::integer)".to_string() + format!("ST_TileEnvelope($1::integer, $2::integer, $3::integer{tile_system_bounds})") + // "ST_TileEnvelope($1::integer, $2::integer, $3::integer)".to_string() } else if pool.supports_tile_margin() { let margin = f64::from(buffer) / f64::from(extent); - format!("ST_TileEnvelope($1::integer, $2::integer, $3::integer, margin => {margin})") + format!("ST_TileEnvelope($1::integer, $2::integer, $3::integer, margin => {margin}{tile_system_bounds})") } else { // TODO: we should use ST_Expand here, but it may require a bit more math work, // so might not be worth it as it is only used for PostGIS < v3.1. @@ -133,12 +148,14 @@ pub async fn table_to_query( // let earth_circumference = 40075016.6855785; // let val = earth_circumference * buffer as f64 / extent as f64; // format!("ST_Expand(ST_TileEnvelope($1::integer, $2::integer, $3::integer), {val}/2^$1::integer)") - "ST_TileEnvelope($1::integer, $2::integer, $3::integer)".to_string() + format!("ST_TileEnvelope($1::integer, $2::integer, $3::integer{tile_system_bounds})") + // "ST_TileEnvelope($1::integer, $2::integer, $3::integer)".to_string() }; let limit_clause = max_feature_count.map_or(String::new(), |v| format!("LIMIT {v}")); let layer_id = escape_literal(info.layer_id.as_ref().unwrap_or(&id)); let clip_geom = info.clip_geom.unwrap_or(DEFAULT_CLIP_GEOM); + let output_srid = tile_system.map_or(3857, |ts| ts.srid); let query = format!( r#" SELECT @@ -146,8 +163,8 @@ SELECT FROM ( SELECT ST_AsMVTGeom( - ST_Transform(ST_CurveToLine({geometry_column}), 3857), - ST_TileEnvelope($1::integer, $2::integer, $3::integer), + ST_Transform(ST_CurveToLine({geometry_column}), {output_srid}), + ST_TileEnvelope($1::integer, $2::integer, $3::integer{tile_system_bounds}), {extent}, {buffer}, {clip_geom} ) AS geom {id_field}{properties} diff --git a/src/tilesystems/config.rs b/src/tilesystems/config.rs new file mode 100644 index 000000000..6cbab8d88 --- /dev/null +++ b/src/tilesystems/config.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use tilejson::Bounds; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct TileSystemConfig { + pub srid: i32, + pub bounds: Bounds, +} + +pub type TileSystemsConfig = HashMap; + +#[cfg(test)] +mod tests { + use crate::config::tests::assert_config; + use crate::pg::PgConfig; + use crate::test_utils::some; + use crate::tilesystems::TileSystemConfig; + use crate::OneOrMany::One; + use crate::{BoolOrObject, Config}; + use indoc::indoc; + use std::collections::HashMap; + use tilejson::Bounds; + + #[test] + pub fn test_parse_tile_systems_config() { + assert_config( + indoc! {" + postgres: + connection_string: 'postgresql://postgres@localhost/db' + tile_systems: + my_custom_tiling: + srid: 4326 + bounds: [-180, -90, 180, 90] + "}, + &Config { + postgres: Some(One(PgConfig { + connection_string: some("postgresql://postgres@localhost/db"), + auto_publish: Some(BoolOrObject::Bool(true)), + ..Default::default() + })), + tile_systems: Some(HashMap::from([( + "my_custom_tiling".to_string(), + TileSystemConfig { + srid: 4326, + bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0), + }, + )])), + ..Default::default() + }, + ); + } +} diff --git a/src/tilesystems/mod.rs b/src/tilesystems/mod.rs new file mode 100644 index 000000000..a2c99bca4 --- /dev/null +++ b/src/tilesystems/mod.rs @@ -0,0 +1,3 @@ +mod config; + +pub use config::{TileSystemConfig, TileSystemsConfig}; From ae08d5f31460aa58ed787fff73768297b4f22d97 Mon Sep 17 00:00:00 2001 From: Tomas Burleigh Date: Mon, 27 Mar 2023 16:35:53 +1300 Subject: [PATCH 2/3] Add tile_system to TableInfo --- src/pg/config_table.rs | 5 ++++ src/pg/configurator.rs | 56 +++++++++++++++++++-------------------- src/pg/table_source.rs | 6 ++--- src/tilesystems/config.rs | 2 ++ 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/pg/config_table.rs b/src/pg/config_table.rs index dfcb30f7f..057f663c0 100644 --- a/src/pg/config_table.rs +++ b/src/pg/config_table.rs @@ -5,6 +5,7 @@ use serde_yaml::Value; use tilejson::{Bounds, TileJSON, VectorLayer}; use crate::pg::config::PgInfo; +use crate::tilesystems::TileSystemConfig; use crate::utils::{sorted_opt_map, InfoMap}; pub type TableInfoSources = InfoMap; @@ -78,6 +79,10 @@ pub struct TableInfo { #[serde(skip_deserializing, skip_serializing)] pub prop_mapping: HashMap, + /// Alternative tiling to standard web-mercator + #[serde(skip_serializing_if = "Option::is_none")] + pub tile_system: Option, + #[serde(flatten, skip_serializing)] pub unrecognized: HashMap, } diff --git a/src/pg/configurator.rs b/src/pg/configurator.rs index 07b6234c3..f00c3e9f2 100755 --- a/src/pg/configurator.rs +++ b/src/pg/configurator.rs @@ -79,14 +79,14 @@ impl PgBuilder { // Match configured sources with the discovered ones and add them to the pending list. let mut used = HashSet::<(&str, &str, &str)>::new(); let mut pending = Vec::new(); - let mut all_tile_systems: Vec> = vec![None]; + let mut all_tile_systems: Vec> = vec![None]; all_tile_systems.extend( tile_systems .as_ref() .iter() - .flat_map(|map| map.iter()) - .map(|(id, ts)| Some((id.as_str(), ts))), + .flat_map(|m| m.values()) + .map(Some), ); for (id, cfg_inf) in &self.tables { @@ -104,26 +104,18 @@ impl PgBuilder { let dup = !used.insert((&cfg_inf.schema, &cfg_inf.table, &cfg_inf.geometry_column)); let dup = if dup { "duplicate " } else { "" }; - for tile_system in &all_tile_systems { - let mut id2 = self.resolve_id(id, cfg_inf); - - if let Some((ts_id, _)) = tile_system { - id2 += format!(":{ts_id}").as_str(); - } - - let Some(cfg_inf) = merge_table_info(self.default_srid, &id2, cfg_inf, src_inf) else { continue }; - warn_on_rename(id, &id2, "Table"); - info!("Configured {dup}source {id2} from {}", summary(&cfg_inf)); - - pending.push(table_to_query( - id2, - cfg_inf, - self.pool.clone(), - self.disable_bounds, - self.max_feature_count, - tile_system.map(|(_, ts)| ts), - )); - } + let id2 = self.resolve_id(id, cfg_inf); + let Some(cfg_inf) = merge_table_info(self.default_srid, &id2, cfg_inf, src_inf) else { continue }; + warn_on_rename(id, &id2, "Table"); + info!("Configured {dup}source {id2} from {}", summary(&cfg_inf)); + + pending.push(table_to_query( + id2, + cfg_inf, + self.pool.clone(), + self.disable_bounds, + self.max_feature_count, + )); } // Sort the discovered sources by schema, table and geometry column to ensure a consistent behavior @@ -137,7 +129,7 @@ impl PgBuilder { let Some(schema) = normalize_key(&all_tables, schema, "schema", "") else { continue }; let tables = all_tables.remove(&schema).unwrap(); for (table, geoms) in tables.into_iter().sorted_by(by_key) { - for (column, mut src_inf) in geoms.into_iter().sorted_by(by_key) { + for (column, src_inf) in geoms.into_iter().sorted_by(by_key) { if used.contains(&(schema.as_str(), table.as_str(), column.as_str())) { continue; } @@ -150,21 +142,27 @@ impl PgBuilder { for tile_system in &all_tile_systems { let mut id2 = self.resolve_id(&source_id, &src_inf); - if let Some((ts_id, _)) = tile_system { - id2 += format!(":{ts_id}").as_str(); + if let Some(ts) = tile_system { + if let Some(id) = &ts.identifier { + id2 += format!(":{id}").as_str(); + } else { + id2 += format!(":{}", &ts.srid).as_str(); + } } let Some(srid) = calc_srid(&src_inf.format_id(), &id2, src_inf.srid, 0, self.default_srid) else { continue }; - src_inf.srid = srid; + let mut table_info = src_inf.clone(); + table_info.srid = srid; + table_info.tile_system = tile_system.map(Clone::clone); + info!("Discovered source {id2} from {}", summary(&src_inf)); pending.push(table_to_query( id2, - src_inf.clone(), + table_info, self.pool.clone(), self.disable_bounds, self.max_feature_count, - tile_system.map(|x| x.1), )); } } diff --git a/src/pg/table_source.rs b/src/pg/table_source.rs index 9e444164a..60cf9e95f 100644 --- a/src/pg/table_source.rs +++ b/src/pg/table_source.rs @@ -12,7 +12,6 @@ use crate::pg::pg_source::PgSqlInfo; use crate::pg::pool::PgPool; use crate::pg::utils::PgError::PostgresError; use crate::pg::utils::{json_to_hashmap, polygon_to_bbox, Result}; -use crate::tilesystems::TileSystemConfig; use crate::utils::normalize_key; static DEFAULT_EXTENT: u32 = 4096; @@ -47,6 +46,7 @@ pub async fn get_table_sources(pool: &PgPool) -> Result { prop_mapping: HashMap::new(), unrecognized: HashMap::new(), bounds: None, + tile_system: None, }; // Warn for missing geometry indices. Ignore views since those can't have indices @@ -91,12 +91,12 @@ pub async fn table_to_query( pool: PgPool, disable_bounds: bool, max_feature_count: Option, - tile_system: Option<&TileSystemConfig>, ) -> Result<(String, PgSqlInfo, TableInfo)> { let schema = escape_identifier(&info.schema); let table = escape_identifier(&info.table); let geometry_column = escape_identifier(&info.geometry_column); let srid = info.srid; + let tile_system = &info.tile_system; if info.bounds.is_none() && !disable_bounds { info.bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid).await?; @@ -155,7 +155,7 @@ pub async fn table_to_query( let limit_clause = max_feature_count.map_or(String::new(), |v| format!("LIMIT {v}")); let layer_id = escape_literal(info.layer_id.as_ref().unwrap_or(&id)); let clip_geom = info.clip_geom.unwrap_or(DEFAULT_CLIP_GEOM); - let output_srid = tile_system.map_or(3857, |ts| ts.srid); + let output_srid = tile_system.as_ref().map_or(3857, |ts| ts.srid); let query = format!( r#" SELECT diff --git a/src/tilesystems/config.rs b/src/tilesystems/config.rs index 6cbab8d88..a586c02bd 100644 --- a/src/tilesystems/config.rs +++ b/src/tilesystems/config.rs @@ -6,6 +6,7 @@ use tilejson::Bounds; pub struct TileSystemConfig { pub srid: i32, pub bounds: Bounds, + pub identifier: Option, } pub type TileSystemsConfig = HashMap; @@ -44,6 +45,7 @@ mod tests { TileSystemConfig { srid: 4326, bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0), + identifier: None, }, )])), ..Default::default() From c003605047dd9d045f67117576f632d0f0d8a05f Mon Sep 17 00:00:00 2001 From: Tomas Burleigh Date: Tue, 28 Mar 2023 16:47:30 +1300 Subject: [PATCH 3/3] Removed global tile_systems property from config postgres.auto_publish.tile_systems now accepts a list of tile systems, replacing the global tile_systems property --- README.md | 21 +++ src/config.rs | 6 +- src/pg/config.rs | 27 ++-- src/pg/configurator.rs | 105 ++++++++++----- src/pg/table_source.rs | 11 +- src/tilesystems/config.rs | 146 ++++++++++++++++----- src/tilesystems/mod.rs | 2 +- tests/config.yaml | 21 +++ tests/expected/configured/catalog_cfg.json | 5 + tests/expected/given_config.yaml | 27 ++++ 10 files changed, 278 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 6dd6f1b59..a43cc23ce 100755 --- a/README.md +++ b/README.md @@ -550,6 +550,18 @@ postgres: id_format: 'table.{schema}.{table}.{column}' # Add more schemas to the ones listed above from_schemas: my_other_schema + # Optionally, publish tiles in more than one projection/tiling scheme + tile_systems: + # the default web mercator tiles, enabled by default + - type: WebMercatorQuad + # an additional custom tiling scheme + - type: Custom + srid: 4326 + # bounds of tile 0/0/0 + bounds: [ -180.0, -90.0, 180.0, 90.0 ] + # name for the tiling scheme + # auto published table will be given the name {source_id}:{tile_system.identifier} + identifier: WGS84Quad functions: id_format: '{schema}.{function}' @@ -601,6 +613,15 @@ postgres: # List of columns, that should be encoded as tile properties (required) properties: gid: int4 + + # Output tiles with a custom projection and tiling + tile_system: + # output srid + srid: 4326 + # bounds of tile 0/0/0 + bounds: [ -180.0, -90.0, 180.0, 90.0 ] + # name for the tiling scheme + identifier: WGS84Quad # Associative arrays of function sources functions: diff --git a/src/config.rs b/src/config.rs index 6943d4e73..314814156 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,7 +16,6 @@ use crate::pg::PgConfig; use crate::pmtiles::PmtSource; use crate::source::{IdResolver, Sources}; use crate::srv::SrvConfig; -use crate::tilesystems::TileSystemsConfig; use crate::utils::{OneOrMany, Result}; use crate::Error::{ConfigLoadError, ConfigParseError, NoSources}; @@ -34,9 +33,6 @@ pub struct Config { #[serde(skip_serializing_if = "Option::is_none")] pub mbtiles: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tile_systems: Option, - #[serde(flatten)] pub unrecognized: HashMap, } @@ -84,7 +80,7 @@ impl Config { let mut sources: Vec>>>> = Vec::new(); if let Some(v) = self.postgres.as_mut() { for s in v.iter_mut() { - sources.push(Box::pin(s.resolve(idr.clone(), &self.tile_systems))); + sources.push(Box::pin(s.resolve(idr.clone()))); } } if let Some(v) = self.pmtiles.as_mut() { diff --git a/src/pg/config.rs b/src/pg/config.rs index ac7fc808e..99a09bf49 100644 --- a/src/pg/config.rs +++ b/src/pg/config.rs @@ -8,7 +8,7 @@ use crate::pg::config_table::TableInfoSources; use crate::pg::configurator::PgBuilder; use crate::pg::utils::Result; use crate::source::{IdResolver, Sources}; -use crate::tilesystems::TileSystemsConfig; +use crate::tilesystems::TileSystem; use crate::utils::{sorted_opt_map, BoolOrObject, OneOrMany}; pub trait PgInfo { @@ -69,6 +69,7 @@ pub struct PgCfgPublish { pub struct PgCfgPublishType { pub from_schemas: Option>, pub id_format: Option, + pub tile_systems: Option>, } impl PgConfig { @@ -92,17 +93,10 @@ impl PgConfig { Ok(res) } - pub async fn resolve( - &mut self, - id_resolver: IdResolver, - tile_systems: &Option, - ) -> crate::Result { + pub async fn resolve(&mut self, id_resolver: IdResolver) -> crate::Result { let pg = PgBuilder::new(self, id_resolver).await?; - let ((mut tables, tbl_info), (funcs, func_info)) = try_join( - pg.instantiate_tables(tile_systems), - pg.instantiate_functions(), - ) - .await?; + let ((mut tables, tbl_info), (funcs, func_info)) = + try_join(pg.instantiate_tables(), pg.instantiate_functions()).await?; self.tables = Some(tbl_info); self.functions = Some(func_info); @@ -124,6 +118,7 @@ mod tests { use crate::pg::config_function::FunctionInfo; use crate::pg::config_table::TableInfo; use crate::test_utils::some; + use crate::tilesystems::TileSystemConfig; use crate::utils::OneOrMany::{Many, One}; #[test] @@ -196,6 +191,11 @@ mod tests { geometry_type: GEOMETRY properties: gid: int4 + tile_system: + identifier: WGS84Quad + srid: 4326 + bounds: [-180, -90, 180, 90] + functions: function_zxy_query: @@ -229,6 +229,11 @@ mod tests { "gid".to_string(), "int4".to_string(), )])), + tile_system: Some(TileSystemConfig { + srid: 4326, + bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0), + identifier: "WGS84Quad".to_string(), + }), ..Default::default() }, )])), diff --git a/src/pg/configurator.rs b/src/pg/configurator.rs index f00c3e9f2..bd334c864 100755 --- a/src/pg/configurator.rs +++ b/src/pg/configurator.rs @@ -1,6 +1,7 @@ use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; +use crate::OneOrMany::Many; use futures::future::join_all; use itertools::Itertools; use log::{debug, error, info, warn}; @@ -15,8 +16,8 @@ use crate::pg::table_source::{calc_srid, get_table_sources, merge_table_info, ta use crate::pg::utils::PgError::InvalidTableExtent; use crate::pg::utils::Result; use crate::source::{IdResolver, Sources}; -use crate::tilesystems::{TileSystemConfig, TileSystemsConfig}; -use crate::utils::{find_info, normalize_key, BoolOrObject, InfoMap, OneOrMany}; +use crate::tilesystems::{TileSystem, TileSystemConfig}; +use crate::utils::{find_info, normalize_key, BoolOrObject, InfoMap, OneOrMany, OneOrMany::One}; pub type SqlFuncInfoMapMap = InfoMap>; pub type SqlTableInfoMapMapMap = InfoMap>>; @@ -25,6 +26,7 @@ pub type SqlTableInfoMapMapMap = InfoMap>>; pub struct PgBuilderPublish { id_format: String, schemas: Option>, + tile_systems: OneOrMany, } impl PgBuilderPublish { @@ -32,11 +34,27 @@ impl PgBuilderPublish { is_function: bool, id_format: Option<&String>, schemas: Option>, + maybe_tile_systems: &Option>, ) -> Self { + let tile_systems = match maybe_tile_systems { + None => One(TileSystem::default()), + Some(tss) => { + if tss.len() < 2 { + One(tss.first().map_or_else(Default::default, Clone::clone)) + } else { + Many(tss.clone()) + } + } + }; + let id_format = id_format .cloned() .unwrap_or_else(|| (if is_function { "{function}" } else { "{table}" }).to_string()); - Self { id_format, schemas } + Self { + id_format, + schemas, + tile_systems, + } } } @@ -70,24 +88,12 @@ impl PgBuilder { }) } - pub async fn instantiate_tables( - &self, - tile_systems: &Option, - ) -> Result<(Sources, TableInfoSources)> { + pub async fn instantiate_tables(&self) -> Result<(Sources, TableInfoSources)> { let mut all_tables = get_table_sources(&self.pool).await?; // Match configured sources with the discovered ones and add them to the pending list. let mut used = HashSet::<(&str, &str, &str)>::new(); let mut pending = Vec::new(); - let mut all_tile_systems: Vec> = vec![None]; - - all_tile_systems.extend( - tile_systems - .as_ref() - .iter() - .flat_map(|m| m.values()) - .map(Some), - ); for (id, cfg_inf) in &self.tables { // TODO: move this validation to serde somehow? @@ -120,6 +126,8 @@ impl PgBuilder { // Sort the discovered sources by schema, table and geometry column to ensure a consistent behavior if let Some(auto_tables) = &self.auto_tables { + let tile_systems = &auto_tables.tile_systems; + let schemas = auto_tables .schemas .as_ref() @@ -139,21 +147,21 @@ impl PgBuilder { .replace("{table}", &table) .replace("{column}", &column); - for tile_system in &all_tile_systems { + for tile_system in tile_systems.clone() { let mut id2 = self.resolve_id(&source_id, &src_inf); - if let Some(ts) = tile_system { - if let Some(id) = &ts.identifier { - id2 += format!(":{id}").as_str(); - } else { - id2 += format!(":{}", &ts.srid).as_str(); + match &tile_system { + TileSystem::Custom(TileSystemConfig { identifier, .. }) => { + id2 += ":"; + id2 += identifier.as_str(); } + TileSystem::WebMercatorQuad => {} } let Some(srid) = calc_srid(&src_inf.format_id(), &id2, src_inf.srid, 0, self.default_srid) else { continue }; let mut table_info = src_inf.clone(); table_info.srid = srid; - table_info.tile_system = tile_system.map(Clone::clone); + table_info.tile_system = tile_system.clone().into(); info!("Discovered source {id2} from {}", summary(&src_inf)); @@ -263,7 +271,7 @@ impl PgBuilder { } fn new_auto_publish(config: &PgConfig, is_function: bool) -> Option { - let default = |schemas| Some(PgBuilderPublish::new(is_function, None, schemas)); + let default = |schemas| Some(PgBuilderPublish::new(is_function, None, schemas, &None)); if let Some(bo_a) = &config.auto_publish { match bo_a { @@ -273,6 +281,7 @@ fn new_auto_publish(config: &PgConfig, is_function: bool) -> Option default(merge_opt_hs(&a.from_schemas, &None)), BoolOrObject::Bool(false) => None, @@ -344,14 +353,20 @@ fn merge_opt_hs( #[cfg(test)] mod tests { use indoc::indoc; + use tilejson::Bounds; use super::*; #[allow(clippy::unnecessary_wraps)] - fn builder(id_format: &str, schemas: Option<&[&str]>) -> Option { + fn builder( + id_format: &str, + schemas: Option<&[&str]>, + tile_systems: Option>, + ) -> Option { Some(PgBuilderPublish { id_format: id_format.to_string(), schemas: schemas.map(|s| s.iter().map(|s| (*s).to_string()).collect()), + tile_systems: tile_systems.map_or_else(|| One(TileSystem::default()), Many), }) } @@ -363,9 +378,9 @@ mod tests { fn test_auto_publish_no_auto() { let config = parse_yaml("{}"); let res = new_auto_publish(&config, false); - assert_eq!(res, builder("{table}", None)); + assert_eq!(res, builder("{table}", None, None)); let res = new_auto_publish(&config, true); - assert_eq!(res, builder("{function}", None)); + assert_eq!(res, builder("{function}", None, None)); let config = parse_yaml("tables: {}"); assert_eq!(new_auto_publish(&config, false), None); @@ -380,9 +395,9 @@ mod tests { fn test_auto_publish_bool() { let config = parse_yaml("auto_publish: true"); let res = new_auto_publish(&config, false); - assert_eq!(res, builder("{table}", None)); + assert_eq!(res, builder("{table}", None, None)); let res = new_auto_publish(&config, true); - assert_eq!(res, builder("{function}", None)); + assert_eq!(res, builder("{function}", None, None)); let config = parse_yaml("auto_publish: false"); assert_eq!(new_auto_publish(&config, false), None); @@ -396,7 +411,7 @@ mod tests { from_schemas: public tables: true"}); let res = new_auto_publish(&config, false); - assert_eq!(res, builder("{table}", Some(&["public"]))); + assert_eq!(res, builder("{table}", Some(&["public"]), None)); assert_eq!(new_auto_publish(&config, true), None); let config = parse_yaml(indoc! {" @@ -405,7 +420,7 @@ mod tests { functions: true"}); assert_eq!(new_auto_publish(&config, false), None); let res = new_auto_publish(&config, true); - assert_eq!(res, builder("{function}", Some(&["public"]))); + assert_eq!(res, builder("{function}", Some(&["public"]), None)); let config = parse_yaml(indoc! {" auto_publish: @@ -413,14 +428,14 @@ mod tests { tables: false"}); assert_eq!(new_auto_publish(&config, false), None); let res = new_auto_publish(&config, true); - assert_eq!(res, builder("{function}", Some(&["public"]))); + assert_eq!(res, builder("{function}", Some(&["public"]), None)); let config = parse_yaml(indoc! {" auto_publish: from_schemas: public functions: false"}); let res = new_auto_publish(&config, false); - assert_eq!(res, builder("{table}", Some(&["public"]))); + assert_eq!(res, builder("{table}", Some(&["public"]), None)); assert_eq!(new_auto_publish(&config, true), None); } @@ -431,9 +446,27 @@ mod tests { from_schemas: public tables: from_schemas: osm - id_format: '{schema}.{table}'"}); + id_format: '{schema}.{table}' + tile_systems: + - type: WebMercatorQuad + - type: Custom + srid: 4326 + identifier: WGS84Quad + bounds: [-180, -90, 180, 90]"}); let res = new_auto_publish(&config, false); - assert_eq!(res, builder("{schema}.{table}", Some(&["public", "osm"]))); + let tile_systems = Some(vec![ + TileSystem::WebMercatorQuad, + TileSystem::Custom(TileSystemConfig { + identifier: "WGS84Quad".to_string(), + srid: 4326, + bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0), + }), + ]); + + assert_eq!( + res, + builder("{schema}.{table}", Some(&["public", "osm"]), tile_systems) + ); assert_eq!(new_auto_publish(&config, true), None); let config = parse_yaml(indoc! {" @@ -443,7 +476,7 @@ mod tests { - osm - public"}); let res = new_auto_publish(&config, false); - assert_eq!(res, builder("{table}", Some(&["public", "osm"]))); + assert_eq!(res, builder("{table}", Some(&["public", "osm"]), None)); assert_eq!(new_auto_publish(&config, true), None); } } diff --git a/src/pg/table_source.rs b/src/pg/table_source.rs index 60cf9e95f..5e6657ffb 100644 --- a/src/pg/table_source.rs +++ b/src/pg/table_source.rs @@ -12,6 +12,7 @@ use crate::pg::pg_source::PgSqlInfo; use crate::pg::pool::PgPool; use crate::pg::utils::PgError::PostgresError; use crate::pg::utils::{json_to_hashmap, polygon_to_bbox, Result}; +use crate::tilesystems::TileSystem; use crate::utils::normalize_key; static DEFAULT_EXTENT: u32 = 4096; @@ -96,7 +97,7 @@ pub async fn table_to_query( let table = escape_identifier(&info.table); let geometry_column = escape_identifier(&info.geometry_column); let srid = info.srid; - let tile_system = &info.tile_system; + let tile_system: TileSystem = info.tile_system.clone().into(); if info.bounds.is_none() && !disable_bounds { info.bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid).await?; @@ -123,8 +124,8 @@ pub async fn table_to_query( let extent = info.extent.unwrap_or(DEFAULT_EXTENT); let buffer = info.buffer.unwrap_or(DEFAULT_BUFFER); - let tile_system_bounds = match tile_system { - Some(ts) => { + let tile_system_bounds = match &tile_system { + TileSystem::Custom(ts) => { let left = ts.bounds.left; let top = ts.bounds.top; let right = ts.bounds.right; @@ -132,7 +133,7 @@ pub async fn table_to_query( let ts_srid = ts.srid; format!(", bounds => ST_MakeEnvelope({left}, {bottom}, {right}, {top}, {ts_srid})") } - None => String::new(), + TileSystem::WebMercatorQuad => String::new(), }; let bbox_search = if buffer == 0 { @@ -155,7 +156,7 @@ pub async fn table_to_query( let limit_clause = max_feature_count.map_or(String::new(), |v| format!("LIMIT {v}")); let layer_id = escape_literal(info.layer_id.as_ref().unwrap_or(&id)); let clip_geom = info.clip_geom.unwrap_or(DEFAULT_CLIP_GEOM); - let output_srid = tile_system.as_ref().map_or(3857, |ts| ts.srid); + let output_srid = tile_system.get_srid(); let query = format!( r#" SELECT diff --git a/src/tilesystems/config.rs b/src/tilesystems/config.rs index a586c02bd..4dc591f1e 100644 --- a/src/tilesystems/config.rs +++ b/src/tilesystems/config.rs @@ -1,55 +1,131 @@ use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use tilejson::Bounds; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct TileSystemConfig { pub srid: i32, pub bounds: Bounds, - pub identifier: Option, + pub identifier: String, } -pub type TileSystemsConfig = HashMap; +impl From for TileSystem { + fn from(value: TileSystemConfig) -> Self { + TileSystem::Custom(value) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum TileSystem { + Custom(TileSystemConfig), + WebMercatorQuad, +} + +impl TileSystem { + pub fn is_web_mercator(&self) -> bool { + matches!(self, TileSystem::WebMercatorQuad) + } + + pub fn get_srid(&self) -> i32 { + match self { + TileSystem::Custom(ts) => ts.srid, + TileSystem::WebMercatorQuad => 3857, + } + } +} + +impl Default for TileSystem { + fn default() -> Self { + TileSystem::WebMercatorQuad + } +} + +impl From for Option { + fn from(val: TileSystem) -> Self { + match val { + TileSystem::WebMercatorQuad => None, + TileSystem::Custom(ts) => Some(ts), + } + } +} + +impl From> for TileSystem { + fn from(value: Option) -> Self { + match value { + None => TileSystem::WebMercatorQuad, + Some(ts) => TileSystem::Custom(ts), + } + } +} #[cfg(test)] mod tests { - use crate::config::tests::assert_config; - use crate::pg::PgConfig; - use crate::test_utils::some; - use crate::tilesystems::TileSystemConfig; - use crate::OneOrMany::One; - use crate::{BoolOrObject, Config}; + use crate::tilesystems::{TileSystem, TileSystemConfig}; use indoc::indoc; - use std::collections::HashMap; use tilejson::Bounds; #[test] pub fn test_parse_tile_systems_config() { - assert_config( - indoc! {" - postgres: - connection_string: 'postgresql://postgres@localhost/db' - tile_systems: - my_custom_tiling: - srid: 4326 - bounds: [-180, -90, 180, 90] - "}, - &Config { - postgres: Some(One(PgConfig { - connection_string: some("postgresql://postgres@localhost/db"), - auto_publish: Some(BoolOrObject::Bool(true)), - ..Default::default() - })), - tile_systems: Some(HashMap::from([( - "my_custom_tiling".to_string(), - TileSystemConfig { - srid: 4326, - bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0), - identifier: None, - }, - )])), - ..Default::default() - }, + println!( + "{}", + serde_yaml::to_string(&TileSystem::Custom(TileSystemConfig { + identifier: "WGS84Quad".to_string(), + srid: 4326, + bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0) + })) + .unwrap() + ); + + let configs: Vec = serde_yaml::from_str(indoc! {" + - type: WebMercatorQuad + - type: Custom + identifier: WGS84Quad + srid: 4326 + bounds: [-180, -90, 180, 90] + "}) + .unwrap(); + + assert_eq!( + configs, + vec![ + TileSystem::WebMercatorQuad, + TileSystem::Custom(TileSystemConfig { + identifier: "WGS84Quad".to_string(), + srid: 4326, + bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0) + }) + ] + ); + + let maybe_config: Option> = serde_yaml::from_str("").unwrap(); + assert!(maybe_config.is_none()); + let maybe_configs: Option> = serde_yaml::from_str( + " + - type: WebMercatorQuad + - type: Custom + identifier: WGS84Quad + srid: 4326 + bounds: [-180, -90, 180, 90] + ", + ) + .unwrap(); + assert!(maybe_configs.is_some()); + assert_eq!(maybe_configs.unwrap().len(), 2); + + let config: TileSystem = serde_yaml::from_str(indoc! {"\ + type: Custom + identifier: WGS84Quad + srid: 4326 + bounds: [-180, -90, 180, 90] + "}) + .unwrap(); + assert_eq!( + config, + TileSystem::Custom(TileSystemConfig { + identifier: "WGS84Quad".to_string(), + srid: 4326, + bounds: Bounds::new(-180.0, -90.0, 180.0, 90.0) + }) ); } } diff --git a/src/tilesystems/mod.rs b/src/tilesystems/mod.rs index a2c99bca4..711072890 100644 --- a/src/tilesystems/mod.rs +++ b/src/tilesystems/mod.rs @@ -1,3 +1,3 @@ mod config; -pub use config::{TileSystemConfig, TileSystemsConfig}; +pub use config::{TileSystem, TileSystemConfig}; diff --git a/tests/config.yaml b/tests/config.yaml index 0019cf255..591ad49d1 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -92,6 +92,27 @@ postgres: properties: gid: int4 + points1_wgs84: + layer_id: abc + schema: public + table: points1 + minzoom: 0 + maxzoom: 30 + bounds: [-180.0, -90.0, 180.0, 90.0] + id_column: ~ + geometry_column: geom + srid: 4326 + extent: 4096 + buffer: 64 + clip_geom: true + geometry_type: POINT + tile_system: + identifier: WGS84Quad + srid: 4326 + bounds: [-180.0, -90.0, 180.0, 90.0] + properties: + gid: int4 + points2: schema: public table: points2 diff --git a/tests/expected/configured/catalog_cfg.json b/tests/expected/configured/catalog_cfg.json index c38f4e467..afa67466b 100644 --- a/tests/expected/configured/catalog_cfg.json +++ b/tests/expected/configured/catalog_cfg.json @@ -23,6 +23,11 @@ "content_type": "application/x-protobuf", "description": "public.points1.geom" }, + { + "id": "points1_wgs84", + "content_type": "application/x-protobuf", + "description": "public.points1.geom" + }, { "id": "points2", "content_type": "application/x-protobuf", diff --git a/tests/expected/given_config.yaml b/tests/expected/given_config.yaml index 30b3f053e..557cac6f9 100644 --- a/tests/expected/given_config.yaml +++ b/tests/expected/given_config.yaml @@ -39,6 +39,33 @@ postgres: geometry_type: POINT properties: gid: int4 + points1_wgs84: + layer_id: abc + schema: public + table: points1 + srid: 4326 + geometry_column: geom + minzoom: 0 + maxzoom: 30 + bounds: + - -180.0 + - -90.0 + - 180.0 + - 90.0 + extent: 4096 + buffer: 64 + clip_geom: true + geometry_type: POINT + properties: + gid: int4 + tile_system: + srid: 4326 + bounds: + - -180.0 + - -90.0 + - 180.0 + - 90.0 + identifier: WGS84Quad points2: schema: public table: points2