From cdb0e85513b07a8005a13dc0498c46f3dde74307 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Wed, 25 Oct 2023 13:25:51 +0200 Subject: [PATCH] Improve query genration in tile store --- .../tilestore/postgres/PostgresTileStore.java | 226 +++++++++++------- .../apache/baremaps/utils/PostgresUtils.java | 11 +- 2 files changed, 144 insertions(+), 93 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java index 4e9358464..e254e7e10 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java @@ -19,19 +19,18 @@ import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; +import java.sql.*; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.zip.GZIPOutputStream; import javax.sql.DataSource; import org.apache.baremaps.tilestore.TileCoord; import org.apache.baremaps.tilestore.TileStore; import org.apache.baremaps.tilestore.TileStoreException; import org.apache.baremaps.vectortile.tileset.Tileset; -import org.apache.baremaps.vectortile.tileset.TilesetQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,97 +52,14 @@ public PostgresTileStore(DataSource datasource, Tileset tileset) { this.tileset = tileset; } - protected boolean zoomPredicate(TilesetQuery query, int zoom) { - return query.getMinzoom() <= zoom && zoom < query.getMaxzoom(); - } - - public String withQuery(TileCoord tileCoord) { - var layers = tileset.getVectorLayers().stream() - .map(layer -> Map.entry(layer.getId(), layer.getQueries().stream() - .filter( - query -> query.getMinzoom() <= tileCoord.z() && tileCoord.z() < query.getMaxzoom()) - .toList())) - .filter(entry -> entry.getValue().size() > 0) - .toList(); - - var queryBuilder = new StringBuilder(); - queryBuilder.append("SELECT ("); - - for (int i = 0; i < layers.size(); i++) { - var layer = layers.get(i); - var layerId = layer.getKey(); - var layerQueries = layer.getValue().stream() - .filter(layerQuery -> zoomPredicate(layerQuery, tileCoord.z())).toList(); - - if (layerQueries.size() > 0) { - if (i > 0) { - queryBuilder.append(" || "); - } - - var sqlBuilder = new StringBuilder(); - sqlBuilder.append("(WITH mvtgeom AS (\n"); - - for (int j = 0; j < layerQueries.size(); j++) { - if (j != 0) { - sqlBuilder.append("UNION\n"); - } - var layerQuery = layerQueries.get(j).getSql().replace(";", ""); - sqlBuilder.append(String.format(""" - SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(%d, %d, %d)) AS geom, t.tags, t.id - FROM (%s) AS t - WHERE t.geom && ST_TileEnvelope(%d, %d, %d, margin => (64.0/4096)) - """, - tileCoord.z(), tileCoord.x(), tileCoord.y(), - layerQuery, - tileCoord.z(), tileCoord.x(), tileCoord.y())); - } - - queryBuilder.append(sqlBuilder) - .append(String.format(") SELECT ST_AsMVT(mvtgeom.*, '%s') FROM mvtgeom\n)", layerId)); - } - } - queryBuilder.append(") mvtTile"); - return queryBuilder.toString().replace("$zoom", String.valueOf(tileCoord.z())); - } + private Map cache = new ConcurrentHashMap<>(); @Override public ByteBuffer read(TileCoord tileCoord) throws TileStoreException { - String query = withQuery(tileCoord); - - logger.debug("Executing query: {}", query); - - long start = System.currentTimeMillis(); - - try (Connection connection = datasource.getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(query); - ByteArrayOutputStream data = new ByteArrayOutputStream()) { - - int length = 0; - - try (OutputStream gzip = new GZIPOutputStream(data)) { - while (resultSet.next()) { - byte[] bytes = resultSet.getBytes(1); - length += bytes.length; - gzip.write(bytes); - } - } - - long stop = System.currentTimeMillis(); - long duration = stop - start; - - // Log slow queries (> 10s) - if (duration > 10_000) { - logger.warn("Executed query for tile {} in {} ms: {}", tileCoord, duration, query); - } - - if (length > 0) { - return ByteBuffer.wrap(data.toByteArray()); - } else { - return ByteBuffer.allocate(0); - } + var query = cache.computeIfAbsent(tileCoord.z(), z -> prepareQuery(tileset, z)); + try (var connection = datasource.getConnection()) { + return query.execute(connection, tileCoord); } catch (Exception e) { - logger.error(e.getMessage()); throw new TileStoreException(e); } } @@ -164,4 +80,132 @@ public void delete(TileCoord tileCoord) { throw new UnsupportedOperationException("The postgis tile store is read only"); } + + public static TileQuery prepareQuery(Tileset tileset, int zoom) { + // Initialize a builder for the tile query + var tileQuery = new StringBuilder(); + tileQuery.append("SELECT ("); + + // Iterate over the layers and keep track of the number of layers and parameters included in the + // final query + var layers = tileset.getVectorLayers(); + var layerCount = 0; + var paramCount = 0; + for (var layer : layers) { + + // Initialize a builder for the layer query + var layerQuery = new StringBuilder(); + var layerHead = "(WITH mvtGeom AS ("; + layerQuery.append(layerHead); + + // Iterate over the queries and keep track of the number of queries included in the final + // query + var queries = layer.getQueries(); + var queryCount = 0; + for (var query : queries) { + + // Only include the query if the zoom level is in the range + if (query.getMinzoom() <= zoom && zoom < query.getMaxzoom()) { + + // Add a union between queries + if (queryCount > 0) { + layerQuery.append("UNION "); + } + + // Add the query to the layer query + var sql = query.getSql() + .replace(";", "") + .replace("?", "??") + .replace("$zoom", String.valueOf(zoom)); + var queryWithParams = String.format(""" + SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id + FROM (%s) AS t + WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096)) + """, sql); + layerQuery.append(queryWithParams); + + // Increase the parameter count (e.g. ?) and query count + paramCount += 6; + queryCount++; + } + } + + // Add the tail of the layer query + var layerQueryTail = + String.format(") SELECT ST_AsMVT(mvtGeom.*, '%s') FROM mvtGeom)", layer.getId()); + layerQuery.append(layerQueryTail); + + // Only include the layer query if queries were included for this layer + if (queryCount > 0) { + + // Add the concatenation between layer queries + if (layerCount > 0) { + tileQuery.append(" || "); + } + + // Add the layer query to the mvt query + tileQuery.append(layerQuery); + + // Increase the layer count + layerCount++; + } + } + + // Add the tail of the tile query + var tileQueryTail = ") mvtTile"; + tileQuery.append(tileQueryTail); + + // Log the resulting query + var query = tileQuery.toString().replace("\n", " "); + logger.debug("query: {}", query); + + return new TileQuery(query, paramCount); + } + + public static class TileQuery { + + private final String query; + + private final int paramCount; + + public TileQuery(String query, int paramCount) { + this.query = query; + this.paramCount = paramCount; + } + + public ByteBuffer execute(Connection connection, TileCoord tileCoord) + throws SQLException, IOException { + long start = System.currentTimeMillis(); + try (var statement = connection.prepareStatement(query)) { + + // Set the parameters for the tile + for (int i = 0; i < paramCount; i += 3) { + statement.setInt(i + 1, tileCoord.z()); + statement.setInt(i + 2, tileCoord.x()); + statement.setInt(i + 3, tileCoord.y()); + } + + // Fetch and compress the tile data + try (ByteArrayOutputStream data = new ByteArrayOutputStream();) { + try (ResultSet resultSet = statement.executeQuery(); + OutputStream gzip = new GZIPOutputStream(data)) { + while (resultSet.next()) { + byte[] bytes = resultSet.getBytes(1); + gzip.write(bytes); + } + } + return ByteBuffer.wrap(data.toByteArray()); + + } finally { + // Log slow queries (> 10s) + long stop = System.currentTimeMillis(); + long duration = stop - start; + if (duration > 10_000) { + logger.warn("Executed query for tile {} in {} ms", tileCoord, duration); + } + } + } + } + } + } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java index cacfab00e..681743fa3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java @@ -100,10 +100,13 @@ public static DataSource createDataSource(String jdbcUrl, int poolSize) { if (poolSize < 1) { throw new IllegalArgumentException("PoolSize cannot be inferior to 1"); } - var multiQueriesJdbcUrl = withAllowMultiQueriesParameter(jdbcUrl); + var config = new HikariConfig(); - config.setJdbcUrl(multiQueriesJdbcUrl); + config.setJdbcUrl(jdbcUrl); config.setMaximumPoolSize(poolSize); + config.addDataSourceProperty("allowMultiQueries", true); + config.addDataSourceProperty("prepareThreshold", 100); + return new HikariDataSource(config); } @@ -154,6 +157,10 @@ public static DataSource createDataSource(Database datasource) { if (datasource.getReadOnly() != null) { config.setReadOnly(datasource.getReadOnly()); } + + config.addDataSourceProperty("allowMultiQueries", true); + config.addDataSourceProperty("prepareThreshold", 100); + return new HikariDataSource(config); }