Skip to content

Commit

Permalink
Improve query genration in tile store
Browse files Browse the repository at this point in the history
  • Loading branch information
@bchapuis
bchapuis committed Oct 25, 2023
1 parent 13d3776 commit cdb0e85
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 93 deletions.
226 changes: 135 additions & 91 deletions baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,18 @@


import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.*;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPOutputStream;
import javax.sql.DataSource;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.TileStoreException;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.apache.baremaps.vectortile.tileset.TilesetQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -53,97 +52,14 @@ public PostgresTileStore(DataSource datasource, Tileset tileset) {
this.tileset = tileset;
}

protected boolean zoomPredicate(TilesetQuery query, int zoom) {
return query.getMinzoom() <= zoom && zoom < query.getMaxzoom();
}

public String withQuery(TileCoord tileCoord) {
var layers = tileset.getVectorLayers().stream()
.map(layer -> Map.entry(layer.getId(), layer.getQueries().stream()
.filter(
query -> query.getMinzoom() <= tileCoord.z() && tileCoord.z() < query.getMaxzoom())
.toList()))
.filter(entry -> entry.getValue().size() > 0)
.toList();

var queryBuilder = new StringBuilder();
queryBuilder.append("SELECT (");

for (int i = 0; i < layers.size(); i++) {
var layer = layers.get(i);
var layerId = layer.getKey();
var layerQueries = layer.getValue().stream()
.filter(layerQuery -> zoomPredicate(layerQuery, tileCoord.z())).toList();

if (layerQueries.size() > 0) {
if (i > 0) {
queryBuilder.append(" || ");
}

var sqlBuilder = new StringBuilder();
sqlBuilder.append("(WITH mvtgeom AS (\n");

for (int j = 0; j < layerQueries.size(); j++) {
if (j != 0) {
sqlBuilder.append("UNION\n");
}
var layerQuery = layerQueries.get(j).getSql().replace(";", "");
sqlBuilder.append(String.format("""
SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(%d, %d, %d)) AS geom, t.tags, t.id
FROM (%s) AS t
WHERE t.geom && ST_TileEnvelope(%d, %d, %d, margin => (64.0/4096))
""",
tileCoord.z(), tileCoord.x(), tileCoord.y(),
layerQuery,
tileCoord.z(), tileCoord.x(), tileCoord.y()));
}

queryBuilder.append(sqlBuilder)
.append(String.format(") SELECT ST_AsMVT(mvtgeom.*, '%s') FROM mvtgeom\n)", layerId));
}
}
queryBuilder.append(") mvtTile");
return queryBuilder.toString().replace("$zoom", String.valueOf(tileCoord.z()));
}
private Map<Integer, TileQuery> cache = new ConcurrentHashMap<>();

@Override
public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
String query = withQuery(tileCoord);

logger.debug("Executing query: {}", query);

long start = System.currentTimeMillis();

try (Connection connection = datasource.getConnection();
Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery(query);
ByteArrayOutputStream data = new ByteArrayOutputStream()) {

int length = 0;

try (OutputStream gzip = new GZIPOutputStream(data)) {
while (resultSet.next()) {
byte[] bytes = resultSet.getBytes(1);
length += bytes.length;
gzip.write(bytes);
}
}

long stop = System.currentTimeMillis();
long duration = stop - start;

// Log slow queries (> 10s)
if (duration > 10_000) {
logger.warn("Executed query for tile {} in {} ms: {}", tileCoord, duration, query);
}

if (length > 0) {
return ByteBuffer.wrap(data.toByteArray());
} else {
return ByteBuffer.allocate(0);
}
var query = cache.computeIfAbsent(tileCoord.z(), z -> prepareQuery(tileset, z));
try (var connection = datasource.getConnection()) {
return query.execute(connection, tileCoord);
} catch (Exception e) {
logger.error(e.getMessage());
throw new TileStoreException(e);
}
}
Expand All @@ -164,4 +80,132 @@ public void delete(TileCoord tileCoord) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}


public static TileQuery prepareQuery(Tileset tileset, int zoom) {
// Initialize a builder for the tile query
var tileQuery = new StringBuilder();
tileQuery.append("SELECT (");

// Iterate over the layers and keep track of the number of layers and parameters included in the
// final query
var layers = tileset.getVectorLayers();
var layerCount = 0;
var paramCount = 0;
for (var layer : layers) {

// Initialize a builder for the layer query
var layerQuery = new StringBuilder();
var layerHead = "(WITH mvtGeom AS (";
layerQuery.append(layerHead);

// Iterate over the queries and keep track of the number of queries included in the final
// query
var queries = layer.getQueries();
var queryCount = 0;
for (var query : queries) {

// Only include the query if the zoom level is in the range
if (query.getMinzoom() <= zoom && zoom < query.getMaxzoom()) {

// Add a union between queries
if (queryCount > 0) {
layerQuery.append("UNION ");
}

// Add the query to the layer query
var sql = query.getSql()
.replace(";", "")
.replace("?", "??")
.replace("$zoom", String.valueOf(zoom));
var queryWithParams = String.format("""
SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id
FROM (%s) AS t
WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))
""", sql);
layerQuery.append(queryWithParams);

// Increase the parameter count (e.g. ?) and query count
paramCount += 6;
queryCount++;
}
}

// Add the tail of the layer query
var layerQueryTail =
String.format(") SELECT ST_AsMVT(mvtGeom.*, '%s') FROM mvtGeom)", layer.getId());
layerQuery.append(layerQueryTail);

// Only include the layer query if queries were included for this layer
if (queryCount > 0) {

// Add the concatenation between layer queries
if (layerCount > 0) {
tileQuery.append(" || ");
}

// Add the layer query to the mvt query
tileQuery.append(layerQuery);

// Increase the layer count
layerCount++;
}
}

// Add the tail of the tile query
var tileQueryTail = ") mvtTile";
tileQuery.append(tileQueryTail);

// Log the resulting query
var query = tileQuery.toString().replace("\n", " ");
logger.debug("query: {}", query);

return new TileQuery(query, paramCount);
}

public static class TileQuery {

private final String query;

private final int paramCount;

public TileQuery(String query, int paramCount) {
this.query = query;
this.paramCount = paramCount;
}

public ByteBuffer execute(Connection connection, TileCoord tileCoord)
throws SQLException, IOException {
long start = System.currentTimeMillis();
try (var statement = connection.prepareStatement(query)) {

// Set the parameters for the tile
for (int i = 0; i < paramCount; i += 3) {
statement.setInt(i + 1, tileCoord.z());
statement.setInt(i + 2, tileCoord.x());
statement.setInt(i + 3, tileCoord.y());
}

// Fetch and compress the tile data
try (ByteArrayOutputStream data = new ByteArrayOutputStream();) {
try (ResultSet resultSet = statement.executeQuery();
OutputStream gzip = new GZIPOutputStream(data)) {
while (resultSet.next()) {
byte[] bytes = resultSet.getBytes(1);
gzip.write(bytes);
}
}
return ByteBuffer.wrap(data.toByteArray());

} finally {
// Log slow queries (> 10s)
long stop = System.currentTimeMillis();
long duration = stop - start;
if (duration > 10_000) {
logger.warn("Executed query for tile {} in {} ms", tileCoord, duration);
}
}
}
}
}

}
11 changes: 9 additions & 2 deletions baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,13 @@ public static DataSource createDataSource(String jdbcUrl, int poolSize) {
if (poolSize < 1) {
throw new IllegalArgumentException("PoolSize cannot be inferior to 1");
}
var multiQueriesJdbcUrl = withAllowMultiQueriesParameter(jdbcUrl);

var config = new HikariConfig();
config.setJdbcUrl(multiQueriesJdbcUrl);
config.setJdbcUrl(jdbcUrl);
config.setMaximumPoolSize(poolSize);
config.addDataSourceProperty("allowMultiQueries", true);
config.addDataSourceProperty("prepareThreshold", 100);

return new HikariDataSource(config);
}

Expand Down Expand Up @@ -154,6 +157,10 @@ public static DataSource createDataSource(Database datasource) {
if (datasource.getReadOnly() != null) {
config.setReadOnly(datasource.getReadOnly());
}

config.addDataSourceProperty("allowMultiQueries", true);
config.addDataSourceProperty("prepareThreshold", 100);

return new HikariDataSource(config);
}

Expand Down

0 comments on commit cdb0e85

Please sign in to comment.