From 2d88eea9df8370ce04553dd6c915d40b2a655540 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sat, 17 Feb 2024 21:00:21 +0100 Subject: [PATCH] Improve the workflow for the daylight distribution (#823) --- .../collection/FixedSizeDataList.java | 8 +- .../function/EntityGeometryBuilder.java | 5 +- .../function/EntityProjectionTransformer.java | 3 +- .../function/RelationMultiPolygonBuilder.java | 6 +- .../postgres/PostgresNodeRepository.java | 4 +- .../postgres/PostgresRelationRepository.java | 4 +- .../postgres/PostgresWayRepository.java | 4 +- .../storage/postgres/PostgresDataTable.java | 2 +- .../baremaps/utils/ProjectionTransformer.java | 101 ++++++++++------- .../baremaps/workflow/WorkflowContext.java | 13 +-- .../tasks/CreateGeocoderOpenStreetMap.java | 4 +- .../baremaps/workflow/tasks/ExecuteSql.java | 1 + .../baremaps/workflow/tasks/ImportOsmOsc.java | 5 +- .../baremaps/workflow/tasks/ImportOsmPbf.java | 4 +- basemap/layers/highway/clean.sql | 2 +- basemap/queries/osm_drop_indexes.sql | 11 ++ basemap/queries/osm_nodes.sql | 4 +- basemap/queries/osm_relations.sql | 4 +- basemap/queries/osm_ways.sql | 4 +- daylight/config.js | 1 + daylight/style.js | 31 +++-- daylight/workflow.js | 106 +++++++++--------- 22 files changed, 171 insertions(+), 156 deletions(-) create mode 100644 basemap/queries/osm_drop_indexes.sql diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/FixedSizeDataList.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/FixedSizeDataList.java index 6b26d0aed..e48dad64f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/FixedSizeDataList.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/FixedSizeDataList.java @@ -67,8 +67,8 @@ public FixedSizeDataList(FixedSizeDataType dataType, Memory memory) { private void write(long index, E value) { long position = index * dataType.size(); - int segmentIndex = (int) (position / dataType.size()); - int segmentOffset = (int) (position % dataType.size()); + int segmentIndex = (int) (position / memory.segmentSize()); + int segmentOffset = (int) (position % memory.segmentSize()); ByteBuffer segment = memory.segment(segmentIndex); dataType.write(segment, segmentOffset, value); } @@ -100,8 +100,8 @@ public void set(long index, E value) { @Override public E get(long index) { long position = index * dataType.size(); - int segmentIndex = (int) (position / dataType.size()); - int segmentOffset = (int) (position % dataType.size()); + int segmentIndex = (int) (position / memory.segmentSize()); + int segmentOffset = (int) (position % memory.segmentSize()); ByteBuffer segment = memory.segment(segmentIndex); return dataType.read(segment, segmentOffset); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityGeometryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityGeometryBuilder.java index 9b0350df0..740d45ff1 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityGeometryBuilder.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityGeometryBuilder.java @@ -22,10 +22,7 @@ import java.util.List; import java.util.function.Consumer; import org.apache.baremaps.database.collection.DataMap; -import org.apache.baremaps.openstreetmap.model.Entity; -import org.apache.baremaps.openstreetmap.model.Node; -import org.apache.baremaps.openstreetmap.model.Relation; -import org.apache.baremaps.openstreetmap.model.Way; +import org.apache.baremaps.openstreetmap.model.*; import org.locationtech.jts.geom.*; /** A consumer that builds and sets the geometry of OpenStreetMap entities via side effects. */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityProjectionTransformer.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityProjectionTransformer.java index 397f7986c..3a3c5157e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityProjectionTransformer.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityProjectionTransformer.java @@ -22,7 +22,6 @@ import java.util.function.Consumer; import org.apache.baremaps.openstreetmap.model.*; import org.apache.baremaps.utils.ProjectionTransformer; -import org.locationtech.jts.geom.Geometry; /** Changes the projection of the geometry of an entity via side-effects. */ public class EntityProjectionTransformer implements Consumer { @@ -51,7 +50,7 @@ public void accept(Entity entity) { if (sourceSrid != targetSrid && entity instanceof Element element && element.getGeometry() != null) { - Geometry geometry = projectionTransformer.transform(element.getGeometry()); + var geometry = projectionTransformer.transform(element.getGeometry()); element.setGeometry(geometry); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationMultiPolygonBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationMultiPolygonBuilder.java index c7981deb8..e80ff665a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationMultiPolygonBuilder.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationMultiPolygonBuilder.java @@ -73,7 +73,8 @@ public void accept(Relation relation) { } catch (Exception e) { logger.debug("Unable to build the geometry for relation #" + relation.getId(), e); - relation.setGeometry(GEOMETRY_FACTORY_WGS84.createMultiPolygon()); + var emptyMultiPolygon = GEOMETRY_FACTORY_WGS84.createMultiPolygon(); + relation.setGeometry(emptyMultiPolygon); } } @@ -162,7 +163,8 @@ private void buildMultiPolygon(Relation relation) { GEOMETRY_FACTORY_WGS84.createMultiPolygon(polygons.toArray(new Polygon[0])); relation.setGeometry(multiPolygon); } else { - relation.setGeometry(GEOMETRY_FACTORY_WGS84.createMultiPolygon()); + var emptyMultiPolygon = GEOMETRY_FACTORY_WGS84.createMultiPolygon(); + relation.setGeometry(emptyMultiPolygon); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java index 4b0a73ccb..829afe53f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java @@ -116,12 +116,12 @@ public PostgresNodeRepository(DataSource dataSource, String schema, String table this.dropTable = String.format("DROP TABLE IF EXISTS %1$s CASCADE", fullTableName); this.truncateTable = String.format("TRUNCATE TABLE %1$s", fullTableName); this.select = String.format( - "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asbinary(%10$s) FROM %1$s WHERE %2$s = ?", + "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asewkb(%10$s) FROM %1$s WHERE %2$s = ?", fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn, longitudeColumn, latitudeColumn, geometryColumn); this.selectIn = String.format( - "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asbinary(%10$s) FROM %1$s WHERE %2$s = ANY (?)", + "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asewkb(%10$s) FROM %1$s WHERE %2$s = ANY (?)", fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn, longitudeColumn, latitudeColumn, geometryColumn); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java index 32f4055e0..caef9d69a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java @@ -114,12 +114,12 @@ public PostgresRelationRepository(DataSource dataSource, String schema, String t this.dropTable = String.format("DROP TABLE IF EXISTS %1$s CASCADE", fullTableName); this.truncateTable = String.format("TRUNCATE TABLE %1$s", fullTableName); this.select = String.format( - "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asbinary(%11$s) FROM %1$s WHERE %2$s = ?", + "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asewkb(%11$s) FROM %1$s WHERE %2$s = ?", fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn, memberRefs, memberTypes, memberRoles, geometryColumn); this.selectIn = String.format( - "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asbinary(%11$s) FROM %1$s WHERE %2$s = ANY (?)", + "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asewkb(%11$s) FROM %1$s WHERE %2$s = ANY (?)", fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn, memberRefs, memberTypes, memberRoles, geometryColumn); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java index 5af4a1c9f..7e1650e22 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java @@ -116,12 +116,12 @@ public PostgresWayRepository(DataSource dataSource, String schema, String table, this.dropTable = String.format("DROP TABLE IF EXISTS %1$s CASCADE", fullTableName); this.truncateTable = String.format("TRUNCATE TABLE %1$s", fullTableName); this.select = String.format( - "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asbinary(%9$s) FROM %1$s WHERE %2$s = ?", + "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asewkb(%9$s) FROM %1$s WHERE %2$s = ?", fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn, nodesColumn, geometryColumn); this.selectIn = String.format( - "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asbinary(%9$s) FROM %1$s WHERE %2$s = ANY (?)", + "SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asewkb(%9$s) FROM %1$s WHERE %2$s = ANY (?)", fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn, nodesColumn, geometryColumn); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataTable.java index 08c804abd..52083dc0e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataTable.java @@ -154,7 +154,7 @@ protected static String select(DataRowType rowType) { var columns = rowType.columns().stream() .map(column -> { if (column.type().binding().isAssignableFrom(Geometry.class)) { - return String.format("st_asbinary(\"%s\") AS \"%s\"", column.name(), column.name()); + return String.format("st_asewkb(\"%s\") AS \"%s\"", column.name(), column.name()); } else { return String.format("\"%s\"", column.name()); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java index ca8328258..01d148232 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java @@ -21,17 +21,7 @@ import java.util.Objects; import java.util.stream.Stream; -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.CoordinateSequence; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.GeometryCollection; -import org.locationtech.jts.geom.LineString; -import org.locationtech.jts.geom.LinearRing; -import org.locationtech.jts.geom.MultiLineString; -import org.locationtech.jts.geom.MultiPoint; -import org.locationtech.jts.geom.MultiPolygon; -import org.locationtech.jts.geom.Point; -import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.geom.*; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.locationtech.jts.geom.util.GeometryTransformer; import org.locationtech.proj4j.CRSFactory; @@ -76,16 +66,6 @@ public ProjectionTransformer(int sourceSrid, int targetSrid) { Math.toDegrees(targetCRS.getProjection().getMaxLatitude())), new ProjCoordinate()); } - @Override - protected CoordinateSequence transformCoordinates( - CoordinateSequence coordinateSequence, - Geometry parent) { - Coordinate[] coordinateArray = Stream.of(coordinateSequence.toCoordinateArray()) - .filter(Objects::nonNull) - .map(this::transformCoordinate).toArray(Coordinate[]::new); - return new CoordinateArraySequence(coordinateArray); - } - private Coordinate transformCoordinate(Coordinate coordinate) { if (sourceSrid == targetSrid) { return coordinate; @@ -97,97 +77,134 @@ private Coordinate transformCoordinate(Coordinate coordinate) { return new Coordinate(c2.x, c2.y); } + @Override + protected CoordinateSequence transformCoordinates( + CoordinateSequence coordinateSequence, + Geometry parent) { + Coordinate[] coordinateArray = Stream.of(coordinateSequence.toCoordinateArray()) + .filter(Objects::nonNull) + .map(this::transformCoordinate).toArray(Coordinate[]::new); + return new CoordinateArraySequence(coordinateArray); + } + + @Override protected Geometry transformPoint(Point geom, Geometry parent) { try { var geometry = super.transformPoint(geom, parent); - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("Point cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createPoint(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformMultiPoint(MultiPoint geom, Geometry parent) { try { var geometry = super.transformMultiPoint(geom, parent); if (geometry instanceof Point point) { geometry = factory.createMultiPoint(new Point[] {point}); } - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("MultiPoint cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createMultiPoint(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformLinearRing(LinearRing geom, Geometry parent) { try { var geometry = super.transformLinearRing(geom, parent); - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("LinearRing cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createLinearRing(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformLineString(LineString geom, Geometry parent) { try { var geometry = super.transformLineString(geom, parent); - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("LineString cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createLineString(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformMultiLineString(MultiLineString geom, Geometry parent) { try { var geometry = super.transformMultiLineString(geom, parent); if (geometry instanceof LineString lineString) { geometry = factory.createMultiLineString(new LineString[] {lineString}); } - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("MultiLineString cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createMultiLineString(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformPolygon(Polygon geom, Geometry parent) { try { var geometry = super.transformPolygon(geom, parent); - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("Polygon cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createPolygon(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformMultiPolygon(MultiPolygon geom, Geometry parent) { try { var geometry = super.transformMultiPolygon(geom, parent); if (geometry instanceof Polygon polygon) { geometry = factory.createMultiPolygon(new Polygon[] {polygon}); } - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("MultiPolygon cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createMultiPolygon(); + geometry.setSRID(targetSrid); + return geometry; } } + @Override protected Geometry transformGeometryCollection(GeometryCollection geom, Geometry parent) { try { var geometry = super.transformGeometryCollection(geom, parent); - return withTargetSRID(geometry); + geometry.setSRID(targetSrid); + return geometry; } catch (Exception e) { logger.error("GeometryCollection cannot be reprojected", e); - return parent.getFactory().createEmpty(0); + var geometry = parent.getFactory().createGeometryCollection(); + geometry.setSRID(targetSrid); + return geometry; } } - - private Geometry withTargetSRID(Geometry outputGeom) { - outputGeom.setSRID(targetSrid); - return outputGeom; - } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java index d991d0718..172be38d5 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java @@ -18,7 +18,6 @@ package org.apache.baremaps.workflow; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -53,7 +52,7 @@ public WorkflowContext(Path dataDir, Path cacheDir) { this.cacheDir = cacheDir; } - private Map dataSources = new ConcurrentHashMap<>() {}; + private Map dataSources = new ConcurrentHashMap<>(); /** * Returns the data source associated with the specified database. @@ -65,15 +64,11 @@ public DataSource getDataSource(Object database) { return dataSources.computeIfAbsent(database, PostgresUtils::createDataSourceFromObject); } - public DataMap getCoordinateMap(Path path) throws IOException { - if (Files.size(path) > 1 << 30) { - return getMemoryAlignedDataMap("coordinates", new LonLatDataType()); - } else { - return getMonotonicDataMap("coordinates", new LonLatDataType()); - } + public DataMap getCoordinateMap() throws IOException { + return getMemoryAlignedDataMap("coordinates", new LonLatDataType()); } - public DataMap> getReferenceMap(Path path) throws IOException { + public DataMap> getReferenceMap() throws IOException { return getMonotonicDataMap("references", new LongListDataType()); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java index fad65d9d1..347fd7940 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java @@ -68,8 +68,8 @@ public CreateGeocoderOpenStreetMap(Path file, Path indexDirectory) { public void execute(WorkflowContext context) throws Exception { var path = file.toAbsolutePath(); - var coordinateMap = context.getCoordinateMap(path); - var referenceMap = context.getReferenceMap(path); + var coordinateMap = context.getCoordinateMap(); + var referenceMap = context.getReferenceMap(); var directory = FSDirectory.open(indexDirectory); var config = new IndexWriterConfig(GeocoderConstants.ANALYZER); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java index 6a067bb19..f167f0855 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java @@ -80,6 +80,7 @@ public void execute(WorkflowContext context) throws Exception { logger.info("Execute SQL query: {}", query.replaceAll("\\s+", " ")); connection.createStatement().execute(query); } catch (SQLException e) { + logger.error("Failed to execute query: {}", query.replaceAll("\\s+", " ")); throw new WorkflowException(e); } }); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index 20f5d6fa7..18bd59915 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -80,11 +80,12 @@ public void execute(WorkflowContext context) throws Exception { var wayRepository = new PostgresWayRepository(datasource); var relationRepository = new PostgresRelationRepository(datasource); - var coordinateMap = context.getCoordinateMap(path); - var referenceMap = context.getReferenceMap(path); + var coordinateMap = context.getCoordinateMap(); + var referenceMap = context.getReferenceMap(); var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap); var referenceMapBuilder = new ReferenceMapBuilder(referenceMap); + var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); var reprojectGeometry = new EntityProjectionTransformer(4326, databaseSrid); var prepareGeometries = coordinateMapBuilder diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index c529c7dae..22637b420 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -103,8 +103,8 @@ public void execute(WorkflowContext context) throws Exception { relationRepository.create(); } - var coordinateMap = context.getCoordinateMap(path); - var referenceMap = context.getReferenceMap(path); + var coordinateMap = context.getCoordinateMap(); + var referenceMap = context.getReferenceMap(); execute( path, diff --git a/basemap/layers/highway/clean.sql b/basemap/layers/highway/clean.sql index b3fa73218..d8c26559b 100644 --- a/basemap/layers/highway/clean.sql +++ b/basemap/layers/highway/clean.sql @@ -13,7 +13,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -DROP VIEW IF EXISTS osm_highway CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_highway CASCADE; DROP VIEW IF EXISTS osm_highway_z20 CASCADE; DROP VIEW IF EXISTS osm_highway_z19 CASCADE; diff --git a/basemap/queries/osm_drop_indexes.sql b/basemap/queries/osm_drop_indexes.sql new file mode 100644 index 000000000..817b8d8be --- /dev/null +++ b/basemap/queries/osm_drop_indexes.sql @@ -0,0 +1,11 @@ +DROP INDEX IF EXISTS osm_nodes_tags_index; +DROP INDEX IF EXISTS osm_nodes_tags_tsvector_index; +DROP INDEX IF EXISTS osm_nodes_geom_index; + +DROP INDEX IF EXISTS osm_ways_tags_index; +DROP INDEX IF EXISTS osm_ways_tags_tsvector_index; +DROP INDEX IF EXISTS osm_ways_geom_index; + +DROP INDEX IF EXISTS osm_relations_tags_index; +DROP INDEX IF EXISTS osm_relations_tags_tsvector_index; +DROP INDEX IF EXISTS osm_relations_geom_index; diff --git a/basemap/queries/osm_nodes.sql b/basemap/queries/osm_nodes.sql index 290b84b20..26ba537a0 100644 --- a/basemap/queries/osm_nodes.sql +++ b/basemap/queries/osm_nodes.sql @@ -13,6 +13,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -CREATE INDEX IF NOT EXISTS osm_nodes_tags_index ON osm_nodes USING gin (tags); -CREATE INDEX IF NOT EXISTS osm_nodes_tags_tsvector_index ON osm_nodes USING gin (to_tsvector('english', tags)); CREATE INDEX IF NOT EXISTS osm_nodes_geom_index ON osm_nodes USING gist (geom); +CREATE INDEX IF NOT EXISTS osm_nodes_tags_index ON osm_nodes USING gin (tags); +-- CREATE INDEX IF NOT EXISTS osm_nodes_tags_tsvector_index ON osm_nodes USING gin (to_tsvector('english', tags)); diff --git a/basemap/queries/osm_relations.sql b/basemap/queries/osm_relations.sql index ceae0432e..80d735f6e 100644 --- a/basemap/queries/osm_relations.sql +++ b/basemap/queries/osm_relations.sql @@ -13,6 +13,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -CREATE INDEX IF NOT EXISTS osm_relations_tags_index ON osm_relations USING gin (tags); -CREATE INDEX IF NOT EXISTS osm_relations_tags_tsvector_index ON osm_relations USING gin (to_tsvector('english', tags)); CREATE INDEX IF NOT EXISTS osm_relations_geom_index ON osm_relations USING gist (geom); +CREATE INDEX IF NOT EXISTS osm_relations_tags_index ON osm_relations USING gin (tags); +--CREATE INDEX IF NOT EXISTS osm_relations_tags_tsvector_index ON osm_relations USING gin (to_tsvector('english', tags)); \ No newline at end of file diff --git a/basemap/queries/osm_ways.sql b/basemap/queries/osm_ways.sql index 6e707f39c..f64cf7034 100644 --- a/basemap/queries/osm_ways.sql +++ b/basemap/queries/osm_ways.sql @@ -14,5 +14,5 @@ -- limitations under the License. CREATE INDEX IF NOT EXISTS osm_ways_geom_index ON osm_ways USING gist (geom); -CREATE INDEX IF NOT EXISTS osm_ways_tags_tsvector_index ON osm_ways USING gin (to_tsvector('english', tags)); -CREATE INDEX IF NOT EXISTS osm_ways_tags_index ON osm_ways USING gin (tags); \ No newline at end of file +CREATE INDEX IF NOT EXISTS osm_ways_tags_index ON osm_ways USING gin (tags); +--CREATE INDEX IF NOT EXISTS osm_ways_tags_tsvector_index ON osm_ways USING gin (to_tsvector('english', tags)); diff --git a/daylight/config.js b/daylight/config.js index 124861f55..d29fd3152 100644 --- a/daylight/config.js +++ b/daylight/config.js @@ -20,4 +20,5 @@ export default { "center": [0, 0], "bounds": [-180, -85.0511, 180, 85.0511], "zoom": 3, + "daylightVersion": "v1.39", } diff --git a/daylight/style.js b/daylight/style.js index b551531a0..5617b0202 100644 --- a/daylight/style.js +++ b/daylight/style.js @@ -16,9 +16,10 @@ **/ import config from "./config.js"; + import background from "../basemap/layers/background/style.js"; import aeroway_line from "../basemap/layers/aeroway/line.js"; -import aeroway_polygon from "../basemap/layers/aeroway/polygon.js"; +import aeroway_fill from "../basemap/layers/aeroway/fill.js"; import amenity_background from "../basemap/layers/amenity/background.js"; import amenity_fountain from "../basemap/layers/amenity/fountain.js"; import amenity_overlay from "../basemap/layers/amenity/overlay.js"; @@ -39,29 +40,27 @@ import railway_line from "../basemap/layers/railway/line.js"; import highway_line from '../basemap/layers/highway/highway_line.js'; import highway_outline from '../basemap/layers/highway/highway_outline.js'; -import highway_dash from '../basemap/layers/highway/highway_dash.js'; import highway_tunnel_line from '../basemap/layers/highway/tunnel_line.js'; import highway_tunnel_outline from '../basemap/layers/highway/tunnel_outline.js'; -import highway_pedestrian_area from '../basemap/layers/highway/pedestrian_area.js'; +import highway_fill from '../basemap/layers/highway/highway_fill.js'; import highway_bridge_line from '../basemap/layers/highway/bridge_line.js'; import highway_bridge_outline from '../basemap/layers/highway/bridge_outline.js'; import highway_construction_line from "../basemap/layers/highway/construction_line.js"; -import highway_construction_dash from "../basemap/layers/highway/construction_dash.js"; import highway_label from '../basemap/layers/highway/highway_label.js'; +import ocean_overlay from '../basemap/layers/ocean/overlay.js'; import route_line from "../basemap/layers/route/style.js" -import building_shape from "../basemap/layers/building/shape.js"; -import building_number from "../basemap/layers/building/number.js"; +import building_fill from "../basemap/layers/building/fill.js"; +import building_extrusion from "../basemap/layers/building/extrusion.js"; import man_made_bridge from "../basemap/layers/man_made/bridge.js"; import man_made_pier_line from "../basemap/layers/man_made/pier_line.js"; import man_made_pier_label from "../basemap/layers/man_made/pier_label.js"; import waterway_line from "../basemap/layers/waterway/line.js" +import waterway_area from "../basemap/layers/waterway/area.js" import waterway_label from "../basemap/layers/waterway/label.js" -import waterway_tunnel_line from "../basemap/layers/waterway/tunnel_line.js" -import waterway_tunnel_casing from "../basemap/layers/waterway/tunnel_casing.js" import icon from "../basemap/layers/point/icon.js"; +import place from '../basemap/layers/point/place.js'; import country_label from '../basemap/layers/point/country_label.js'; -import point_label from '../basemap/layers/point/point_label.js'; import coastline from './layers/coastline/style.js'; // import landcover from './layers/landcover/style.js'; @@ -82,7 +81,7 @@ export default { "layers": [ background, power_background, - aeroway_polygon, + aeroway_fill, landuse_background, leisure_background, amenity_background, @@ -94,21 +93,17 @@ export default { coastline, // landcover, waterway_line, - waterway_tunnel_casing, - waterway_tunnel_line, + waterway_area, man_made_bridge, amenity_fountain, highway_tunnel_outline, highway_tunnel_line, railway_tunnel, - building_shape, - building_number, - highway_construction_dash, + building_fill, highway_construction_line, highway_outline, highway_line, - highway_dash, - highway_pedestrian_area, + highway_fill, railway_line, highway_bridge_outline, highway_bridge_line, @@ -124,7 +119,7 @@ export default { boundary_line, waterway_label, icon, - point_label, + place, country_label, ], }; \ No newline at end of file diff --git a/daylight/workflow.js b/daylight/workflow.js index aef8e1a95..62bde2fc6 100644 --- a/daylight/workflow.js +++ b/daylight/workflow.js @@ -20,33 +20,31 @@ export default { "tasks": [ { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/planet-v1.33.osm.pbf", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/planet-${config.daylightVersion}.osm.pbf`, "target": "data/data.osm.pbf" }, { "type": "ImportOsmPbf", "file": "data/data.osm.pbf", - "cache": "cache/", "database": config.database, "databaseSrid": 3857, "replaceExisting": true, }, { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/ml-buildings-v1.33.osm.pbf", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/ml-buildings-${config.daylightVersion}.osm.pbf`, "target": "data/buildings.osm.pbf" }, { "type": "ImportOsmPbf", "file": "data/buildings.osm.pbf", - "cache": "building_cache/", "database": config.database, "databaseSrid": 3857, "replaceExisting": false, }, { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/fb-ml-roads-v1.33.osc.gz", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/fb-ml-roads-${config.daylightVersion}.osc.gz`, "target": "data/roads.osc.gz" }, { @@ -58,7 +56,7 @@ export default { }, { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/admin-v1.33.osc.gz", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/admin-${config.daylightVersion}.osc.gz`, "target": "data/admin.osc.gz" }, { @@ -70,7 +68,7 @@ export default { }, { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/coastlines-v1.33.tgz", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/coastlines-${config.daylightVersion}.tgz`, "target": "data/coastlines.tgz" }, { @@ -93,7 +91,7 @@ export default { }, { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/preferred-localization-v1.33.tsv", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/preferred-localization-${config.daylightVersion}.tsv`, "target": "data/preferred-localization.tsv" }, { @@ -103,7 +101,7 @@ export default { }, { "type": "DownloadUrl", - "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/important-features-v1.33.json", + "source": `https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/${config.daylightVersion}/important-features-${config.daylightVersion}.json`, "target": "data/important-features.json" }, { @@ -231,27 +229,6 @@ export default { }, ] }, - { - "id": "daylight-polygon", - "needs": ["daylight-member"], - "tasks": [ - { - "type": "ExecuteSql", - "file": "../basemap/layers/polygon/clean.sql", - "database": config.database, - }, - { - "type": "ExecuteSql", - "file": "../basemap/layers/polygon/prepare.sql", - "database": config.database, - }, - { - "type": "ExecuteSql", - "file": "../basemap/layers/polygon/index.sql", - "database": config.database, - }, - ] - }, { "id": "daylight-highway", "needs": ["daylight-linestring"], @@ -337,114 +314,133 @@ export default { ] }, { - "id": "daylight-natural", - "needs": ["daylight-polygon"], + "id": "daylight-waterway", + "needs": ["daylight-linestring"], "tasks": [ { "type": "ExecuteSql", - "file": "../basemap/layers/natural/clean.sql", + "file": "../basemap/layers/waterway/clean.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/natural/prepare.sql", + "file": "../basemap/layers/waterway/prepare.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/natural/simplify.sql", + "file": "../basemap/layers/waterway/simplify.sql", "database": config.database, "parallel": true, }, { "type": "ExecuteSql", - "file": "../basemap/layers/natural/index.sql", + "file": "../basemap/layers/waterway/index.sql", "database": config.database, "parallel": true }, ] }, { - "id": "daylight-landuse", + "id": "daylight-polygon", + "needs": ["daylight-member"], + "tasks": [ + { + "type": "ExecuteSql", + "file": "../basemap/layers/polygon/clean.sql", + "database": config.database, + }, + { + "type": "ExecuteSql", + "file": "../basemap/layers/polygon/prepare.sql", + "database": config.database, + }, + { + "type": "ExecuteSql", + "file": "../basemap/layers/polygon/index.sql", + "database": config.database, + }, + ] + }, + { + "id": "daylight-natural", "needs": ["daylight-polygon"], "tasks": [ { "type": "ExecuteSql", - "file": "../basemap/layers/landuse/clean.sql", + "file": "../basemap/layers/natural/clean.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/landuse/prepare.sql", + "file": "../basemap/layers/natural/prepare.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/landuse/simplify.sql", + "file": "../basemap/layers/natural/simplify.sql", "database": config.database, "parallel": true, }, { "type": "ExecuteSql", - "file": "../basemap/layers/landuse/index.sql", + "file": "../basemap/layers/natural/index.sql", "database": config.database, "parallel": true }, ] }, { - "id": "daylight-leisure", - "needs": [ - "daylight-polygon" - ], + "id": "daylight-landuse", + "needs": ["daylight-polygon"], "tasks": [ { "type": "ExecuteSql", - "file": "../basemap/layers/leisure/clean.sql", + "file": "../basemap/layers/landuse/clean.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/leisure/prepare.sql", + "file": "../basemap/layers/landuse/prepare.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/leisure/simplify.sql", + "file": "../basemap/layers/landuse/simplify.sql", "database": config.database, "parallel": true, }, { "type": "ExecuteSql", - "file": "../basemap/layers/leisure/index.sql", + "file": "../basemap/layers/landuse/index.sql", "database": config.database, "parallel": true }, ] }, { - "id": "daylight-waterway", - "needs": ["daylight-linestring"], + "id": "daylight-leisure", + "needs": ["daylight-polygon"], "tasks": [ { "type": "ExecuteSql", - "file": "../basemap/layers/waterway/clean.sql", + "file": "../basemap/layers/leisure/clean.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/waterway/prepare.sql", + "file": "../basemap/layers/leisure/prepare.sql", "database": config.database, }, { "type": "ExecuteSql", - "file": "../basemap/layers/waterway/simplify.sql", + "file": "../basemap/layers/leisure/simplify.sql", "database": config.database, "parallel": true, }, { "type": "ExecuteSql", - "file": "../basemap/layers/waterway/index.sql", + "file": "../basemap/layers/leisure/index.sql", "database": config.database, "parallel": true },