Skip to content

Commit

Permalink
Improve the workflow for the daylight distribution (#823)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bchapuis
bchapuis authored Feb 17, 2024
1 parent 4941a5d commit 2d88eea
Show file tree
Hide file tree
Showing 22 changed files with 171 additions and 156 deletions.
8 changes: 4 additions & 4 deletions baremaps-core/src/main/java/org/apache/baremaps/database/collection/FixedSizeDataList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ public FixedSizeDataList(FixedSizeDataType<E> dataType, Memory memory) {

private void write(long index, E value) {
long position = index * dataType.size();
int segmentIndex = (int) (position / dataType.size());
int segmentOffset = (int) (position % dataType.size());
int segmentIndex = (int) (position / memory.segmentSize());
int segmentOffset = (int) (position % memory.segmentSize());
ByteBuffer segment = memory.segment(segmentIndex);
dataType.write(segment, segmentOffset, value);
}
Expand Down Expand Up @@ -100,8 +100,8 @@ public void set(long index, E value) {
@Override
public E get(long index) {
long position = index * dataType.size();
int segmentIndex = (int) (position / dataType.size());
int segmentOffset = (int) (position % dataType.size());
int segmentIndex = (int) (position / memory.segmentSize());
int segmentOffset = (int) (position % memory.segmentSize());
ByteBuffer segment = memory.segment(segmentIndex);
return dataType.read(segment, segmentOffset);
}
Expand Down
5 changes: 1 addition & 4 deletions ...-core/src/main/java/org/apache/baremaps/openstreetmap/function/EntityGeometryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,7 @@
import java.util.List;
import java.util.function.Consumer;
import org.apache.baremaps.database.collection.DataMap;
import org.apache.baremaps.openstreetmap.model.Entity;
import org.apache.baremaps.openstreetmap.model.Node;
import org.apache.baremaps.openstreetmap.model.Relation;
import org.apache.baremaps.openstreetmap.model.Way;
import org.apache.baremaps.openstreetmap.model.*;
import org.locationtech.jts.geom.*;

/** A consumer that builds and sets the geometry of OpenStreetMap entities via side effects. */
Expand Down
3 changes: 1 addition & 2 deletions ...src/main/java/org/apache/baremaps/openstreetmap/function/EntityProjectionTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import java.util.function.Consumer;
import org.apache.baremaps.openstreetmap.model.*;
import org.apache.baremaps.utils.ProjectionTransformer;
import org.locationtech.jts.geom.Geometry;

/** Changes the projection of the geometry of an entity via side-effects. */
public class EntityProjectionTransformer implements Consumer<Entity> {
Expand Down Expand Up @@ -51,7 +50,7 @@ public void accept(Entity entity) {
if (sourceSrid != targetSrid
&& entity instanceof Element element
&& element.getGeometry() != null) {
Geometry geometry = projectionTransformer.transform(element.getGeometry());
var geometry = projectionTransformer.transform(element.getGeometry());
element.setGeometry(geometry);
}
}
Expand Down
6 changes: 4 additions & 2 deletions ...src/main/java/org/apache/baremaps/openstreetmap/function/RelationMultiPolygonBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ public void accept(Relation relation) {

} catch (Exception e) {
logger.debug("Unable to build the geometry for relation #" + relation.getId(), e);
relation.setGeometry(GEOMETRY_FACTORY_WGS84.createMultiPolygon());
var emptyMultiPolygon = GEOMETRY_FACTORY_WGS84.createMultiPolygon();
relation.setGeometry(emptyMultiPolygon);
}
}

Expand Down Expand Up @@ -162,7 +163,8 @@ private void buildMultiPolygon(Relation relation) {
GEOMETRY_FACTORY_WGS84.createMultiPolygon(polygons.toArray(new Polygon[0]));
relation.setGeometry(multiPolygon);
} else {
relation.setGeometry(GEOMETRY_FACTORY_WGS84.createMultiPolygon());
var emptyMultiPolygon = GEOMETRY_FACTORY_WGS84.createMultiPolygon();
relation.setGeometry(emptyMultiPolygon);
}
}

Expand Down
4 changes: 2 additions & 2 deletions ...core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,12 @@ public PostgresNodeRepository(DataSource dataSource, String schema, String table
this.dropTable = String.format("DROP TABLE IF EXISTS %1$s CASCADE", fullTableName);
this.truncateTable = String.format("TRUNCATE TABLE %1$s", fullTableName);
this.select = String.format(
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asbinary(%10$s) FROM %1$s WHERE %2$s = ?",
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asewkb(%10$s) FROM %1$s WHERE %2$s = ?",
fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn,
longitudeColumn, latitudeColumn, geometryColumn);
this.selectIn = String.format(
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asbinary(%10$s) FROM %1$s WHERE %2$s = ANY (?)",
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, st_asewkb(%10$s) FROM %1$s WHERE %2$s = ANY (?)",
fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn,
longitudeColumn, latitudeColumn, geometryColumn);
Expand Down
4 changes: 2 additions & 2 deletions .../src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,12 @@ public PostgresRelationRepository(DataSource dataSource, String schema, String t
this.dropTable = String.format("DROP TABLE IF EXISTS %1$s CASCADE", fullTableName);
this.truncateTable = String.format("TRUNCATE TABLE %1$s", fullTableName);
this.select = String.format(
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asbinary(%11$s) FROM %1$s WHERE %2$s = ?",
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asewkb(%11$s) FROM %1$s WHERE %2$s = ?",
fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn,
memberRefs, memberTypes, memberRoles, geometryColumn);
this.selectIn = String.format(
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asbinary(%11$s) FROM %1$s WHERE %2$s = ANY (?)",
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, st_asewkb(%11$s) FROM %1$s WHERE %2$s = ANY (?)",
fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn,
memberRefs, memberTypes, memberRoles, geometryColumn);
Expand Down
4 changes: 2 additions & 2 deletions ...-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,12 @@ public PostgresWayRepository(DataSource dataSource, String schema, String table,
this.dropTable = String.format("DROP TABLE IF EXISTS %1$s CASCADE", fullTableName);
this.truncateTable = String.format("TRUNCATE TABLE %1$s", fullTableName);
this.select = String.format(
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asbinary(%9$s) FROM %1$s WHERE %2$s = ?",
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asewkb(%9$s) FROM %1$s WHERE %2$s = ?",
fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn,
nodesColumn, geometryColumn);
this.selectIn = String.format(
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asbinary(%9$s) FROM %1$s WHERE %2$s = ANY (?)",
"SELECT %2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, st_asewkb(%9$s) FROM %1$s WHERE %2$s = ANY (?)",
fullTableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn,
nodesColumn, geometryColumn);
Expand Down
2 changes: 1 addition & 1 deletion baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ protected static String select(DataRowType rowType) {
var columns = rowType.columns().stream()
.map(column -> {
if (column.type().binding().isAssignableFrom(Geometry.class)) {
return String.format("st_asbinary(\"%s\") AS \"%s\"", column.name(), column.name());
return String.format("st_asewkb(\"%s\") AS \"%s\"", column.name(), column.name());
} else {
return String.format("\"%s\"", column.name());
}
Expand Down
101 changes: 59 additions & 42 deletions baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,7 @@

import java.util.Objects;
import java.util.stream.Stream;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.CoordinateSequence;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
import org.locationtech.jts.geom.LineString;
import org.locationtech.jts.geom.LinearRing;
import org.locationtech.jts.geom.MultiLineString;
import org.locationtech.jts.geom.MultiPoint;
import org.locationtech.jts.geom.MultiPolygon;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Polygon;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.geom.impl.CoordinateArraySequence;
import org.locationtech.jts.geom.util.GeometryTransformer;
import org.locationtech.proj4j.CRSFactory;
Expand Down Expand Up @@ -76,16 +66,6 @@ public ProjectionTransformer(int sourceSrid, int targetSrid) {
Math.toDegrees(targetCRS.getProjection().getMaxLatitude())), new ProjCoordinate());
}

@Override
protected CoordinateSequence transformCoordinates(
CoordinateSequence coordinateSequence,
Geometry parent) {
Coordinate[] coordinateArray = Stream.of(coordinateSequence.toCoordinateArray())
.filter(Objects::nonNull)
.map(this::transformCoordinate).toArray(Coordinate[]::new);
return new CoordinateArraySequence(coordinateArray);
}

private Coordinate transformCoordinate(Coordinate coordinate) {
if (sourceSrid == targetSrid) {
return coordinate;
Expand All @@ -97,97 +77,134 @@ private Coordinate transformCoordinate(Coordinate coordinate) {
return new Coordinate(c2.x, c2.y);
}

@Override
protected CoordinateSequence transformCoordinates(
CoordinateSequence coordinateSequence,
Geometry parent) {
Coordinate[] coordinateArray = Stream.of(coordinateSequence.toCoordinateArray())
.filter(Objects::nonNull)
.map(this::transformCoordinate).toArray(Coordinate[]::new);
return new CoordinateArraySequence(coordinateArray);
}

@Override
protected Geometry transformPoint(Point geom, Geometry parent) {
try {
var geometry = super.transformPoint(geom, parent);
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("Point cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createPoint();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformMultiPoint(MultiPoint geom, Geometry parent) {
try {
var geometry = super.transformMultiPoint(geom, parent);
if (geometry instanceof Point point) {
geometry = factory.createMultiPoint(new Point[] {point});
}
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("MultiPoint cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createMultiPoint();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformLinearRing(LinearRing geom, Geometry parent) {
try {
var geometry = super.transformLinearRing(geom, parent);
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("LinearRing cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createLinearRing();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformLineString(LineString geom, Geometry parent) {
try {
var geometry = super.transformLineString(geom, parent);
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("LineString cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createLineString();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformMultiLineString(MultiLineString geom, Geometry parent) {
try {
var geometry = super.transformMultiLineString(geom, parent);
if (geometry instanceof LineString lineString) {
geometry = factory.createMultiLineString(new LineString[] {lineString});
}
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("MultiLineString cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createMultiLineString();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformPolygon(Polygon geom, Geometry parent) {
try {
var geometry = super.transformPolygon(geom, parent);
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("Polygon cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createPolygon();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformMultiPolygon(MultiPolygon geom, Geometry parent) {
try {
var geometry = super.transformMultiPolygon(geom, parent);
if (geometry instanceof Polygon polygon) {
geometry = factory.createMultiPolygon(new Polygon[] {polygon});
}
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("MultiPolygon cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createMultiPolygon();
geometry.setSRID(targetSrid);
return geometry;
}
}

@Override
protected Geometry transformGeometryCollection(GeometryCollection geom, Geometry parent) {
try {
var geometry = super.transformGeometryCollection(geom, parent);
return withTargetSRID(geometry);
geometry.setSRID(targetSrid);
return geometry;
} catch (Exception e) {
logger.error("GeometryCollection cannot be reprojected", e);
return parent.getFactory().createEmpty(0);
var geometry = parent.getFactory().createGeometryCollection();
geometry.setSRID(targetSrid);
return geometry;
}
}

private Geometry withTargetSRID(Geometry outputGeom) {
outputGeom.setSRID(targetSrid);
return outputGeom;
}
}
13 changes: 4 additions & 9 deletions baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.baremaps.workflow;



import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -53,7 +52,7 @@ public WorkflowContext(Path dataDir, Path cacheDir) {
this.cacheDir = cacheDir;
}

private Map<Object, DataSource> dataSources = new ConcurrentHashMap<>() {};
private Map<Object, DataSource> dataSources = new ConcurrentHashMap<>();

/**
* Returns the data source associated with the specified database.
Expand All @@ -65,15 +64,11 @@ public DataSource getDataSource(Object database) {
return dataSources.computeIfAbsent(database, PostgresUtils::createDataSourceFromObject);
}

public DataMap<Long, Coordinate> getCoordinateMap(Path path) throws IOException {
if (Files.size(path) > 1 << 30) {
return getMemoryAlignedDataMap("coordinates", new LonLatDataType());
} else {
return getMonotonicDataMap("coordinates", new LonLatDataType());
}
public DataMap<Long, Coordinate> getCoordinateMap() throws IOException {
return getMemoryAlignedDataMap("coordinates", new LonLatDataType());
}

public DataMap<Long, List<Long>> getReferenceMap(Path path) throws IOException {
public DataMap<Long, List<Long>> getReferenceMap() throws IOException {
return getMonotonicDataMap("references", new LongListDataType());
}

Expand Down
4 changes: 2 additions & 2 deletions ...ps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,8 @@ public CreateGeocoderOpenStreetMap(Path file, Path indexDirectory) {
public void execute(WorkflowContext context) throws Exception {
var path = file.toAbsolutePath();

var coordinateMap = context.getCoordinateMap(path);
var referenceMap = context.getReferenceMap(path);
var coordinateMap = context.getCoordinateMap();
var referenceMap = context.getReferenceMap();

var directory = FSDirectory.open(indexDirectory);
var config = new IndexWriterConfig(GeocoderConstants.ANALYZER);
Expand Down
1 change: 1 addition & 0 deletions baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public void execute(WorkflowContext context) throws Exception {
logger.info("Execute SQL query: {}", query.replaceAll("\\s+", " "));
connection.createStatement().execute(query);
} catch (SQLException e) {
logger.error("Failed to execute query: {}", query.replaceAll("\\s+", " "));
throw new WorkflowException(e);
}
});
Expand Down
5 changes: 3 additions & 2 deletions baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,12 @@ public void execute(WorkflowContext context) throws Exception {
var wayRepository = new PostgresWayRepository(datasource);
var relationRepository = new PostgresRelationRepository(datasource);

var coordinateMap = context.getCoordinateMap(path);
var referenceMap = context.getReferenceMap(path);
var coordinateMap = context.getCoordinateMap();
var referenceMap = context.getReferenceMap();

var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap);
var referenceMapBuilder = new ReferenceMapBuilder(referenceMap);

var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap);
var reprojectGeometry = new EntityProjectionTransformer(4326, databaseSrid);
var prepareGeometries = coordinateMapBuilder
Expand Down
4 changes: 2 additions & 2 deletions baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ public void execute(WorkflowContext context) throws Exception {
relationRepository.create();
}

var coordinateMap = context.getCoordinateMap(path);
var referenceMap = context.getReferenceMap(path);
var coordinateMap = context.getCoordinateMap();
var referenceMap = context.getReferenceMap();

execute(
path,
Expand Down
Loading

0 comments on commit 2d88eea

Please sign in to comment.