From 79432a57cb4d904568760358823c9d6a3b1f4805 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 13 Nov 2023 14:46:21 +0100 Subject: [PATCH 01/21] Improve workflow and replace records by classes --- .../apache/baremaps/cli/workflow/Execute.java | 2 +- .../baremaps/workflow/WorkflowExecutor.java | 40 +++++----- .../tasks/CreateGeocoderOpenStreetMap.java | 33 +++++++- .../workflow/tasks/CreateGeonamesIndex.java | 31 +++++++- .../workflow/tasks/CreateIplocIndex.java | 41 +++++++++- .../workflow/tasks/DecompressBZip2.java | 32 +++++++- .../workflow/tasks/DecompressFile.java | 42 +++++++++- .../baremaps/workflow/tasks/DownloadUrl.java | 44 ++++++++++- .../workflow/tasks/ExecuteCommand.java | 18 ++++- .../baremaps/workflow/tasks/ExecuteSql.java | 42 +++++++++- .../workflow/tasks/ExecuteSqlScript.java | 35 ++++++++- .../workflow/tasks/ExportVectorTiles.java | 68 ++++++++++++++-- .../tasks/ImportDaylightFeatures.java | 35 ++++++++- .../tasks/ImportDaylightTranslations.java | 36 ++++++++- .../workflow/tasks/ImportGeoPackage.java | 52 ++++++++++++- .../baremaps/workflow/tasks/ImportOsmOsc.java | 66 ++++++++++++++-- .../baremaps/workflow/tasks/ImportOsmPbf.java | 77 +++++++++++++++++-- .../workflow/tasks/ImportShapefile.java | 52 ++++++++++++- .../baremaps/workflow/tasks/LogMessage.java | 20 ++++- .../baremaps/workflow/tasks/UngzipFile.java | 30 +++++++- .../baremaps/workflow/tasks/UnzipFile.java | 34 +++++++- .../workflow/tasks/UpdateOsmDatabase.java | 40 +++++++++- .../baremaps/workflow/WorkflowTest.java | 10 +-- 23 files changed, 805 insertions(+), 75 deletions(-) diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/workflow/Execute.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/workflow/Execute.java index b19c70aaa..6e67645e4 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/workflow/Execute.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/workflow/Execute.java @@ -50,7 +50,7 @@ public Integer call() throws Exception { var configReader = new ConfigReader(); var workflow = mapper.readValue(configReader.read(file.toAbsolutePath()), Workflow.class); try (var executor = new WorkflowExecutor(workflow)) { - executor.execute().get(); + executor.execute(); } logger.info("Finished executing the workflow {}", file); return 0; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java index 43de015c8..6d37e270e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java @@ -105,17 +105,21 @@ public WorkflowExecutor(Workflow workflow, ExecutorService executorService) { /** * Executes the workflow. */ - public CompletableFuture execute() { - // Create futures for each end step - var endSteps = graph.nodes().stream() - .filter(this::isEndStep) - .map(this::getFutureStep) - .toArray(CompletableFuture[]::new); - - // Create a future that logs the stepMeasures when all the futures are completed - var future = CompletableFuture.allOf(endSteps).thenRun(this::logStepMeasures); - - return future; + public void execute() { + try { + // Create futures for each end step + var endSteps = graph.nodes().stream() + .filter(this::isEndStep) + .map(this::getStep) + .toArray(CompletableFuture[]::new); + + // Wait for all the end steps to complete + CompletableFuture.allOf(endSteps).join(); + logStepMeasures(); + + } catch (Exception e) { + logger.error("Error while executing the workflow", e); + } } /** @@ -125,8 +129,8 @@ public CompletableFuture execute() { * @param step the step id * @return the future step */ - private CompletableFuture getFutureStep(String step) { - return futures.computeIfAbsent(step, this::createFutureStep); + private CompletableFuture getStep(String step) { + return futures.computeIfAbsent(step, this::createStep); } /** @@ -135,10 +139,10 @@ private CompletableFuture getFutureStep(String step) { * @param stepId the step id * @return the future step */ - private CompletableFuture createFutureStep(String stepId) { + private CompletableFuture createStep(String stepId) { // Initialize the future step with the previous future step // as it depends on its completion. - var future = getPreviousFutureStep(stepId); + var future = getPreviousStep(stepId); // Time the execution of the tasks var measures = new ArrayList(); @@ -182,7 +186,7 @@ private CompletableFuture createFutureStep(String stepId) { * @param stepId the step id * @return the future step */ - private CompletableFuture getPreviousFutureStep(String stepId) { + private CompletableFuture getPreviousStep(String stepId) { var predecessors = graph.predecessors(stepId).stream().toList(); // If the step has no predecessor, @@ -194,13 +198,13 @@ private CompletableFuture getPreviousFutureStep(String stepId) { // If the step has one predecessor, // return the future step associated to it. if (predecessors.size() == 1) { - return getFutureStep(predecessors.get(0)); + return getStep(predecessors.get(0)); } // If the step has multiple predecessors, // return a future step that completes when all the predecessors complete. var futurePredecessors = predecessors.stream() - .map(this::getFutureStep) + .map(this::getStep) .toArray(CompletableFuture[]::new); return CompletableFuture.allOf(futurePredecessors); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java index 055ed5cb1..41f72d657 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java @@ -48,15 +48,40 @@ /** * Experimental feature. - * + * * @see org.apache.baremaps.geocoderosm */ -public record CreateGeocoderOpenStreetMap(Path file, Path indexDirectory) - implements - Task { +public class CreateGeocoderOpenStreetMap implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateGeocoderOpenStreetMap.class); + private Path file; + + private Path indexDirectory; + + public CreateGeocoderOpenStreetMap() {} + + public CreateGeocoderOpenStreetMap(Path file, Path indexDirectory) { + this.file = file; + this.indexDirectory = indexDirectory; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Path getIndexDirectory() { + return indexDirectory; + } + + public void setIndexDirectory(Path indexDirectory) { + this.indexDirectory = indexDirectory; + } + @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java index 79ed11580..3c4d974d0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java @@ -35,10 +35,39 @@ /** * A task that creates a geonames index. */ -public record CreateGeonamesIndex(Path dataFile, Path indexDirectory) implements Task { +public class CreateGeonamesIndex implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateGeonamesIndex.class); + private Path dataFile; + + private Path indexDirectory; + + public CreateGeonamesIndex() { + + } + + public CreateGeonamesIndex(Path dataFile, Path indexDirectory) { + this.dataFile = dataFile; + this.indexDirectory = indexDirectory; + } + + public Path getDataFile() { + return dataFile; + } + + public void setDataFile(Path dataFile) { + this.dataFile = dataFile; + } + + public Path getIndexDirectory() { + return indexDirectory; + } + + public void setIndexDirectory(Path indexDirectory) { + this.indexDirectory = indexDirectory; + } + @Override public void execute(WorkflowContext context) throws Exception { var directory = MMapDirectory.open(indexDirectory); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java index bfbd5d9bd..9ae4fe6f0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java @@ -36,13 +36,46 @@ import org.sqlite.SQLiteConfig; import org.sqlite.SQLiteDataSource; -public record CreateIplocIndex( - Path geonamesIndexPath, - List nicPaths, - Path targetIplocIndexPath) implements Task { +public class CreateIplocIndex implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateIplocIndex.class); + private Path geonamesIndexPath; + private List nicPaths; + private Path targetIplocIndexPath; + + public CreateIplocIndex() {} + + public CreateIplocIndex(Path geonamesIndexPath, List nicPaths, Path targetIplocIndexPath) { + this.geonamesIndexPath = geonamesIndexPath; + this.nicPaths = nicPaths; + this.targetIplocIndexPath = targetIplocIndexPath; + } + + public Path getGeonamesIndexPath() { + return geonamesIndexPath; + } + + public void setGeonamesIndexPath(Path geonamesIndexPath) { + this.geonamesIndexPath = geonamesIndexPath; + } + + public List getNicPaths() { + return nicPaths; + } + + public void setNicPaths(List nicPaths) { + this.nicPaths = nicPaths; + } + + public Path getTargetIplocIndexPath() { + return targetIplocIndexPath; + } + + public void setTargetIplocIndexPath(Path targetIplocIndexPath) { + this.targetIplocIndexPath = targetIplocIndexPath; + } + @Override public void execute(WorkflowContext context) throws Exception { try (var directory = MMapDirectory.open(geonamesIndexPath); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java index 9ed966304..9217f0772 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java @@ -25,8 +25,38 @@ import org.apache.baremaps.workflow.WorkflowContext; import org.apache.baremaps.workflow.WorkflowException; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public record DecompressBZip2(Path source, Path target) implements Task { +public class DecompressBZip2 implements Task { + + private static final Logger logger = LoggerFactory.getLogger(DecompressBZip2.class); + + private Path source; + private Path target; + + public DecompressBZip2() {} + + public DecompressBZip2(Path source, Path target) { + this.source = source; + this.target = target; + } + + public Path getSource() { + return source; + } + + public void setSource(Path source) { + this.source = source; + } + + public Path getTarget() { + return target; + } + + public void setTarget(Path target) { + this.target = target; + } @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index 4aa1bb388..2551389ab 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -32,7 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record DecompressFile(Path source, Path target, Compression compression) implements Task { +public class DecompressFile implements Task { + + private static final Logger logger = LoggerFactory.getLogger(DecompressFile.class); public enum Compression { zip, @@ -42,7 +44,43 @@ public enum Compression { bzip2; } - private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class); + private Path source; + + private Path target; + + private Compression compression; + + public DecompressFile() {} + + public DecompressFile(Path source, Path target, Compression compression) { + this.source = source; + this.target = target; + this.compression = compression; + } + + public Path getSource() { + return source; + } + + public void setSource(Path source) { + this.source = source; + } + + public Path getTarget() { + return target; + } + + public void setTarget(Path target) { + this.target = target; + } + + public Compression getCompression() { + return compression; + } + + public void setCompression(Compression compression) { + this.compression = compression; + } @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java index c6362ea59..1955fbfbd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java @@ -29,17 +29,57 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record DownloadUrl(String url, Path path, boolean replaceExisting) implements Task { +public class DownloadUrl implements Task { + + private static final Logger logger = LoggerFactory.getLogger(DownloadUrl.class); private static final String PROTOCOL_FTP = "ftp"; + private static final String PROTOCOL_HTTP = "http"; + private static final String PROTOCOL_HTTPS = "https"; + private String url; + + private Path path; + + private Boolean replaceExisting; + + public DownloadUrl() {} + public DownloadUrl(String url, Path path) { this(url, path, false); } - private static final Logger logger = LoggerFactory.getLogger(DownloadUrl.class); + public DownloadUrl(String url, Path path, boolean replaceExisting) { + this.url = url; + this.path = path; + this.replaceExisting = replaceExisting; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Path getPath() { + return path; + } + + public void setPath(Path path) { + this.path = path; + } + + public Boolean getReplaceExisting() { + return replaceExisting; + } + + public void setReplaceExisting(Boolean replaceExisting) { + this.replaceExisting = replaceExisting; + } @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java index 4cb4218ba..4c21502c3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java @@ -22,10 +22,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ExecuteCommand(String command) implements Task { +public class ExecuteCommand implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteCommand.class); + private String command; + + public ExecuteCommand() {} + + public ExecuteCommand(String command) { + this.command = command; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + @Override public void execute(WorkflowContext context) throws Exception { new ProcessBuilder().command("/bin/sh", "-c", command).start().waitFor(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java index 25058c8d0..12c35844f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java @@ -29,10 +29,50 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ExecuteSql(Object database, Path file, boolean parallel) implements Task { +public class ExecuteSql implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteSql.class); + private Object database; + + private Path file; + + private boolean parallel; + + public ExecuteSql() { + + } + + public ExecuteSql(Object database, Path file, boolean parallel) { + this.database = database; + this.file = file; + this.parallel = parallel; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public boolean isParallel() { + return parallel; + } + + public void setParallel(boolean parallel) { + this.parallel = parallel; + } + @Override public void execute(WorkflowContext context) throws Exception { var script = clean(Files.readString(file)); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java index 50341a47a..432d9ef9e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java @@ -23,8 +23,41 @@ import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.apache.baremaps.workflow.WorkflowException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public record ExecuteSqlScript(Object database, Path file) implements Task { +public class ExecuteSqlScript implements Task { + + private static final Logger logger = LoggerFactory.getLogger(ExecuteSqlScript.class); + + private Object database; + + private Path file; + + public ExecuteSqlScript() { + + } + + public ExecuteSqlScript(Object database, Path file) { + this.database = database; + this.file = file; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index 93a27a2c1..0be49792b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -44,12 +44,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ExportVectorTiles( - Path tileset, - Path repository, - int batchArraySize, - int batchArrayIndex, - Format format) implements Task { +public class ExportVectorTiles implements Task { + + private static final Logger logger = LoggerFactory.getLogger(ExportVectorTiles.class); public enum Format { file, @@ -57,7 +54,64 @@ public enum Format { pmtiles } - private static final Logger logger = LoggerFactory.getLogger(ExportVectorTiles.class); + private Path tileset; + private Path repository; + private Integer batchArraySize; + private Integer batchArrayIndex; + private Format format; + + public ExportVectorTiles() { + + } + + public ExportVectorTiles(Path tileset, Path repository, Integer batchArraySize, + Integer batchArrayIndex, Format format) { + this.tileset = tileset; + this.repository = repository; + this.batchArraySize = batchArraySize; + this.batchArrayIndex = batchArrayIndex; + this.format = format; + } + + public Path getTileset() { + return tileset; + } + + public void setTileset(Path tileset) { + this.tileset = tileset; + } + + public Path getRepository() { + return repository; + } + + public void setRepository(Path repository) { + this.repository = repository; + } + + public Integer getBatchArraySize() { + return batchArraySize; + } + + public void setBatchArraySize(Integer batchArraySize) { + this.batchArraySize = batchArraySize; + } + + public Integer getBatchArrayIndex() { + return batchArrayIndex; + } + + public void setBatchArrayIndex(Integer batchArrayIndex) { + this.batchArrayIndex = batchArrayIndex; + } + + public Format getFormat() { + return format; + } + + public void setFormat(Format format) { + this.format = format; + } @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java index 8422d8fc2..495d15c3e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java @@ -28,8 +28,12 @@ import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public record ImportDaylightFeatures(Path file, Object database) implements Task { +public class ImportDaylightFeatures implements Task { + + private static final Logger logger = LoggerFactory.getLogger(ImportDaylightFeatures.class); record Feature( @JsonProperty("osm_type") String type, @@ -39,6 +43,35 @@ record Feature( @JsonProperty("category") String category) { } + private Path file; + + private Object database; + + public ImportDaylightFeatures() { + + } + + public ImportDaylightFeatures(Path file, Object database) { + this.file = file; + this.database = database; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java index c5e9d86a3..63b4a013f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java @@ -26,8 +26,12 @@ import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public record ImportDaylightTranslations(Path file, Object database) implements Task { +public class ImportDaylightTranslations implements Task { + + private static final Logger logger = LoggerFactory.getLogger(ImportDaylightTranslations.class); record Group(String type, Long id, String name) { @@ -50,6 +54,36 @@ public static Line parse(String line) { } } + + private Path file; + + private Object database; + + public ImportDaylightTranslations() { + + } + + public ImportDaylightTranslations(Path file, Object database) { + this.file = file; + this.database = database; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index a8a6b52c6..477133835 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -29,12 +29,58 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer targetSRID) - implements - Task { +public class ImportGeoPackage implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportGeoPackage.class); + private Path file; + private Object database; + private Integer sourceSRID; + private Integer targetSRID; + + public ImportGeoPackage() { + + } + + public ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer targetSRID) { + this.file = file; + this.database = database; + this.sourceSRID = sourceSRID; + this.targetSRID = targetSRID; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Integer getSourceSRID() { + return sourceSRID; + } + + public void setSourceSRID(Integer sourceSRID) { + this.sourceSRID = sourceSRID; + } + + public Integer getTargetSRID() { + return targetSRID; + } + + public void setTargetSRID(Integer targetSRID) { + this.targetSRID = targetSRID; + } + @Override public void execute(WorkflowContext context) throws Exception { var path = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index b16fff32e..69baa1680 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -41,15 +41,69 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ImportOsmOsc( - Path file, - Path cache, - Object database, - Integer srid, - Compression compression) implements Task { +public class ImportOsmOsc implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmOsc.class); + private Path file; + private Path cache; + private Object database; + private Integer srid; + private Compression compression; + + public ImportOsmOsc() { + + } + + public ImportOsmOsc(Path file, Path cache, Object database, Integer srid, + Compression compression) { + this.file = file; + this.cache = cache; + this.database = database; + this.srid = srid; + this.compression = compression; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Path getCache() { + return cache; + } + + public void setCache(Path cache) { + this.cache = cache; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Integer getSrid() { + return srid; + } + + public void setSrid(Integer srid) { + this.srid = srid; + } + + public Compression getCompression() { + return compression; + } + + public void setCompression(Compression compression) { + this.compression = compression; + } + @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index 499d244f4..4cf69326d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -46,16 +46,79 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ImportOsmPbf( - Path file, - Path cache, - Boolean cleanCache, - Object database, - Integer databaseSrid, - Boolean replaceExisting) implements Task { +public class ImportOsmPbf implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class); + private Path file; + private Path cache; + private Boolean cleanCache; + private Object database; + private Integer databaseSrid; + private Boolean replaceExisting; + + public ImportOsmPbf() { + + } + + public ImportOsmPbf(Path file, Path cache, Boolean cleanCache, Object database, + Integer databaseSrid, Boolean replaceExisting) { + this.file = file; + this.cache = cache; + this.cleanCache = cleanCache; + this.database = database; + this.databaseSrid = databaseSrid; + this.replaceExisting = replaceExisting; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Path getCache() { + return cache; + } + + public void setCache(Path cache) { + this.cache = cache; + } + + public Boolean getCleanCache() { + return cleanCache; + } + + public void setCleanCache(Boolean cleanCache) { + this.cleanCache = cleanCache; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Integer getDatabaseSrid() { + return databaseSrid; + } + + public void setDatabaseSrid(Integer databaseSrid) { + this.databaseSrid = databaseSrid; + } + + public Boolean getReplaceExisting() { + return replaceExisting; + } + + public void setReplaceExisting(Boolean replaceExisting) { + this.replaceExisting = replaceExisting; + } + @Override public void execute(WorkflowContext context) throws Exception { var dataSource = context.getDataSource(database); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index b37f56b7a..693f9c4c2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -29,12 +29,58 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record ImportShapefile(Path file, Object database, Integer sourceSRID, Integer targetSRID) - implements - Task { +public class ImportShapefile implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportShapefile.class); + private Path file; + private Object database; + private Integer sourceSRID; + private Integer targetSRID; + + public ImportShapefile() { + + } + + public ImportShapefile(Path file, Object database, Integer sourceSRID, Integer targetSRID) { + this.file = file; + this.database = database; + this.sourceSRID = sourceSRID; + this.targetSRID = targetSRID; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Integer getSourceSRID() { + return sourceSRID; + } + + public void setSourceSRID(Integer sourceSRID) { + this.sourceSRID = sourceSRID; + } + + public Integer getTargetSRID() { + return targetSRID; + } + + public void setTargetSRID(Integer targetSRID) { + this.targetSRID = targetSRID; + } + @Override public void execute(WorkflowContext context) throws Exception { var path = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java index a3f6b8520..d548dbcdd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java @@ -22,10 +22,28 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record LogMessage(String message) implements Task { +public class LogMessage implements Task { private static final Logger logger = LoggerFactory.getLogger(LogMessage.class); + private String message; + + public LogMessage() { + + } + + public LogMessage(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + @Override public void execute(WorkflowContext context) throws Exception { logger.info(message); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java index d55d76620..40935dcf0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java @@ -28,10 +28,38 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record UngzipFile(Path file, Path directory) implements Task { +public class UngzipFile implements Task { private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class); + private Path file; + private Path directory; + + public UngzipFile() { + + } + + public UngzipFile(Path file, Path directory) { + this.file = file; + this.directory = directory; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Path getDirectory() { + return directory; + } + + public void setDirectory(Path directory) { + this.directory = directory; + } + @Override public void execute(WorkflowContext context) throws Exception { var filePath = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java index dc89d9c54..4e45aafe1 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java @@ -22,8 +22,40 @@ import java.util.zip.ZipFile; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public record UnzipFile(Path file, Path directory) implements Task { +public class UnzipFile implements Task { + + private static final Logger logger = LoggerFactory.getLogger(UnzipFile.class); + + private Path file; + private Path directory; + + public UnzipFile() { + + } + + public UnzipFile(Path file, Path directory) { + this.file = file; + this.directory = directory; + } + + public Path getFile() { + return file; + } + + public void setFile(Path file) { + this.file = file; + } + + public Path getDirectory() { + return directory; + } + + public void setDirectory(Path directory) { + this.directory = directory; + } @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index 1a2005d2e..43b156144 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -46,15 +46,49 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public record UpdateOsmDatabase(Object database, Integer databaseSrid, - String replicationUrl) implements Task { +public class UpdateOsmDatabase implements Task { private static final Logger logger = LoggerFactory.getLogger(UpdateOsmDatabase.class); + private Object database; + + private Integer databaseSrid; + + private String replicationUrl; + + public UpdateOsmDatabase() { + + } + public UpdateOsmDatabase(Object database, Integer databaseSrid) { - this(database, databaseSrid, null); + this.database = database; + this.databaseSrid = databaseSrid; + } + + public Object getDatabase() { + return database; + } + + public void setDatabase(Object database) { + this.database = database; + } + + public Integer getDatabaseSrid() { + return databaseSrid; + } + + public void setDatabaseSrid(Integer databaseSrid) { + this.databaseSrid = databaseSrid; } + public String getReplicationUrl() { + return replicationUrl; + } + + public void setReplicationUrl(String replicationUrl) { + this.replicationUrl = replicationUrl; + } + @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index 4fb5b32de..d52a27b65 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -43,7 +43,7 @@ void naturalearthGeoPackage() { new ImportGeoPackage(Paths.get("natural_earth_vector/packages/natural_earth_vector.gpkg"), jdbcUrl(), 4326, 3857))))); - new WorkflowExecutor(workflow).execute().join(); + new WorkflowExecutor(workflow).execute(); } @Test @@ -58,7 +58,7 @@ void coastlineShapefile() { new ImportShapefile(Paths.get("coastlines-split-4326/coastlines-split-4326/lines.shp"), jdbcUrl(), 4326, 3857))))); - new WorkflowExecutor(workflow).execute().join(); + new WorkflowExecutor(workflow).execute(); } @Test @@ -76,7 +76,7 @@ void simplifiedWaterPolygonsShapefile() { "simplified-water-polygons-split-3857/simplified-water-polygons-split-3857/simplified_water_polygons.shp"), "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", 3857, 3857))))); - new WorkflowExecutor(workflow).execute().join(); + new WorkflowExecutor(workflow).execute(); } @Test @@ -86,7 +86,7 @@ void workflow() { new DownloadUrl("https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", Paths.get("downloads/import_db.gpkg")), new ImportShapefile(Paths.get("downloads/import_db.gpkg"), jdbcUrl(), 4326, 3857))))); - new WorkflowExecutor(workflow).execute().join(); + new WorkflowExecutor(workflow).execute(); } @Test @@ -122,6 +122,6 @@ void execute() { Paths.get( "archives/simplified-water-polygons-split-3857/simplified_water_polygons.shp"), jdbcUrl(), 3857, 3857))))); - new WorkflowExecutor(workflow).execute().join(); + new WorkflowExecutor(workflow).execute(); } } From 8563c890e10c10f863ba408e14c0f22aa761e168 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 13 Nov 2023 14:57:42 +0100 Subject: [PATCH 02/21] Move the type name in the sub classes --- .../org/apache/baremaps/workflow/Task.java | 22 ------------------- .../tasks/CreateGeocoderOpenStreetMap.java | 2 ++ .../workflow/tasks/CreateGeonamesIndex.java | 2 ++ .../workflow/tasks/CreateIplocIndex.java | 2 ++ .../workflow/tasks/DecompressBZip2.java | 2 ++ .../workflow/tasks/DecompressFile.java | 2 ++ .../baremaps/workflow/tasks/DownloadUrl.java | 2 ++ .../workflow/tasks/ExecuteCommand.java | 2 ++ .../baremaps/workflow/tasks/ExecuteSql.java | 2 ++ .../workflow/tasks/ExecuteSqlScript.java | 2 ++ .../workflow/tasks/ExportVectorTiles.java | 2 ++ .../tasks/ImportDaylightFeatures.java | 2 ++ .../tasks/ImportDaylightTranslations.java | 2 ++ .../workflow/tasks/ImportGeoPackage.java | 2 ++ .../baremaps/workflow/tasks/ImportOsmOsc.java | 2 ++ .../baremaps/workflow/tasks/ImportOsmPbf.java | 2 ++ .../workflow/tasks/ImportShapefile.java | 2 ++ .../baremaps/workflow/tasks/LogMessage.java | 2 ++ .../baremaps/workflow/tasks/UngzipFile.java | 2 ++ .../baremaps/workflow/tasks/UnzipFile.java | 2 ++ .../workflow/tasks/UpdateOsmDatabase.java | 2 ++ 21 files changed, 40 insertions(+), 22 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 06fdad9e8..68e364436 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -32,27 +31,6 @@ @JsonSerialize @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) @JsonTypeInfo(use = Id.NAME, property = "type") -@JsonSubTypes({@JsonSubTypes.Type(value = DownloadUrl.class, name = "DownloadUrl"), - @JsonSubTypes.Type(value = ExecuteCommand.class, name = "ExecuteCommand"), - @JsonSubTypes.Type(value = ExecuteSql.class, name = "ExecuteSql"), - @JsonSubTypes.Type(value = ExecuteSqlScript.class, name = "ExecuteSqlScript"), - @JsonSubTypes.Type(value = ExportVectorTiles.class, name = "ExportVectorTiles"), - @JsonSubTypes.Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"), - @JsonSubTypes.Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"), - @JsonSubTypes.Type(value = ImportOsmOsc.class, name = "ImportOsmOsc"), - @JsonSubTypes.Type(value = ImportShapefile.class, name = "ImportShapefile"), - @JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"), - @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), - @JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"), - @JsonSubTypes.Type(value = DecompressBZip2.class, name = "DecompressBZip2"), - @JsonSubTypes.Type(value = DecompressFile.class, name = "DecompressFile"), - @JsonSubTypes.Type(value = UpdateOsmDatabase.class, name = "UpdateOsmDatabase"), - @JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"), - @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex"), - @JsonSubTypes.Type(value = ImportDaylightTranslations.class, - name = "ImportDaylightTranslations"), - @JsonSubTypes.Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures") -}) public interface Task { /** diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java index 41f72d657..81691994f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -51,6 +52,7 @@ * * @see org.apache.baremaps.geocoderosm */ +@JsonTypeName("CreateGeocoderOpenStreetMap") public class CreateGeocoderOpenStreetMap implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateGeocoderOpenStreetMap.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java index 3c4d974d0..884c39abc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -35,6 +36,7 @@ /** * A task that creates a geonames index. */ +@JsonTypeName("CreateGeonamesIndex") public class CreateGeonamesIndex implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateGeonamesIndex.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java index 9ae4fe6f0..a3da337a3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -36,6 +37,7 @@ import org.sqlite.SQLiteConfig; import org.sqlite.SQLiteDataSource; +@JsonTypeName("CreateIplocIndex") public class CreateIplocIndex implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateIplocIndex.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java index 9217f0772..ef17fe00a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -28,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("DecompressBZip2") public class DecompressBZip2 implements Task { private static final Logger logger = LoggerFactory.getLogger(DecompressBZip2.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index 2551389ab..4d9910779 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.*; import java.nio.file.Files; import java.nio.file.Path; @@ -32,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("DecompressFile") public class DecompressFile implements Task { private static final Logger logger = LoggerFactory.getLogger(DecompressFile.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java index 1955fbfbd..3b7bd5b68 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; @@ -29,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("DownloadUrl") public class DownloadUrl implements Task { private static final Logger logger = LoggerFactory.getLogger(DownloadUrl.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java index 4c21502c3..6ccd6ddbc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java @@ -17,11 +17,13 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ExecuteCommand") public class ExecuteCommand implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteCommand.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java index 12c35844f..f405250c8 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; @@ -29,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ExecuteSql") public class ExecuteSql implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteSql.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java index 432d9ef9e..e8b95e821 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; @@ -26,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ExecuteSqlScript") public class ExecuteSqlScript implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteSqlScript.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index 0be49792b..a0cb6b53d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -19,6 +19,7 @@ import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -44,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ExportVectorTiles") public class ExportVectorTiles implements Task { private static final Logger logger = LoggerFactory.getLogger(ExportVectorTiles.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java index 495d15c3e..ba00a2552 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java @@ -18,6 +18,7 @@ package org.apache.baremaps.workflow.tasks; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Path; import java.util.HashMap; @@ -31,6 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ImportDaylightFeatures") public class ImportDaylightFeatures implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportDaylightFeatures.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java index 63b4a013f..dc6dc6193 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; @@ -29,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ImportDaylightTranslations") public class ImportDaylightTranslations implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportDaylightTranslations.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index 477133835..882f85486 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Path; import org.apache.baremaps.database.schema.DataTableAdapter; import org.apache.baremaps.database.schema.DataTableGeometryTransformer; @@ -29,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ImportGeoPackage") public class ImportGeoPackage implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportGeoPackage.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index 69baa1680..da65e5238 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -19,6 +19,7 @@ import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -41,6 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ImportOsmOsc") public class ImportOsmOsc implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmOsc.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index 4cf69326d..8a14d78cd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -46,6 +47,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ImportOsmPbf") public class ImportOsmPbf implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index 693f9c4c2..5a68ce033 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Path; import org.apache.baremaps.database.schema.DataTableAdapter; import org.apache.baremaps.database.schema.DataTableGeometryTransformer; @@ -29,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("ImportShapefile") public class ImportShapefile implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportShapefile.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java index d548dbcdd..f57387bd3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java @@ -17,11 +17,13 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("LogMessage") public class LogMessage implements Task { private static final Logger logger = LoggerFactory.getLogger(LogMessage.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java index 40935dcf0..cd9d3f8c2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -28,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("UngzipFile") public class UngzipFile implements Task { private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java index 4e45aafe1..4c95f599e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java @@ -17,6 +17,7 @@ package org.apache.baremaps.workflow.tasks; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.*; import java.nio.file.*; import java.util.zip.ZipFile; @@ -25,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("UnzipFile") public class UnzipFile implements Task { private static final Logger logger = LoggerFactory.getLogger(UnzipFile.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index 43b156144..b11e84498 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -19,6 +19,7 @@ import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; +import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.util.List; import java.util.zip.GZIPInputStream; @@ -46,6 +47,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@JsonTypeName("UpdateOsmDatabase") public class UpdateOsmDatabase implements Task { private static final Logger logger = LoggerFactory.getLogger(UpdateOsmDatabase.class); From 339f2fbad1efb37c792f3caf5b8c1acde61dedb7 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 13 Nov 2023 16:33:29 +0100 Subject: [PATCH 03/21] Add javadoc --- .../workflow/tasks/DecompressBZip2.java | 78 ------------ .../workflow/tasks/DecompressFile.java | 117 +++++++++++++++--- .../baremaps/workflow/tasks/DownloadUrl.java | 17 ++- .../workflow/tasks/ExecuteCommand.java | 25 ++++ .../baremaps/workflow/tasks/ExecuteSql.java | 50 +++++++- .../workflow/tasks/ExecuteSqlScript.java | 35 ++++++ .../workflow/tasks/ExportVectorTiles.java | 1 + .../tasks/ImportDaylightFeatures.java | 44 +++++-- .../tasks/ImportDaylightTranslations.java | 35 ++++++ .../workflow/tasks/ImportGeoPackage.java | 57 +++++++++ .../baremaps/workflow/tasks/ImportOsmOsc.java | 68 ++++++++++ .../baremaps/workflow/tasks/ImportOsmPbf.java | 92 ++++++++++++++ .../workflow/tasks/ImportShapefile.java | 57 +++++++++ .../baremaps/workflow/tasks/LogMessage.java | 24 ++++ .../baremaps/workflow/tasks/UngzipFile.java | 35 ++++++ .../baremaps/workflow/tasks/UnzipFile.java | 35 ++++++ .../workflow/tasks/UpdateOsmDatabase.java | 66 ++++++++++ .../baremaps/workflow/ObjectMapperTest.java | 2 +- .../baremaps/workflow/WorkflowTest.java | 15 +-- .../workflow/tasks/DownloadUrlTest.java | 6 +- 20 files changed, 738 insertions(+), 121 deletions(-) delete mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java deleted file mode 100644 index ef17fe00a..000000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.workflow.tasks; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import java.io.BufferedInputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import org.apache.baremaps.workflow.Task; -import org.apache.baremaps.workflow.WorkflowContext; -import org.apache.baremaps.workflow.WorkflowException; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@JsonTypeName("DecompressBZip2") -public class DecompressBZip2 implements Task { - - private static final Logger logger = LoggerFactory.getLogger(DecompressBZip2.class); - - private Path source; - private Path target; - - public DecompressBZip2() {} - - public DecompressBZip2(Path source, Path target) { - this.source = source; - this.target = target; - } - - public Path getSource() { - return source; - } - - public void setSource(Path source) { - this.source = source; - } - - public Path getTarget() { - return target; - } - - public void setTarget(Path target) { - this.target = target; - } - - @Override - public void execute(WorkflowContext context) throws Exception { - var sourcePath = source.toAbsolutePath(); - try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(sourcePath)); - var compressedInputStream = new BZip2CompressorInputStream(bufferedInputStream)) { - var targetPath = target.toAbsolutePath(); - if (!Files.exists(targetPath)) { - Files.createDirectories(targetPath.getParent()); - Files.createFile(targetPath); - } - Files.copy(compressedInputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception e) { - throw new WorkflowException(e); - } - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index 4d9910779..37b0ac0e2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -33,11 +33,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Decompresses a file based on a given compression format. The supported formats are zip, targz, + * tarbz2, gzip and bzip2. + */ @JsonTypeName("DecompressFile") public class DecompressFile implements Task { private static final Logger logger = LoggerFactory.getLogger(DecompressFile.class); + /** + * The compression format. + */ public enum Compression { zip, targz, @@ -47,43 +54,84 @@ public enum Compression { } private Path source; - private Path target; - private Compression compression; + /** + * Constructs a {@code DecompressFile}. + */ public DecompressFile() {} + /** + * Constructs a {@code DecompressFile}. + * + * @param source the source file + * @param target the target file + * @param compression the compression format (zip, targz, tarbz2, gzip or bzip2) + */ public DecompressFile(Path source, Path target, Compression compression) { this.source = source; this.target = target; this.compression = compression; } + /** + * Returns the source file. + * + * @return the source file + */ public Path getSource() { return source; } + /** + * Sets the source file. + * + * @param source the source file + */ public void setSource(Path source) { this.source = source; } + /** + * Returns the target file. + * + * @return the target file + */ public Path getTarget() { return target; } + /** + * Sets the target file. + * + * @param target the target file + */ public void setTarget(Path target) { this.target = target; } + /** + * Returns the compression format. + * + * @return the compression format + */ public Compression getCompression() { return compression; } + /** + * Sets the compression format. + * + * @param compression the compression format + */ public void setCompression(Compression compression) { this.compression = compression; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var sourcePath = source.toAbsolutePath(); @@ -97,26 +145,47 @@ public void execute(WorkflowContext context) throws Exception { } } - public static void decompressBzip2(Path sourcePath, Path targetPath) throws IOException { - try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(sourcePath)); + /** + * Decompresses a bzip2 file. + * + * @param source the source file + * @param target the target file + * @throws IOException if an I/O error occurs + */ + protected static void decompressBzip2(Path source, Path target) throws IOException { + try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(source)); var bzip2InputStream = new BZip2CompressorInputStream(bufferedInputStream)) { - Files.copy(bzip2InputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(bzip2InputStream, target, StandardCopyOption.REPLACE_EXISTING); } } - public static void decompressGzip(Path sourcePath, Path targetPath) throws IOException { - try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { - Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + /** + * Decompresses a gzip file. + * + * @param source the source file + * @param target the target file + * @throws IOException if an I/O error occurs + */ + protected static void decompressGzip(Path source, Path target) throws IOException { + try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(source)))) { + Files.copy(zis, target, StandardCopyOption.REPLACE_EXISTING); } } - public static void decompressTarGz(Path sourcePath, Path targetPath) throws IOException { - try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(sourcePath)); + /** + * Decompresses a tar.gz file. + * + * @param source the source file + * @param target the target file + * @throws IOException if an I/O error occurs + */ + protected static void decompressTarGz(Path source, Path target) throws IOException { + try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(source)); var gzipInputStream = new GZIPInputStream(bufferedInputStream); var tarInputStream = new TarArchiveInputStream(gzipInputStream)) { TarArchiveEntry entry; while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { - var path = targetPath.resolve(entry.getName()); + var path = target.resolve(entry.getName()); if (entry.isDirectory()) { Files.createDirectories(path); } else { @@ -134,13 +203,20 @@ public static void decompressTarGz(Path sourcePath, Path targetPath) throws IOEx } } - public static void decompressTarBz2(Path sourcePath, Path targetPath) throws IOException { - try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(sourcePath)); + /** + * Decompresses a tar.bz2 file. + * + * @param source the source file + * @param target the target file + * @throws IOException if an I/O error occurs + */ + protected static void decompressTarBz2(Path source, Path target) throws IOException { + try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(source)); var bzip2InputStream = new BZip2CompressorInputStream(bufferedInputStream); var tarInputStream = new TarArchiveInputStream(bzip2InputStream)) { TarArchiveEntry entry; while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { - var path = targetPath.resolve(entry.getName()); + var path = target.resolve(entry.getName()); if (entry.isDirectory()) { Files.createDirectories(path); } else { @@ -158,12 +234,19 @@ public static void decompressTarBz2(Path sourcePath, Path targetPath) throws IOE } } - public static void decompressZip(Path sourcePath, Path targetPath) throws IOException { - try (var zipFile = new ZipFile(sourcePath.toFile())) { + /** + * Decompresses a zip file. + * + * @param source the source file + * @param target the target file + * @throws IOException if an I/O error occurs + */ + protected static void decompressZip(Path source, Path target) throws IOException { + try (var zipFile = new ZipFile(source.toFile())) { var entries = zipFile.entries(); while (entries.hasMoreElements()) { var entry = entries.nextElement(); - var path = targetPath.resolve(entry.getName()); + var path = target.resolve(entry.getName()); Files.createDirectories(path.getParent()); Files.write(path, new byte[] {}, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java index 3b7bd5b68..767de3477 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Downloads a file from a URL. + */ @JsonTypeName("DownloadUrl") public class DownloadUrl implements Task { @@ -47,12 +50,18 @@ public class DownloadUrl implements Task { private Boolean replaceExisting; + /** + * Constructs an {@code DownloadUrl}. + */ public DownloadUrl() {} - public DownloadUrl(String url, Path path) { - this(url, path, false); - } - + /** + * Constructs an {@code DownloadUrl}. + * + * @param url the url + * @param path the path + * @param replaceExisting whether to replace existing files + */ public DownloadUrl(String url, Path path, boolean replaceExisting) { this.url = url; this.path = path; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java index 6ccd6ddbc..6d273a091 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java @@ -23,6 +23,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Execute a bash command. + */ @JsonTypeName("ExecuteCommand") public class ExecuteCommand implements Task { @@ -30,22 +33,44 @@ public class ExecuteCommand implements Task { private String command; + /** + * Constructs an {@code ExecuteCommand}. + */ public ExecuteCommand() {} + /** + * Constructs an {@code ExecuteCommand}. + * + * @param command the bash command + */ public ExecuteCommand(String command) { this.command = command; } + /** + * Returns the bash command. + * + * @return the bash command + */ public String getCommand() { return command; } + /** + * Sets the bash command. + * + * @param command the bash command + */ public void setCommand(String command) { this.command = command; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { new ProcessBuilder().command("/bin/sh", "-c", command).start().waitFor(); } + } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java index f405250c8..96bb151e6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Execute a SQL query (single statement). + */ @JsonTypeName("ExecuteSql") public class ExecuteSql implements Task { @@ -39,42 +42,85 @@ public class ExecuteSql implements Task { private Path file; - private boolean parallel; + private Boolean parallel; + /** + * Constructs an {@code ExecuteSql}. + */ public ExecuteSql() { } - public ExecuteSql(Object database, Path file, boolean parallel) { + /** + * Constructs an {@code ExecuteSql}. + * + * @param database the database + * @param file the SQL file + * @param parallel whether to execute the queries in parallel + */ + public ExecuteSql(Object database, Path file, Boolean parallel) { this.database = database; this.file = file; this.parallel = parallel; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the SQL file. + * + * @return the SQL file + */ public Path getFile() { return file; } + /** + * Sets the SQL file. + * + * @param file the SQL file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns whether to execute the queries in parallel. + * + * @return whether to execute the queries in parallel + */ public boolean isParallel() { return parallel; } + /** + * Sets whether to execute the queries in parallel. + * + * @param parallel whether to execute the queries in parallel + */ public void setParallel(boolean parallel) { this.parallel = parallel; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var script = clean(Files.readString(file)); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java index e8b95e821..af8dff0df 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java @@ -27,6 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Execute a SQL script (multiple statements). + */ @JsonTypeName("ExecuteSqlScript") public class ExecuteSqlScript implements Task { @@ -36,31 +39,63 @@ public class ExecuteSqlScript implements Task { private Path file; + /** + * Constructs an {@code ExecuteSqlScript}. + */ public ExecuteSqlScript() { } + /** + * Constructs an {@code ExecuteSqlScript}. + * + * @param database the database + * @param file the SQL file + */ public ExecuteSqlScript(Object database, Path file) { this.database = database; this.file = file; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the SQL file. + * + * @return the SQL file + */ public Path getFile() { return file; } + /** + * Sets the SQL file. + * + * @param file the SQL file + */ public void setFile(Path file) { this.file = file; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var script = Files.readString(file); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index a0cb6b53d..2e21d0a0f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + @JsonTypeName("ExportVectorTiles") public class ExportVectorTiles implements Task { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java index ba00a2552..d116e339b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java @@ -32,6 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Import daylight features. + */ @JsonTypeName("ImportDaylightFeatures") public class ImportDaylightFeatures implements Task { @@ -49,31 +52,63 @@ record Feature( private Object database; + /** + * Constructs an {@code ImportDaylightFeatures}. + */ public ImportDaylightFeatures() { } + /** + * Constructs an {@code ImportDaylightFeatures}. + * + * @param file the daylight file + * @param database the database + */ public ImportDaylightFeatures(Path file, Object database) { this.file = file; this.database = database; } + /** + * Returns the daylight file. + * + * @return the daylight file + */ public Path getFile() { return file; } + /** + * Sets the daylight file. + * + * @param file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the database. + * + * @return + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database + */ public void setDatabase(Object database) { this.database = database; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); @@ -95,42 +130,33 @@ public void execute(WorkflowContext context) throws Exception { case "node" -> { var node = nodeRepository.get(feature.id()); if (node != null) { - // Merge the tags var tags = new HashMap<>(feature.tags()); if (node.getTags() != null) { tags.putAll(node.getTags()); } node.setTags(tags); - - // Update the node nodeRepository.put(node); } } case "way" -> { var way = wayRepository.get(feature.id()); if (way != null) { - // Merge the tags var tags = new HashMap<>(feature.tags()); if (way.getTags() != null) { tags.putAll(way.getTags()); } way.setTags(tags); - - // Update the way wayRepository.put(way); } } case "relation" -> { var relation = relationRepository.get(feature.id()); if (relation != null) { - // Merge the tags var tags = new HashMap<>(feature.tags()); if (relation.getTags() != null) { tags.putAll(relation.getTags()); } relation.setTags(tags); - - // Update the relation relationRepository.put(relation); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java index dc6dc6193..93ef1b09c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Import daylight translations. + */ @JsonTypeName("ImportDaylightTranslations") public class ImportDaylightTranslations implements Task { @@ -61,31 +64,63 @@ public static Line parse(String line) { private Object database; + /** + * Constructs an {@code ImportDaylightTranslations}. + */ public ImportDaylightTranslations() { } + /** + * Constructs an {@code ImportDaylightTranslations}. + * + * @param file the daylight file + * @param database the database + */ public ImportDaylightTranslations(Path file, Object database) { this.file = file; this.database = database; } + /** + * Returns the daylight file. + * + * @return the daylight file + */ public Path getFile() { return file; } + /** + * Sets the daylight file. + * + * @param file the daylight file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index 882f85486..c2e73f5ab 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Import a GeoPackage into a database. + */ @JsonTypeName("ImportGeoPackage") public class ImportGeoPackage implements Task { @@ -40,10 +43,21 @@ public class ImportGeoPackage implements Task { private Integer sourceSRID; private Integer targetSRID; + /** + * Constructs an {@code ImportGeoPackage}. + */ public ImportGeoPackage() { } + /** + * Constructs an {@code ImportGeoPackage}. + * + * @param file the GeoPackage file + * @param database the database + * @param sourceSRID the source SRID + * @param targetSRID the target SRID + */ public ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer targetSRID) { this.file = file; this.database = database; @@ -51,38 +65,81 @@ public ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer this.targetSRID = targetSRID; } + /** + * Returns the GeoPackage file. + * + * @return the GeoPackage file + */ public Path getFile() { return file; } + /** + * Sets the GeoPackage file. + * + * @param file the GeoPackage file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the source SRID. + * + * @return the source SRID + */ public Integer getSourceSRID() { return sourceSRID; } + /** + * Sets the source SRID. + * + * @param sourceSRID the source SRID + */ public void setSourceSRID(Integer sourceSRID) { this.sourceSRID = sourceSRID; } + /** + * Returns the target SRID. + * + * @return the target SRID + */ public Integer getTargetSRID() { return targetSRID; } + /** + * Sets the target SRID. + * + * @param targetSRID the target SRID + */ public void setTargetSRID(Integer targetSRID) { this.targetSRID = targetSRID; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var path = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index da65e5238..045ce4c2f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -42,6 +42,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Import an OSM OSC file into a database. + */ @JsonTypeName("ImportOsmOsc") public class ImportOsmOsc implements Task { @@ -53,10 +56,22 @@ public class ImportOsmOsc implements Task { private Integer srid; private Compression compression; + /** + * Constructs an {@code ImportOsmOsc}. + */ public ImportOsmOsc() { } + /** + * Constructs an {@code ImportOsmOsc}. + * + * @param file the OSM OSC file + * @param cache the cache directory + * @param database the database + * @param srid the database SRID + * @param compression the compression + */ public ImportOsmOsc(Path file, Path cache, Object database, Integer srid, Compression compression) { this.file = file; @@ -66,46 +81,99 @@ public ImportOsmOsc(Path file, Path cache, Object database, Integer srid, this.compression = compression; } + /** + * Returns the OSM OSC file. + * + * @return the OSM OSC file + */ public Path getFile() { return file; } + /** + * Sets the OSM OSC file. + * + * @param file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the cache directory. + * + * @return the cache directory + */ public Path getCache() { return cache; } + /** + * Sets the cache directory. + * + * @param cache the cache directory + */ public void setCache(Path cache) { this.cache = cache; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the database SRID. + * + * @return the database SRID + */ public Integer getSrid() { return srid; } + /** + * Sets the database SRID. + * + * @param srid the database SRID + */ public void setSrid(Integer srid) { this.srid = srid; } + /** + * Returns the compression. + * + * @return the compression + */ public Compression getCompression() { return compression; } + /** + * Sets the compression. + * + * @param compression the compression + */ public void setCompression(Compression compression) { this.compression = compression; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index 8a14d78cd..d996037cd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -47,6 +47,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Import an OSM PBF file into a database. + */ @JsonTypeName("ImportOsmPbf") public class ImportOsmPbf implements Task { @@ -59,10 +62,23 @@ public class ImportOsmPbf implements Task { private Integer databaseSrid; private Boolean replaceExisting; + /** + * Constructs an {@code ImportOsmPbf}. + */ public ImportOsmPbf() { } + /** + * Constructs an {@code ImportOsmPbf}. + * + * @param file the OSM PBF file + * @param cache the cache directory + * @param cleanCache whether to clean the cache directory + * @param database the database + * @param databaseSrid the database SRID + * @param replaceExisting whether to replace the existing tables + */ public ImportOsmPbf(Path file, Path cache, Boolean cleanCache, Object database, Integer databaseSrid, Boolean replaceExisting) { this.file = file; @@ -73,54 +89,117 @@ public ImportOsmPbf(Path file, Path cache, Boolean cleanCache, Object database, this.replaceExisting = replaceExisting; } + /** + * Returns the OSM PBF file. + * + * @return the OSM PBF file + */ public Path getFile() { return file; } + /** + * Sets the OSM PBF file. + * + * @param file the OSM PBF file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the cache directory. + * + * @return the cache directory + */ public Path getCache() { return cache; } + /** + * Sets the cache directory. + * + * @param cache the cache directory + */ public void setCache(Path cache) { this.cache = cache; } + /** + * Returns whether to clean the cache directory. + * + * @return whether to clean the cache directory + */ public Boolean getCleanCache() { return cleanCache; } + /** + * Sets whether to clean the cache directory. + * + * @param cleanCache whether to clean the cache directory + */ public void setCleanCache(Boolean cleanCache) { this.cleanCache = cleanCache; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the database SRID. + * + * @return the database SRID + */ public Integer getDatabaseSrid() { return databaseSrid; } + /** + * Sets the database SRID. + * + * @param databaseSrid the database SRID + */ public void setDatabaseSrid(Integer databaseSrid) { this.databaseSrid = databaseSrid; } + /** + * Returns whether to replace the existing tables. + * + * @return whether to replace the existing tables + */ public Boolean getReplaceExisting() { return replaceExisting; } + /** + * Sets whether to replace the existing tables. + * + * @param replaceExisting whether to replace the existing tables + */ public void setReplaceExisting(Boolean replaceExisting) { this.replaceExisting = replaceExisting; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var dataSource = context.getDataSource(database); @@ -189,6 +268,19 @@ public void execute(WorkflowContext context) throws Exception { } } + /** + * Imports an OSM PBF file into a database. + * + * @param path the OSM PBF file + * @param coordinateMap the coordinate map + * @param referenceMap the reference map + * @param headerRepository the header repository + * @param nodeRepository the node repository + * @param wayRepository the way repository + * @param relationRepository the relation repository + * @param databaseSrid the database SRID + * @throws IOException + */ public static void execute( Path path, DataMap coordinateMap, diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index 5a68ce033..d67cad514 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Import a shapefile into a database. + */ @JsonTypeName("ImportShapefile") public class ImportShapefile implements Task { @@ -40,10 +43,21 @@ public class ImportShapefile implements Task { private Integer sourceSRID; private Integer targetSRID; + /** + * Constructs an {@code ImportShapefile}. + */ public ImportShapefile() { } + /** + * Constructs an {@code ImportShapefile}. + * + * @param file the shapefile file + * @param database the database + * @param sourceSRID the source SRID + * @param targetSRID the target SRID + */ public ImportShapefile(Path file, Object database, Integer sourceSRID, Integer targetSRID) { this.file = file; this.database = database; @@ -51,38 +65,81 @@ public ImportShapefile(Path file, Object database, Integer sourceSRID, Integer t this.targetSRID = targetSRID; } + /** + * Returns the shapefile file. + * + * @return the shapefile file + */ public Path getFile() { return file; } + /** + * Sets the shapefile file. + * + * @param file the shapefile file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the source SRID. + * + * @return the source SRID + */ public Integer getSourceSRID() { return sourceSRID; } + /** + * Sets the source SRID. + * + * @param sourceSRID the source SRID + */ public void setSourceSRID(Integer sourceSRID) { this.sourceSRID = sourceSRID; } + /** + * Returns the target SRID. + * + * @return the target SRID + */ public Integer getTargetSRID() { return targetSRID; } + /** + * Sets the target SRID. + * + * @param targetSRID the target SRID + */ public void setTargetSRID(Integer targetSRID) { this.targetSRID = targetSRID; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var path = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java index f57387bd3..2261d3f65 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java @@ -23,6 +23,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Log a message. + */ @JsonTypeName("LogMessage") public class LogMessage implements Task { @@ -30,22 +33,43 @@ public class LogMessage implements Task { private String message; + /** + * Constructs an {@code LogMessage}. + */ public LogMessage() { } + /** + * Constructs an {@code LogMessage}. + * + * @param message the message + */ public LogMessage(String message) { this.message = message; } + /** + * Returns the message. + * + * @return the message + */ public String getMessage() { return message; } + /** + * Sets the message. + * + * @param message the message + */ public void setMessage(String message) { this.message = message; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { logger.info(message); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java index cd9d3f8c2..c34bcb1d4 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java @@ -29,6 +29,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Ungzip a file. + */ @JsonTypeName("UngzipFile") public class UngzipFile implements Task { @@ -37,31 +40,63 @@ public class UngzipFile implements Task { private Path file; private Path directory; + /** + * Constructs an {@code UngzipFile}. + */ public UngzipFile() { } + /** + * Constructs an {@code UngzipFile}. + * + * @param file the file + * @param directory the directory + */ public UngzipFile(Path file, Path directory) { this.file = file; this.directory = directory; } + /** + * Returns the file. + * + * @return the file + */ public Path getFile() { return file; } + /** + * Sets the file. + * + * @param file the file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the directory. + * + * @return the directory + */ public Path getDirectory() { return directory; } + /** + * Sets the directory. + * + * @param directory the directory + */ public void setDirectory(Path directory) { this.directory = directory; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var filePath = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java index 4c95f599e..02185a90c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java @@ -26,6 +26,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Unzip a file. + */ @JsonTypeName("UnzipFile") public class UnzipFile implements Task { @@ -34,31 +37,63 @@ public class UnzipFile implements Task { private Path file; private Path directory; + /** + * Constructs an {@code UnzipFile}. + */ public UnzipFile() { } + /** + * Constructs an {@code UnzipFile}. + * + * @param file the file + * @param directory the directory + */ public UnzipFile(Path file, Path directory) { this.file = file; this.directory = directory; } + /** + * Returns the file. + * + * @return the file + */ public Path getFile() { return file; } + /** + * Sets the file. + * + * @param file the file + */ public void setFile(Path file) { this.file = file; } + /** + * Returns the directory. + * + * @return the directory + */ public Path getDirectory() { return directory; } + /** + * Sets the directory. + * + * @param directory the directory + */ public void setDirectory(Path directory) { this.directory = directory; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var filePath = file.toAbsolutePath(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index b11e84498..9dce1563e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -21,6 +21,9 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; import java.util.List; import java.util.zip.GZIPInputStream; import org.apache.baremaps.database.collection.DataMap; @@ -47,6 +50,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Update an OSM database based on the header data stored in the database. + */ @JsonTypeName("UpdateOsmDatabase") public class UpdateOsmDatabase implements Task { @@ -58,27 +64,56 @@ public class UpdateOsmDatabase implements Task { private String replicationUrl; + /** + * Constructs an {@code UpdateOsmDatabase}. + */ public UpdateOsmDatabase() { } + /** + * Constructs an {@code UpdateOsmDatabase}. + * + * @param database the database + * @param databaseSrid the database SRID + */ public UpdateOsmDatabase(Object database, Integer databaseSrid) { this.database = database; this.databaseSrid = databaseSrid; } + /** + * Returns the database. + * + * @return the database + */ public Object getDatabase() { return database; } + /** + * Sets the database. + * + * @param database the database + */ public void setDatabase(Object database) { this.database = database; } + /** + * Returns the database SRID. + * + * @return the database SRID + */ public Integer getDatabaseSrid() { return databaseSrid; } + /** + * Sets the database SRID. + * + * @param databaseSrid the database SRID + */ public void setDatabaseSrid(Integer databaseSrid) { this.databaseSrid = databaseSrid; } @@ -91,6 +126,9 @@ public void setReplicationUrl(String replicationUrl) { this.replicationUrl = replicationUrl; } + /** + * {@inheritDoc} + */ @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); @@ -111,6 +149,18 @@ public void execute(WorkflowContext context) throws Exception { replicationUrl); } + /** + * Executes the task. + * + * @param coordinateMap the coordinate map + * @param referenceMap the reference map + * @param headerRepository the header repository + * @param nodeRepository the node repository + * @param wayRepository the way repository + * @param relationRepository the relation repository + * @param databaseSrid the SRID + * @throws Exception if something went wrong + */ public static void execute(DataMap coordinateMap, DataMap> referenceMap, HeaderRepository headerRepository, Repository nodeRepository, @@ -160,4 +210,20 @@ public static void execute(DataMap coordinateMap, } } + /** + * Returns the URL of the file with the given sequence number. + * + * @param replicationUrl the replication URL + * @param sequenceNumber the sequence number + * @param extension the file extension + * @return the URL of the file + * @throws MalformedURLException if the URL is malformed + */ + public static URL resolve(String replicationUrl, Long sequenceNumber, String extension) + throws MalformedURLException { + var s = String.format("%09d", sequenceNumber); + var uri = String.format("%s/%s/%s/%s.%s", replicationUrl, s.substring(0, 3), s.substring(3, 6), + s.substring(6, 9), extension); + return URI.create(uri).toURL(); + } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java index 0bfd1daba..a754c92dc 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java @@ -39,7 +39,7 @@ public void test() throws IOException { new Step("download", List.of(), List.of(new DownloadUrl( "https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf", - Paths.get("liechtenstein-latest.osm.pbf")))), + Paths.get("liechtenstein-latest.osm.pbf"), false))), new Step("import", List.of("download"), List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"), null, diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index d52a27b65..c8ba01d80 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -37,7 +37,7 @@ class WorkflowTest extends PostgresContainerTest { void naturalearthGeoPackage() { var workflow = new Workflow(List.of(new Step("fetch-geopackage", List.of(), List.of( new DownloadUrl("https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", - Paths.get("natural_earth_vector.gpkg.zip")), + Paths.get("natural_earth_vector.gpkg.zip"), false), new UnzipFile(Paths.get("natural_earth_vector.gpkg.zip"), Paths.get("natural_earth_vector")), new ImportGeoPackage(Paths.get("natural_earth_vector/packages/natural_earth_vector.gpkg"), @@ -52,7 +52,7 @@ void coastlineShapefile() { var workflow = new Workflow(List.of(new Step("fetch-geopackage", List.of(), List.of( new DownloadUrl("https://osmdata.openstreetmap.de/download/coastlines-split-4326.zip", - Paths.get("coastlines-split-4326.zip")), + Paths.get("coastlines-split-4326.zip"), false), new UnzipFile(Paths.get("coastlines-split-4326.zip"), Paths.get("coastlines-split-4326")), new ImportShapefile(Paths.get("coastlines-split-4326/coastlines-split-4326/lines.shp"), @@ -84,7 +84,7 @@ void simplifiedWaterPolygonsShapefile() { void workflow() { var workflow = new Workflow(List.of(new Step("fetch-geopackage", List.of(), List.of( new DownloadUrl("https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", - Paths.get("downloads/import_db.gpkg")), + Paths.get("downloads/import_db.gpkg"), false), new ImportShapefile(Paths.get("downloads/import_db.gpkg"), jdbcUrl(), 4326, 3857))))); new WorkflowExecutor(workflow).execute(); } @@ -95,20 +95,20 @@ void execute() { var workflow = new Workflow(List.of( new Step("fetch-geopackage", List.of(), List.of(new DownloadUrl("https://tiles.baremaps.com/samples/import_db.gpkg", - Paths.get("downloads/import_db.gpkg")))), + Paths.get("downloads/import_db.gpkg"), false))), new Step("import-geopackage", List.of("fetch-geopackage"), List.of(new ImportGeoPackage(Paths.get("downloads/import_db.gpkg"), jdbcUrl(), 4326, 3857))), new Step("fetch-osmpbf", List.of(), List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf", - Paths.get("downloads/liechtenstein.osm.pbf")))), + Paths.get("downloads/liechtenstein.osm.pbf"), false))), new Step("import-osmpbf", List.of("fetch-osmpbf"), List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), null, true, jdbcUrl(), 3857, true))), new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl( "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip", - Paths.get("downloads/simplified-water-polygons-split-3857.zip")))), + Paths.get("downloads/simplified-water-polygons-split-3857.zip"), false))), new Step("unzip-shapefile", List.of("fetch-shapefile"), List.of( new UnzipFile(Paths.get("downloads/simplified-water-polygons-split-3857.zip"), @@ -116,7 +116,8 @@ void execute() { new Step("fetch-projection", List.of("unzip-shapefile"), List.of(new DownloadUrl("https://spatialreference.org/ref/sr-org/epsg3857/prj/", Paths.get( - "archives/simplified-water-polygons-split-3857/simplified_water_polygons.prj")))), + "archives/simplified-water-polygons-split-3857/simplified_water_polygons.prj"), + false))), new Step("import-shapefile", List.of("fetch-projection"), List.of(new ImportShapefile( Paths.get( diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java index ac8c68b7d..1c2526f10 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java @@ -49,7 +49,7 @@ void testDownloadFtp() throws Exception { var file = directory.resolve("file"); // TODO: do not use a 3rd party server, replaces test URL to a baremaps owned test resource. var task = new DownloadUrl("ftp://whois.in.bell.ca/bell.db.gz", - file); + file, false); task.execute(new WorkflowContext()); assertTrue(file.toFile().length() > 50, "file is less than 50 bytes"); FileUtils.deleteRecursively(directory); @@ -62,7 +62,7 @@ void testDownloadUnsupportedProtocol() throws Exception { var file = directory.resolve("file"); assertThrows(IllegalArgumentException.class, () -> { var task = new DownloadUrl("file://not-existing-file-243jhks", - file); + file, false); task.execute(new WorkflowContext()); }, "Unsupported protocol throws IOException"); FileUtils.deleteRecursively(directory); @@ -74,7 +74,7 @@ void executeFileThatDoesntExist() throws Exception { var directory = Files.createTempDirectory("tmp_"); var file = directory.resolve("README.md"); var task = new DownloadUrl("https://raw.githubusercontent.com/baremaps/baremaps/main/README.md", - file.toAbsolutePath()); + file.toAbsolutePath(), false); task.execute(new WorkflowContext()); assertTrue(Files.readString(file).contains("Baremaps")); FileUtils.deleteRecursively(directory); From 15539e8a27bf78e91fb5f2fd055f9a739bdbd14a Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Thu, 16 Nov 2023 22:25:18 +0100 Subject: [PATCH 04/21] Remove unused method --- .../workflow/tasks/UpdateOsmDatabase.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index 9dce1563e..b6513d04d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -209,21 +209,4 @@ public static void execute(DataMap coordinateMap, header.getReplicationUrl(), header.getSource(), header.getWritingProgram())); } } - - /** - * Returns the URL of the file with the given sequence number. - * - * @param replicationUrl the replication URL - * @param sequenceNumber the sequence number - * @param extension the file extension - * @return the URL of the file - * @throws MalformedURLException if the URL is malformed - */ - public static URL resolve(String replicationUrl, Long sequenceNumber, String extension) - throws MalformedURLException { - var s = String.format("%09d", sequenceNumber); - var uri = String.format("%s/%s/%s/%s.%s", replicationUrl, s.substring(0, 3), s.substring(3, 6), - s.substring(6, 9), extension); - return URI.create(uri).toURL(); - } } From f6d50133f297cff399c641bd758581c03e86723f Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Thu, 16 Nov 2023 23:32:07 +0100 Subject: [PATCH 05/21] Format code --- .../workflow/tasks/UpdateOsmDatabase.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index b6513d04d..d374be1ac 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -21,9 +21,6 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; import java.util.List; import java.util.zip.GZIPInputStream; import org.apache.baremaps.database.collection.DataMap; @@ -118,13 +115,13 @@ public void setDatabaseSrid(Integer databaseSrid) { this.databaseSrid = databaseSrid; } - public String getReplicationUrl() { - return replicationUrl; - } + public String getReplicationUrl() { + return replicationUrl; + } - public void setReplicationUrl(String replicationUrl) { - this.replicationUrl = replicationUrl; - } + public void setReplicationUrl(String replicationUrl) { + this.replicationUrl = replicationUrl; + } /** * {@inheritDoc} From 3ae0b0327a43db5dc843298b1edd6cc857bed234 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Thu, 16 Nov 2023 23:36:42 +0100 Subject: [PATCH 06/21] Add dependency to hk2 --- baremaps-core/pom.xml | 8 ++++++++ pom.xml | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/baremaps-core/pom.xml b/baremaps-core/pom.xml index 2b40f1306..bd424dc01 100644 --- a/baremaps-core/pom.xml +++ b/baremaps-core/pom.xml @@ -102,6 +102,14 @@ limitations under the License. org.apache.lucene lucene-spatial-extras + + org.glassfish.hk2 + hk2-api + + + org.glassfish.hk2 + hk2-locator + org.graalvm.js js diff --git a/pom.xml b/pom.xml index 7bb72aff0..54a751957 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,7 @@ limitations under the License. 23.0.2 32.1.3-jre 5.0.1 + 2.6.1 1.52 2.13.0 2.1.6 @@ -342,6 +343,16 @@ limitations under the License. ${version.lib.awaitability} test + + org.glassfish.hk2 + hk2-api + ${version.lib.hk2} + + + org.glassfish.hk2 + hk2-locator + ${version.lib.hk2} + org.glassfish.jersey jersey-bom From 256bbce493a5049c016e64b0f1b639de30ecc6b6 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 14:06:23 +0100 Subject: [PATCH 07/21] Remove unused methods --- .../baremaps/cli/database/UpdateOsm.java | 6 +- .../tasks/CreateGeocoderOpenStreetMap.java | 18 --- .../workflow/tasks/CreateGeonamesIndex.java | 20 --- .../workflow/tasks/CreateIplocIndex.java | 26 ---- .../workflow/tasks/DecompressFile.java | 59 --------- .../baremaps/workflow/tasks/DownloadUrl.java | 29 ----- .../workflow/tasks/ExecuteCommand.java | 23 ---- .../baremaps/workflow/tasks/ExecuteSql.java | 61 ---------- .../workflow/tasks/ExecuteSqlScript.java | 43 ------- .../workflow/tasks/ExportVectorTiles.java | 44 ------- .../tasks/ImportDaylightFeatures.java | 43 ------- .../tasks/ImportDaylightTranslations.java | 43 ------- .../workflow/tasks/ImportGeoPackage.java | 79 ------------ .../baremaps/workflow/tasks/ImportOsmOsc.java | 97 --------------- .../baremaps/workflow/tasks/ImportOsmPbf.java | 115 ------------------ .../workflow/tasks/ImportShapefile.java | 79 ------------ .../baremaps/workflow/tasks/LogMessage.java | 25 ---- .../baremaps/workflow/tasks/UngzipFile.java | 43 ------- .../baremaps/workflow/tasks/UnzipFile.java | 43 ------- .../workflow/tasks/UpdateOsmDatabase.java | 52 +------- 20 files changed, 6 insertions(+), 942 deletions(-) diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java index 886fd0a6d..832a6f728 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java @@ -41,9 +41,13 @@ public class UpdateOsm implements Callable { description = "The projection used by the database.") private int srid = 3857; + @Option(names = {"--replication-url"}, paramLabel = "REPLICATION_URL", + description = "The replication url of the OpenStreetMap server.") + private String replicationUrl = "https://planet.osm.org/replication/hour"; + @Override public Integer call() throws Exception { - new UpdateOsmDatabase(database, srid) + new UpdateOsmDatabase(database, srid, replicationUrl) .execute(new WorkflowContext()); return 0; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java index 81691994f..c45cc0861 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java @@ -61,29 +61,11 @@ public class CreateGeocoderOpenStreetMap implements Task { private Path indexDirectory; - public CreateGeocoderOpenStreetMap() {} - public CreateGeocoderOpenStreetMap(Path file, Path indexDirectory) { this.file = file; this.indexDirectory = indexDirectory; } - public Path getFile() { - return file; - } - - public void setFile(Path file) { - this.file = file; - } - - public Path getIndexDirectory() { - return indexDirectory; - } - - public void setIndexDirectory(Path indexDirectory) { - this.indexDirectory = indexDirectory; - } - @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java index 884c39abc..49fc25ccb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java @@ -45,31 +45,11 @@ public class CreateGeonamesIndex implements Task { private Path indexDirectory; - public CreateGeonamesIndex() { - - } - public CreateGeonamesIndex(Path dataFile, Path indexDirectory) { this.dataFile = dataFile; this.indexDirectory = indexDirectory; } - public Path getDataFile() { - return dataFile; - } - - public void setDataFile(Path dataFile) { - this.dataFile = dataFile; - } - - public Path getIndexDirectory() { - return indexDirectory; - } - - public void setIndexDirectory(Path indexDirectory) { - this.indexDirectory = indexDirectory; - } - @Override public void execute(WorkflowContext context) throws Exception { var directory = MMapDirectory.open(indexDirectory); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java index a3da337a3..01cedbfb8 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java @@ -46,38 +46,12 @@ public class CreateIplocIndex implements Task { private List nicPaths; private Path targetIplocIndexPath; - public CreateIplocIndex() {} - public CreateIplocIndex(Path geonamesIndexPath, List nicPaths, Path targetIplocIndexPath) { this.geonamesIndexPath = geonamesIndexPath; this.nicPaths = nicPaths; this.targetIplocIndexPath = targetIplocIndexPath; } - public Path getGeonamesIndexPath() { - return geonamesIndexPath; - } - - public void setGeonamesIndexPath(Path geonamesIndexPath) { - this.geonamesIndexPath = geonamesIndexPath; - } - - public List getNicPaths() { - return nicPaths; - } - - public void setNicPaths(List nicPaths) { - this.nicPaths = nicPaths; - } - - public Path getTargetIplocIndexPath() { - return targetIplocIndexPath; - } - - public void setTargetIplocIndexPath(Path targetIplocIndexPath) { - this.targetIplocIndexPath = targetIplocIndexPath; - } - @Override public void execute(WorkflowContext context) throws Exception { try (var directory = MMapDirectory.open(geonamesIndexPath); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index 37b0ac0e2..272ad1a3a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -57,11 +57,6 @@ public enum Compression { private Path target; private Compression compression; - /** - * Constructs a {@code DecompressFile}. - */ - public DecompressFile() {} - /** * Constructs a {@code DecompressFile}. * @@ -75,60 +70,6 @@ public DecompressFile(Path source, Path target, Compression compression) { this.compression = compression; } - /** - * Returns the source file. - * - * @return the source file - */ - public Path getSource() { - return source; - } - - /** - * Sets the source file. - * - * @param source the source file - */ - public void setSource(Path source) { - this.source = source; - } - - /** - * Returns the target file. - * - * @return the target file - */ - public Path getTarget() { - return target; - } - - /** - * Sets the target file. - * - * @param target the target file - */ - public void setTarget(Path target) { - this.target = target; - } - - /** - * Returns the compression format. - * - * @return the compression format - */ - public Compression getCompression() { - return compression; - } - - /** - * Sets the compression format. - * - * @param compression the compression format - */ - public void setCompression(Compression compression) { - this.compression = compression; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java index 767de3477..d25167d2a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java @@ -50,11 +50,6 @@ public class DownloadUrl implements Task { private Boolean replaceExisting; - /** - * Constructs an {@code DownloadUrl}. - */ - public DownloadUrl() {} - /** * Constructs an {@code DownloadUrl}. * @@ -68,30 +63,6 @@ public DownloadUrl(String url, Path path, boolean replaceExisting) { this.replaceExisting = replaceExisting; } - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public Path getPath() { - return path; - } - - public void setPath(Path path) { - this.path = path; - } - - public Boolean getReplaceExisting() { - return replaceExisting; - } - - public void setReplaceExisting(Boolean replaceExisting) { - this.replaceExisting = replaceExisting; - } - @Override public void execute(WorkflowContext context) throws Exception { var targetUrl = new URL(url); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java index 6d273a091..a19b23d6a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java @@ -33,11 +33,6 @@ public class ExecuteCommand implements Task { private String command; - /** - * Constructs an {@code ExecuteCommand}. - */ - public ExecuteCommand() {} - /** * Constructs an {@code ExecuteCommand}. * @@ -47,24 +42,6 @@ public ExecuteCommand(String command) { this.command = command; } - /** - * Returns the bash command. - * - * @return the bash command - */ - public String getCommand() { - return command; - } - - /** - * Sets the bash command. - * - * @param command the bash command - */ - public void setCommand(String command) { - this.command = command; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java index 96bb151e6..7cd78544b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java @@ -44,13 +44,6 @@ public class ExecuteSql implements Task { private Boolean parallel; - /** - * Constructs an {@code ExecuteSql}. - */ - public ExecuteSql() { - - } - /** * Constructs an {@code ExecuteSql}. * @@ -64,60 +57,6 @@ public ExecuteSql(Object database, Path file, Boolean parallel) { this.parallel = parallel; } - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - - /** - * Returns the SQL file. - * - * @return the SQL file - */ - public Path getFile() { - return file; - } - - /** - * Sets the SQL file. - * - * @param file the SQL file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns whether to execute the queries in parallel. - * - * @return whether to execute the queries in parallel - */ - public boolean isParallel() { - return parallel; - } - - /** - * Sets whether to execute the queries in parallel. - * - * @param parallel whether to execute the queries in parallel - */ - public void setParallel(boolean parallel) { - this.parallel = parallel; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java index af8dff0df..efd3b3c29 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java @@ -39,13 +39,6 @@ public class ExecuteSqlScript implements Task { private Path file; - /** - * Constructs an {@code ExecuteSqlScript}. - */ - public ExecuteSqlScript() { - - } - /** * Constructs an {@code ExecuteSqlScript}. * @@ -57,42 +50,6 @@ public ExecuteSqlScript(Object database, Path file) { this.file = file; } - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - - /** - * Returns the SQL file. - * - * @return the SQL file - */ - public Path getFile() { - return file; - } - - /** - * Sets the SQL file. - * - * @param file the SQL file - */ - public void setFile(Path file) { - this.file = file; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index 2e21d0a0f..2a7b077e7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -63,10 +63,6 @@ public enum Format { private Integer batchArrayIndex; private Format format; - public ExportVectorTiles() { - - } - public ExportVectorTiles(Path tileset, Path repository, Integer batchArraySize, Integer batchArrayIndex, Format format) { this.tileset = tileset; @@ -76,46 +72,6 @@ public ExportVectorTiles(Path tileset, Path repository, Integer batchArraySize, this.format = format; } - public Path getTileset() { - return tileset; - } - - public void setTileset(Path tileset) { - this.tileset = tileset; - } - - public Path getRepository() { - return repository; - } - - public void setRepository(Path repository) { - this.repository = repository; - } - - public Integer getBatchArraySize() { - return batchArraySize; - } - - public void setBatchArraySize(Integer batchArraySize) { - this.batchArraySize = batchArraySize; - } - - public Integer getBatchArrayIndex() { - return batchArrayIndex; - } - - public void setBatchArrayIndex(Integer batchArrayIndex) { - this.batchArrayIndex = batchArrayIndex; - } - - public Format getFormat() { - return format; - } - - public void setFormat(Format format) { - this.format = format; - } - @Override public void execute(WorkflowContext context) throws Exception { var configReader = new ConfigReader(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java index d116e339b..af05f9f7f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java @@ -52,13 +52,6 @@ record Feature( private Object database; - /** - * Constructs an {@code ImportDaylightFeatures}. - */ - public ImportDaylightFeatures() { - - } - /** * Constructs an {@code ImportDaylightFeatures}. * @@ -70,42 +63,6 @@ public ImportDaylightFeatures(Path file, Object database) { this.database = database; } - /** - * Returns the daylight file. - * - * @return the daylight file - */ - public Path getFile() { - return file; - } - - /** - * Sets the daylight file. - * - * @param file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the database. - * - * @return - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database - */ - public void setDatabase(Object database) { - this.database = database; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java index 93ef1b09c..f747f31fa 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java @@ -64,13 +64,6 @@ public static Line parse(String line) { private Object database; - /** - * Constructs an {@code ImportDaylightTranslations}. - */ - public ImportDaylightTranslations() { - - } - /** * Constructs an {@code ImportDaylightTranslations}. * @@ -82,42 +75,6 @@ public ImportDaylightTranslations(Path file, Object database) { this.database = database; } - /** - * Returns the daylight file. - * - * @return the daylight file - */ - public Path getFile() { - return file; - } - - /** - * Sets the daylight file. - * - * @param file the daylight file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index c2e73f5ab..9bfab836d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -43,13 +43,6 @@ public class ImportGeoPackage implements Task { private Integer sourceSRID; private Integer targetSRID; - /** - * Constructs an {@code ImportGeoPackage}. - */ - public ImportGeoPackage() { - - } - /** * Constructs an {@code ImportGeoPackage}. * @@ -65,78 +58,6 @@ public ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer this.targetSRID = targetSRID; } - /** - * Returns the GeoPackage file. - * - * @return the GeoPackage file - */ - public Path getFile() { - return file; - } - - /** - * Sets the GeoPackage file. - * - * @param file the GeoPackage file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - - /** - * Returns the source SRID. - * - * @return the source SRID - */ - public Integer getSourceSRID() { - return sourceSRID; - } - - /** - * Sets the source SRID. - * - * @param sourceSRID the source SRID - */ - public void setSourceSRID(Integer sourceSRID) { - this.sourceSRID = sourceSRID; - } - - /** - * Returns the target SRID. - * - * @return the target SRID - */ - public Integer getTargetSRID() { - return targetSRID; - } - - /** - * Sets the target SRID. - * - * @param targetSRID the target SRID - */ - public void setTargetSRID(Integer targetSRID) { - this.targetSRID = targetSRID; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index 045ce4c2f..b6a731d1e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -56,13 +56,6 @@ public class ImportOsmOsc implements Task { private Integer srid; private Compression compression; - /** - * Constructs an {@code ImportOsmOsc}. - */ - public ImportOsmOsc() { - - } - /** * Constructs an {@code ImportOsmOsc}. * @@ -81,96 +74,6 @@ public ImportOsmOsc(Path file, Path cache, Object database, Integer srid, this.compression = compression; } - /** - * Returns the OSM OSC file. - * - * @return the OSM OSC file - */ - public Path getFile() { - return file; - } - - /** - * Sets the OSM OSC file. - * - * @param file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the cache directory. - * - * @return the cache directory - */ - public Path getCache() { - return cache; - } - - /** - * Sets the cache directory. - * - * @param cache the cache directory - */ - public void setCache(Path cache) { - this.cache = cache; - } - - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - - /** - * Returns the database SRID. - * - * @return the database SRID - */ - public Integer getSrid() { - return srid; - } - - /** - * Sets the database SRID. - * - * @param srid the database SRID - */ - public void setSrid(Integer srid) { - this.srid = srid; - } - - /** - * Returns the compression. - * - * @return the compression - */ - public Compression getCompression() { - return compression; - } - - /** - * Sets the compression. - * - * @param compression the compression - */ - public void setCompression(Compression compression) { - this.compression = compression; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index d996037cd..34364056a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -62,13 +62,6 @@ public class ImportOsmPbf implements Task { private Integer databaseSrid; private Boolean replaceExisting; - /** - * Constructs an {@code ImportOsmPbf}. - */ - public ImportOsmPbf() { - - } - /** * Constructs an {@code ImportOsmPbf}. * @@ -89,114 +82,6 @@ public ImportOsmPbf(Path file, Path cache, Boolean cleanCache, Object database, this.replaceExisting = replaceExisting; } - /** - * Returns the OSM PBF file. - * - * @return the OSM PBF file - */ - public Path getFile() { - return file; - } - - /** - * Sets the OSM PBF file. - * - * @param file the OSM PBF file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the cache directory. - * - * @return the cache directory - */ - public Path getCache() { - return cache; - } - - /** - * Sets the cache directory. - * - * @param cache the cache directory - */ - public void setCache(Path cache) { - this.cache = cache; - } - - /** - * Returns whether to clean the cache directory. - * - * @return whether to clean the cache directory - */ - public Boolean getCleanCache() { - return cleanCache; - } - - /** - * Sets whether to clean the cache directory. - * - * @param cleanCache whether to clean the cache directory - */ - public void setCleanCache(Boolean cleanCache) { - this.cleanCache = cleanCache; - } - - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - - /** - * Returns the database SRID. - * - * @return the database SRID - */ - public Integer getDatabaseSrid() { - return databaseSrid; - } - - /** - * Sets the database SRID. - * - * @param databaseSrid the database SRID - */ - public void setDatabaseSrid(Integer databaseSrid) { - this.databaseSrid = databaseSrid; - } - - /** - * Returns whether to replace the existing tables. - * - * @return whether to replace the existing tables - */ - public Boolean getReplaceExisting() { - return replaceExisting; - } - - /** - * Sets whether to replace the existing tables. - * - * @param replaceExisting whether to replace the existing tables - */ - public void setReplaceExisting(Boolean replaceExisting) { - this.replaceExisting = replaceExisting; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index d67cad514..864fd220d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -43,13 +43,6 @@ public class ImportShapefile implements Task { private Integer sourceSRID; private Integer targetSRID; - /** - * Constructs an {@code ImportShapefile}. - */ - public ImportShapefile() { - - } - /** * Constructs an {@code ImportShapefile}. * @@ -65,78 +58,6 @@ public ImportShapefile(Path file, Object database, Integer sourceSRID, Integer t this.targetSRID = targetSRID; } - /** - * Returns the shapefile file. - * - * @return the shapefile file - */ - public Path getFile() { - return file; - } - - /** - * Sets the shapefile file. - * - * @param file the shapefile file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { - this.database = database; - } - - /** - * Returns the source SRID. - * - * @return the source SRID - */ - public Integer getSourceSRID() { - return sourceSRID; - } - - /** - * Sets the source SRID. - * - * @param sourceSRID the source SRID - */ - public void setSourceSRID(Integer sourceSRID) { - this.sourceSRID = sourceSRID; - } - - /** - * Returns the target SRID. - * - * @return the target SRID - */ - public Integer getTargetSRID() { - return targetSRID; - } - - /** - * Sets the target SRID. - * - * @param targetSRID the target SRID - */ - public void setTargetSRID(Integer targetSRID) { - this.targetSRID = targetSRID; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java index 2261d3f65..e0de8e00b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java @@ -33,13 +33,6 @@ public class LogMessage implements Task { private String message; - /** - * Constructs an {@code LogMessage}. - */ - public LogMessage() { - - } - /** * Constructs an {@code LogMessage}. * @@ -49,24 +42,6 @@ public LogMessage(String message) { this.message = message; } - /** - * Returns the message. - * - * @return the message - */ - public String getMessage() { - return message; - } - - /** - * Sets the message. - * - * @param message the message - */ - public void setMessage(String message) { - this.message = message; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java index c34bcb1d4..79c89a81c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java @@ -40,13 +40,6 @@ public class UngzipFile implements Task { private Path file; private Path directory; - /** - * Constructs an {@code UngzipFile}. - */ - public UngzipFile() { - - } - /** * Constructs an {@code UngzipFile}. * @@ -58,42 +51,6 @@ public UngzipFile(Path file, Path directory) { this.directory = directory; } - /** - * Returns the file. - * - * @return the file - */ - public Path getFile() { - return file; - } - - /** - * Sets the file. - * - * @param file the file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the directory. - * - * @return the directory - */ - public Path getDirectory() { - return directory; - } - - /** - * Sets the directory. - * - * @param directory the directory - */ - public void setDirectory(Path directory) { - this.directory = directory; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java index 02185a90c..0804168bb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java @@ -37,13 +37,6 @@ public class UnzipFile implements Task { private Path file; private Path directory; - /** - * Constructs an {@code UnzipFile}. - */ - public UnzipFile() { - - } - /** * Constructs an {@code UnzipFile}. * @@ -55,42 +48,6 @@ public UnzipFile(Path file, Path directory) { this.directory = directory; } - /** - * Returns the file. - * - * @return the file - */ - public Path getFile() { - return file; - } - - /** - * Sets the file. - * - * @param file the file - */ - public void setFile(Path file) { - this.file = file; - } - - /** - * Returns the directory. - * - * @return the directory - */ - public Path getDirectory() { - return directory; - } - - /** - * Sets the directory. - * - * @param directory the directory - */ - public void setDirectory(Path directory) { - this.directory = directory; - } - /** * {@inheritDoc} */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index d374be1ac..a2aa11bd3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -61,65 +61,15 @@ public class UpdateOsmDatabase implements Task { private String replicationUrl; - /** - * Constructs an {@code UpdateOsmDatabase}. - */ - public UpdateOsmDatabase() { - - } - /** * Constructs an {@code UpdateOsmDatabase}. * * @param database the database * @param databaseSrid the database SRID */ - public UpdateOsmDatabase(Object database, Integer databaseSrid) { - this.database = database; - this.databaseSrid = databaseSrid; - } - - /** - * Returns the database. - * - * @return the database - */ - public Object getDatabase() { - return database; - } - - /** - * Sets the database. - * - * @param database the database - */ - public void setDatabase(Object database) { + public UpdateOsmDatabase(Object database, Integer databaseSrid, String replicationUrl) { this.database = database; - } - - /** - * Returns the database SRID. - * - * @return the database SRID - */ - public Integer getDatabaseSrid() { - return databaseSrid; - } - - /** - * Sets the database SRID. - * - * @param databaseSrid the database SRID - */ - public void setDatabaseSrid(Integer databaseSrid) { this.databaseSrid = databaseSrid; - } - - public String getReplicationUrl() { - return replicationUrl; - } - - public void setReplicationUrl(String replicationUrl) { this.replicationUrl = replicationUrl; } From dadcac7642c2975ccf38038ac6009d7d65fa11b4 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 16:44:34 +0100 Subject: [PATCH 08/21] Simplify file decompression --- .../baremaps/cli/database/UpdateOsm.java | 2 +- .../workflow/tasks/DecompressFile.java | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java index 832a6f728..11fb8d6c2 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java @@ -42,7 +42,7 @@ public class UpdateOsm implements Callable { private int srid = 3857; @Option(names = {"--replication-url"}, paramLabel = "REPLICATION_URL", - description = "The replication url of the OpenStreetMap server.") + description = "The replication url of the OpenStreetMap server.") private String replicationUrl = "https://planet.osm.org/replication/hour"; @Override diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index 272ad1a3a..e5a4108bc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -133,11 +133,7 @@ protected static void decompressTarGz(Path source, Path target) throws IOExcepti Files.createDirectories(path.getParent()); try (BufferedOutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(path))) { - int bytesRead; - byte[] buffer = new byte[4096]; - while ((bytesRead = tarInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } + tarInputStream.transferTo(outputStream); } } } @@ -164,11 +160,7 @@ protected static void decompressTarBz2(Path source, Path target) throws IOExcept Files.createDirectories(path.getParent()); try (BufferedOutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(path))) { - int bytesRead; - byte[] buffer = new byte[4096]; - while ((bytesRead = tarInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } + tarInputStream.transferTo(outputStream); } } } @@ -193,11 +185,7 @@ protected static void decompressZip(Path source, Path target) throws IOException StandardOpenOption.TRUNCATE_EXISTING); try (var input = new BufferedInputStream(zipFile.getInputStream(entry)); var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) { - int nBytes = -1; - byte[] buffer = new byte[4096]; - while ((nBytes = input.read(buffer)) > 0) { - output.write(buffer, 0, nBytes); - } + input.transferTo(output); } } } From 63f4689aea38863fbe2329074dc1341ee379aaab Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 16:49:19 +0100 Subject: [PATCH 09/21] Replace UnzipFile by DecompressFile task in workflows --- .../workflow/tasks/DecompressFile.java | 3 ++- basemap/import.js | 21 +++++++++++-------- examples/geocoding/workflow.js | 2 +- examples/ip-to-location/workflow.js | 7 ++++--- examples/naturalearth/workflow.json | 7 ++++--- examples/shadedrelief/workflow.json | 7 ++++--- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index e5a4108bc..f9398307d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -181,7 +181,8 @@ protected static void decompressZip(Path source, Path target) throws IOException var entry = entries.nextElement(); var path = target.resolve(entry.getName()); Files.createDirectories(path.getParent()); - Files.write(path, new byte[] {}, StandardOpenOption.CREATE, + Files.write(path, new byte[] {}, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); try (var input = new BufferedInputStream(zipFile.getInputStream(entry)); var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) { diff --git a/basemap/import.js b/basemap/import.js index 757159915..4e77e8d1a 100644 --- a/basemap/import.js +++ b/basemap/import.js @@ -44,9 +44,10 @@ export default { "path": "data/natural_earth_vector.gpkg.zip" }, { - "type": "UnzipFile", - "file": "data/natural_earth_vector.gpkg.zip", - "directory": "data/natural_earth_vector" + "type": "DecompressFile", + "source": "data/natural_earth_vector.gpkg.zip", + "target": "data/natural_earth_vector", + "compression": "zip" }, { "type": "ImportGeoPackage", @@ -73,9 +74,10 @@ export default { "path": "data/water-polygons-split-3857.zip" }, { - "type": "UnzipFile", - "file": "data/water-polygons-split-3857.zip", - "directory": "data" + "type": "DecompressFile", + "source": "data/water-polygons-split-3857.zip", + "target": "data", + "compression": "zip" }, { "type": "ImportShapefile", @@ -96,9 +98,10 @@ export default { "path": "data/simplified-water-polygons-split-3857.zip" }, { - "type": "UnzipFile", - "file": "data/simplified-water-polygons-split-3857.zip", - "directory": "data" + "type": "DecompressFile", + "source": "data/simplified-water-polygons-split-3857.zip", + "target": "data", + "compression": "zip" }, { "type": "ImportShapefile", diff --git a/examples/geocoding/workflow.js b/examples/geocoding/workflow.js index bee197f87..ba54fcc05 100644 --- a/examples/geocoding/workflow.js +++ b/examples/geocoding/workflow.js @@ -19,7 +19,7 @@ const geonamesUrl = "https://download.geonames.org/export/dump/allCountries.zip" // Fetch and unzip Geonames const FetchAndUnzipGeonames = {id: "fetch-geonames-allcountries", needs: [], tasks: [ {type: "DownloadUrl", url: geonamesUrl, path: "downloads/geonames-allcountries.zip", force: true}, - {type: "UnzipFile", file: "downloads/geonames-allcountries.zip", directory: "archives"} + {type: "DecompressFile", source: "downloads/geonames-allcountries.zip", target: "archives", compression: "zip"} ]}; // Create the Geocoder index diff --git a/examples/ip-to-location/workflow.js b/examples/ip-to-location/workflow.js index c262e1791..67c5d9db8 100644 --- a/examples/ip-to-location/workflow.js +++ b/examples/ip-to-location/workflow.js @@ -77,9 +77,10 @@ export default {"steps": [ force: true }, { - type: "UnzipFile", - file: "downloads/geonames-allcountries.zip", - directory: "archives" + type: "DecompressFile", + source: "downloads/geonames-allcountries.zip", + target: "archives", + compression: "zip" }, { type: "CreateGeonamesIndex", diff --git a/examples/naturalearth/workflow.json b/examples/naturalearth/workflow.json index bd9d64449..b7a66864e 100644 --- a/examples/naturalearth/workflow.json +++ b/examples/naturalearth/workflow.json @@ -10,9 +10,10 @@ "path": "natural_earth_vector.gpkg.zip" }, { - "type": "UnzipFile", - "file": "natural_earth_vector.gpkg.zip", - "directory": "natural_earth_vector" + "type": "DecompressFile", + "source": "natural_earth_vector.gpkg.zip", + "target": "natural_earth_vector", + "compression": "zip" }, { "type": "ImportGeoPackage", diff --git a/examples/shadedrelief/workflow.json b/examples/shadedrelief/workflow.json index 48407d52b..f88c5d06e 100644 --- a/examples/shadedrelief/workflow.json +++ b/examples/shadedrelief/workflow.json @@ -10,9 +10,10 @@ "path": "shadedrelief.zip" }, { - "type": "UnzipFile", - "file": "shadedrelief.zip", - "directory": "shadedrelief" + "type": "DecompressFile", + "source": "shadedrelief.zip", + "target": "shadedrelief", + "compression": "zip" } ] } From 6eab90661ef479ec236aec6d16bf48bbb5b332ec Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 16:52:54 +0100 Subject: [PATCH 10/21] Remove UnzipFile task --- .../baremaps/workflow/tasks/UnzipFile.java | 92 ------------------- .../baremaps/workflow/WorkflowTest.java | 19 ++-- .../workflow/tasks/ImportShapefileTest.java | 15 +-- .../workflow/tasks/UnzipFileTest.java | 40 -------- 4 files changed, 16 insertions(+), 150 deletions(-) delete mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java delete mode 100644 baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java deleted file mode 100644 index 0804168bb..000000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.workflow.tasks; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import java.io.*; -import java.nio.file.*; -import java.util.zip.ZipFile; -import org.apache.baremaps.workflow.Task; -import org.apache.baremaps.workflow.WorkflowContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Unzip a file. - */ -@JsonTypeName("UnzipFile") -public class UnzipFile implements Task { - - private static final Logger logger = LoggerFactory.getLogger(UnzipFile.class); - - private Path file; - private Path directory; - - /** - * Constructs an {@code UnzipFile}. - * - * @param file the file - * @param directory the directory - */ - public UnzipFile(Path file, Path directory) { - this.file = file; - this.directory = directory; - } - - /** - * {@inheritDoc} - */ - @Override - public void execute(WorkflowContext context) throws Exception { - var filePath = file.toAbsolutePath(); - var directoryPath = directory.toAbsolutePath(); - - try (var zipFile = new ZipFile(filePath.toFile())) { - var entries = zipFile.entries(); - - while (entries.hasMoreElements()) { - var ze = entries.nextElement(); - if (ze.isDirectory()) { - continue; - } - - var path = directoryPath.resolve(ze.getName()); - - var file = path.toFile().getCanonicalFile(); - var directory = directoryPath.toFile().getCanonicalFile(); - if (!file.toPath().startsWith(directory.toPath())) { - throw new IOException("Entry is outside of the target directory"); - } - - Files.createDirectories(path.getParent()); - Files.write(path, new byte[] {}, StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING); - - try (var input = new BufferedInputStream(zipFile.getInputStream(ze)); - var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) { - - int nBytes; - byte[] buffer = new byte[4096]; - while ((nBytes = input.read(buffer)) > 0) { - output.write(buffer, 0, nBytes); - } - } - } - } - } -} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index c8ba01d80..6e7d220f4 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -22,11 +22,8 @@ import java.nio.file.Paths; import java.util.List; import org.apache.baremaps.testing.PostgresContainerTest; -import org.apache.baremaps.workflow.tasks.DownloadUrl; -import org.apache.baremaps.workflow.tasks.ImportGeoPackage; -import org.apache.baremaps.workflow.tasks.ImportOsmPbf; -import org.apache.baremaps.workflow.tasks.ImportShapefile; -import org.apache.baremaps.workflow.tasks.UnzipFile; +import org.apache.baremaps.workflow.tasks.*; +import org.apache.baremaps.workflow.tasks.DecompressFile.Compression; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -38,8 +35,8 @@ void naturalearthGeoPackage() { var workflow = new Workflow(List.of(new Step("fetch-geopackage", List.of(), List.of( new DownloadUrl("https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", Paths.get("natural_earth_vector.gpkg.zip"), false), - new UnzipFile(Paths.get("natural_earth_vector.gpkg.zip"), - Paths.get("natural_earth_vector")), + new DecompressFile(Paths.get("natural_earth_vector.gpkg.zip"), + Paths.get("natural_earth_vector"), Compression.zip), new ImportGeoPackage(Paths.get("natural_earth_vector/packages/natural_earth_vector.gpkg"), jdbcUrl(), 4326, 3857))))); @@ -53,8 +50,8 @@ void coastlineShapefile() { List.of( new DownloadUrl("https://osmdata.openstreetmap.de/download/coastlines-split-4326.zip", Paths.get("coastlines-split-4326.zip"), false), - new UnzipFile(Paths.get("coastlines-split-4326.zip"), - Paths.get("coastlines-split-4326")), + new DecompressFile(Paths.get("coastlines-split-4326.zip"), + Paths.get("coastlines-split-4326"), Compression.zip), new ImportShapefile(Paths.get("coastlines-split-4326/coastlines-split-4326/lines.shp"), jdbcUrl(), 4326, 3857))))); @@ -111,8 +108,8 @@ void execute() { Paths.get("downloads/simplified-water-polygons-split-3857.zip"), false))), new Step("unzip-shapefile", List.of("fetch-shapefile"), List.of( - new UnzipFile(Paths.get("downloads/simplified-water-polygons-split-3857.zip"), - Paths.get("archives")))), + new DecompressFile(Paths.get("downloads/simplified-water-polygons-split-3857.zip"), + Paths.get("archives"), Compression.zip))), new Step("fetch-projection", List.of("unzip-shapefile"), List.of(new DownloadUrl("https://spatialreference.org/ref/sr-org/epsg3857/prj/", Paths.get( diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java index d611efb01..1f63b9864 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java @@ -24,6 +24,7 @@ import org.apache.baremaps.testing.TestFiles; import org.apache.baremaps.utils.FileUtils; import org.apache.baremaps.workflow.WorkflowContext; +import org.apache.baremaps.workflow.tasks.DecompressFile.Compression; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -32,13 +33,13 @@ class ImportShapefileTest extends PostgresContainerTest { @Test @Tag("integration") void execute() throws Exception { - var zip = TestFiles.resolve("monaco-shapefile.zip"); - var directory = Files.createTempDirectory("tmp_"); - var unzip = new UnzipFile(zip, directory); - unzip.execute(new WorkflowContext()); - var task = new ImportShapefile(directory.resolve("gis_osm_buildings_a_free_1.shp"), + var source = TestFiles.resolve("monaco-shapefile.zip"); + var target = Files.createTempDirectory("tmp_"); + var decompressFile = new DecompressFile(source, target, Compression.zip); + decompressFile.execute(new WorkflowContext()); + var importShapefile = new ImportShapefile(target.resolve("gis_osm_buildings_a_free_1.shp"), jdbcUrl(), 4326, 3857); - task.execute(new WorkflowContext()); - FileUtils.deleteRecursively(directory); + importShapefile.execute(new WorkflowContext()); + FileUtils.deleteRecursively(target); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java deleted file mode 100644 index a93501390..000000000 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.workflow.tasks; - - - -import java.nio.file.Files; -import org.apache.baremaps.testing.TestFiles; -import org.apache.baremaps.utils.FileUtils; -import org.apache.baremaps.workflow.WorkflowContext; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -class UnzipFileTest { - - @Test - @Tag("integration") - void execute() throws Exception { - var zip = TestFiles.resolve("monaco-shapefile.zip"); - var directory = Files.createTempDirectory("tmp_"); - var task = new UnzipFile(zip, directory); - task.execute(new WorkflowContext()); - FileUtils.deleteRecursively(directory); - } -} From d4f6d394b380f465438fa6a2e222d2e63d321f80 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 16:56:51 +0100 Subject: [PATCH 11/21] Replace ungzip task by decompress task --- .../baremaps/workflow/tasks/UngzipFile.java | 73 ------------------- ...portPbfTest.java => ImportOsmPbfTest.java} | 0 .../workflow/tasks/UngzipFileTest.java | 40 ---------- examples/ip-to-location/workflow.js | 7 +- 4 files changed, 4 insertions(+), 116 deletions(-) delete mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java rename baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/{ImportPbfTest.java => ImportOsmPbfTest.java} (100%) delete mode 100644 baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UngzipFileTest.java diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java deleted file mode 100644 index 79c89a81c..000000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UngzipFile.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.workflow.tasks; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import java.io.BufferedInputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.zip.GZIPInputStream; -import org.apache.baremaps.workflow.Task; -import org.apache.baremaps.workflow.WorkflowContext; -import org.apache.baremaps.workflow.WorkflowException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Ungzip a file. - */ -@JsonTypeName("UngzipFile") -public class UngzipFile implements Task { - - private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class); - - private Path file; - private Path directory; - - /** - * Constructs an {@code UngzipFile}. - * - * @param file the file - * @param directory the directory - */ - public UngzipFile(Path file, Path directory) { - this.file = file; - this.directory = directory; - } - - /** - * {@inheritDoc} - */ - @Override - public void execute(WorkflowContext context) throws Exception { - var filePath = file.toAbsolutePath(); - var directoryPath = directory.toAbsolutePath(); - try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(filePath)))) { - var decompressed = directoryPath.resolve(filePath.getFileName().toString().substring(0, - filePath.getFileName().toString().length() - 3)); - if (!Files.exists(decompressed)) { - Files.createDirectories(decompressed.getParent()); - Files.createFile(decompressed); - } - Files.copy(zis, decompressed, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception e) { - throw new WorkflowException(e); - } - } -} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOsmPbfTest.java similarity index 100% rename from baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java rename to baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOsmPbfTest.java diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UngzipFileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UngzipFileTest.java deleted file mode 100644 index 78ab3504a..000000000 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UngzipFileTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.workflow.tasks; - - - -import java.nio.file.Files; -import org.apache.baremaps.testing.TestFiles; -import org.apache.baremaps.utils.FileUtils; -import org.apache.baremaps.workflow.WorkflowContext; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -class UngzipFileTest { - - @Test - @Tag("integration") - void run() throws Exception { - var gzip = TestFiles.resolve("ripe/sample.txt.gz"); - var directory = Files.createTempDirectory("tmp_"); - var task = new UngzipFile(gzip, directory); - task.execute(new WorkflowContext()); - FileUtils.deleteRecursively(directory); - } -} diff --git a/examples/ip-to-location/workflow.js b/examples/ip-to-location/workflow.js index 67c5d9db8..4ed538dce 100644 --- a/examples/ip-to-location/workflow.js +++ b/examples/ip-to-location/workflow.js @@ -65,9 +65,10 @@ export default {"steps": [ path: `downloads/${nic.filename}.gz` }, { - type: "UngzipFile", - file: `downloads/${nic.filename}.gz`, - directory: "archives" + type: "DecompressFile", + source: `downloads/${nic.filename}.gz`, + target: "archives", + compression: "gzip" } ]), { From 5f6382805b5f0b47b93b4bd09f0016c3cc815b41 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 17:07:55 +0100 Subject: [PATCH 12/21] Rename the parameters of the DownloadUrl task --- .../baremaps/workflow/tasks/DownloadUrl.java | 28 ++++++------ basemap/import.js | 16 +++---- daylight/workflow.js | 44 +++++++++---------- examples/extrusion/workflow.json | 4 +- examples/geocoding/workflow.js | 2 +- examples/ip-to-location/workflow.js | 8 ++-- examples/naturalearth/workflow.json | 4 +- examples/openstreetmap/workflow.json | 4 +- examples/shadedrelief/workflow.json | 4 +- 9 files changed, 57 insertions(+), 57 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java index d25167d2a..14bed10dc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java @@ -44,43 +44,43 @@ public class DownloadUrl implements Task { private static final String PROTOCOL_HTTPS = "https"; - private String url; + private String source; - private Path path; + private Path target; private Boolean replaceExisting; /** * Constructs an {@code DownloadUrl}. * - * @param url the url - * @param path the path + * @param source the url + * @param target the path * @param replaceExisting whether to replace existing files */ - public DownloadUrl(String url, Path path, boolean replaceExisting) { - this.url = url; - this.path = path; + public DownloadUrl(String source, Path target, boolean replaceExisting) { + this.source = source; + this.target = target; this.replaceExisting = replaceExisting; } @Override public void execute(WorkflowContext context) throws Exception { - var targetUrl = new URL(url); - var targetPath = path.toAbsolutePath(); + var sourceURL = new URL(source); + var targetPath = target.toAbsolutePath(); if (Files.exists(targetPath) && !replaceExisting) { - logger.info("Skipping download of {} to {}", url, path); + logger.info("Skipping download of {} to {}", source, target); return; } - if (isHttp(targetUrl)) { - var get = (HttpURLConnection) targetUrl.openConnection(); + if (isHttp(sourceURL)) { + var get = (HttpURLConnection) sourceURL.openConnection(); get.setInstanceFollowRedirects(true); get.setRequestMethod("GET"); urlDownloadToFile(get, targetPath); get.disconnect(); - } else if (isFtp(targetUrl)) { - urlDownloadToFile(targetUrl.openConnection(), targetPath); + } else if (isFtp(sourceURL)) { + urlDownloadToFile(sourceURL.openConnection(), targetPath); } else { throw new IllegalArgumentException("Unsupported URL protocol (supported: http(s)/ftp)"); } diff --git a/basemap/import.js b/basemap/import.js index 4e77e8d1a..cc868eba3 100644 --- a/basemap/import.js +++ b/basemap/import.js @@ -40,8 +40,8 @@ export default { "tasks": [ { "type": "DownloadUrl", - "url": "https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", - "path": "data/natural_earth_vector.gpkg.zip" + "source": "https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", + "target": "data/natural_earth_vector.gpkg.zip" }, { "type": "DecompressFile", @@ -70,8 +70,8 @@ export default { "tasks": [ { "type": "DownloadUrl", - "url": "https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip", - "path": "data/water-polygons-split-3857.zip" + "source": "https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip", + "target": "data/water-polygons-split-3857.zip" }, { "type": "DecompressFile", @@ -94,8 +94,8 @@ export default { "tasks": [ { "type": "DownloadUrl", - "url": "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip", - "path": "data/simplified-water-polygons-split-3857.zip" + "source": "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip", + "target": "data/simplified-water-polygons-split-3857.zip" }, { "type": "DecompressFile", @@ -142,8 +142,8 @@ export default { "tasks": [ { "type": "DownloadUrl", - "url": config.osmPbfUrl, - "path": "data/data.osm.pbf" + "source": config.osmPbfUrl, + "target": "data/data.osm.pbf" }, { "type": "ImportOsmPbf", diff --git a/daylight/workflow.js b/daylight/workflow.js index c256c87b7..78a8ea648 100644 --- a/daylight/workflow.js +++ b/daylight/workflow.js @@ -20,8 +20,8 @@ export default { "tasks": [ { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/planet-v1.33.osm.pbf", - "path": "data/data.osm.pbf" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/planet-v1.33.osm.pbf", + "target": "data/data.osm.pbf" }, { "type": "ImportOsmPbf", @@ -34,8 +34,8 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/ml-buildings-v1.33.osm.pbf", - "path": "data/buildings.osm.pbf" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/ml-buildings-v1.33.osm.pbf", + "target": "data/buildings.osm.pbf" }, { "type": "ImportOsmPbf", @@ -48,8 +48,8 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/fb-ml-roads-v1.33.osc.gz", - "path": "data/roads.osc.gz" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/fb-ml-roads-v1.33.osc.gz", + "target": "data/roads.osc.gz" }, { "type": "ImportOsmOsc", @@ -62,8 +62,8 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/admin-v1.33.osc.gz", - "path": "data/admin.osc.gz" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/admin-v1.33.osc.gz", + "target": "data/admin.osc.gz" }, { "type": "ImportOsmOsc", @@ -76,8 +76,8 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/coastlines-v1.33.tgz", - "path": "data/coastlines.tgz" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/coastlines-v1.33.tgz", + "target": "data/coastlines.tgz" }, { "type": "DecompressFile", @@ -99,8 +99,8 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/preferred-localization-v1.33.tsv", - "path": "data/preferred-localization.tsv" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/preferred-localization-v1.33.tsv", + "target": "data/preferred-localization.tsv" }, { "type": "ImportDaylightTranslations", @@ -109,8 +109,8 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/important-features-v1.33.json", - "path": "data/important-features.json" + "source": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/important-features-v1.33.json", + "target": "data/important-features.json" }, { "type": "ImportDaylightFeatures", @@ -119,23 +119,23 @@ export default { }, { "type": "DownloadUrl", - "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.shp", - "path": "data/landcover/low.shp" + "source": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.shp", + "target": "data/landcover/low.shp" }, { "type": "DownloadUrl", - "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.dbf", - "path": "data/landcover/low.dbf" + "source": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.dbf", + "target": "data/landcover/low.dbf" }, { "type": "DownloadUrl", - "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.prj", - "path": "data/landcover/low.prj" + "source": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.prj", + "target": "data/landcover/low.prj" }, { "type": "DownloadUrl", - "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.shx", - "path": "data/landcover/low.shx" + "source": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.shx", + "target": "data/landcover/low.shx" }, { "type": "ImportShapefile", diff --git a/examples/extrusion/workflow.json b/examples/extrusion/workflow.json index 039ad0d1a..c555df65e 100644 --- a/examples/extrusion/workflow.json +++ b/examples/extrusion/workflow.json @@ -6,8 +6,8 @@ "tasks": [ { "type": "DownloadUrl", - "url": "https://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf", - "path": "greater-london-latest.osm.pbf" + "source": "https://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf", + "target": "greater-london-latest.osm.pbf" } ] }, diff --git a/examples/geocoding/workflow.js b/examples/geocoding/workflow.js index ba54fcc05..f6ad29bc5 100644 --- a/examples/geocoding/workflow.js +++ b/examples/geocoding/workflow.js @@ -18,7 +18,7 @@ const geonamesUrl = "https://download.geonames.org/export/dump/allCountries.zip" // Fetch and unzip Geonames const FetchAndUnzipGeonames = {id: "fetch-geonames-allcountries", needs: [], tasks: [ - {type: "DownloadUrl", url: geonamesUrl, path: "downloads/geonames-allcountries.zip", force: true}, + {type: "DownloadUrl", source: geonamesUrl, target: "downloads/geonames-allcountries.zip", replaceExisting: true}, {type: "DecompressFile", source: "downloads/geonames-allcountries.zip", target: "archives", compression: "zip"} ]}; diff --git a/examples/ip-to-location/workflow.js b/examples/ip-to-location/workflow.js index 4ed538dce..cdbad8a6a 100644 --- a/examples/ip-to-location/workflow.js +++ b/examples/ip-to-location/workflow.js @@ -61,8 +61,8 @@ export default {"steps": [ ...nics.flatMap(nic => [ { type: "DownloadUrl", - url: nic.url, - path: `downloads/${nic.filename}.gz` + source: nic.url, + target: `downloads/${nic.filename}.gz` }, { type: "DecompressFile", @@ -73,8 +73,8 @@ export default {"steps": [ ]), { type: "DownloadUrl", - url: "https://download.geonames.org/export/dump/allCountries.zip", - path: "downloads/geonames-allcountries.zip", + source: "https://download.geonames.org/export/dump/allCountries.zip", + target: "downloads/geonames-allcountries.zip", force: true }, { diff --git a/examples/naturalearth/workflow.json b/examples/naturalearth/workflow.json index b7a66864e..96071919b 100644 --- a/examples/naturalearth/workflow.json +++ b/examples/naturalearth/workflow.json @@ -6,8 +6,8 @@ "tasks": [ { "type": "DownloadUrl", - "url": "https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", - "path": "natural_earth_vector.gpkg.zip" + "source": "https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", + "target": "natural_earth_vector.gpkg.zip" }, { "type": "DecompressFile", diff --git a/examples/openstreetmap/workflow.json b/examples/openstreetmap/workflow.json index d488cb4bf..ca62ac082 100644 --- a/examples/openstreetmap/workflow.json +++ b/examples/openstreetmap/workflow.json @@ -6,8 +6,8 @@ "tasks": [ { "type": "DownloadUrl", - "url": "https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf", - "path": "liechtenstein-latest.osm.pbf" + "source": "https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf", + "target": "liechtenstein-latest.osm.pbf" } ] }, diff --git a/examples/shadedrelief/workflow.json b/examples/shadedrelief/workflow.json index f88c5d06e..b530fac6d 100644 --- a/examples/shadedrelief/workflow.json +++ b/examples/shadedrelief/workflow.json @@ -6,8 +6,8 @@ "tasks": [ { "type": "DownloadUrl", - "url": "http://www.shadedrelief.com/ne-draft/World-Base-Map-Shapefiles.zip", - "path": "shadedrelief.zip" + "source": "http://www.shadedrelief.com/ne-draft/World-Base-Map-Shapefiles.zip", + "target": "shadedrelief.zip" }, { "type": "DecompressFile", From 0ce6ecea1923d8e9a9642397da69839e54adf902 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Fri, 17 Nov 2023 17:10:49 +0100 Subject: [PATCH 13/21] Remove unused parameters --- .../main/java/org/apache/baremaps/cli/map/Export.java | 10 +--------- .../baremaps/workflow/tasks/ExportVectorTiles.java | 7 +------ 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Export.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Export.java index 2c13a09b1..661fb81f1 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Export.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Export.java @@ -50,14 +50,6 @@ public class Export implements Callable { @Option(names = {"--tiles"}, paramLabel = "TILES", description = "The tiles to export.") private URI tiles; - @Option(names = {"--batch-array-size"}, paramLabel = "BATCH_ARRAY_SIZE", - description = "The size of the batch array.") - private int batchArraySize = 1; - - @Option(names = {"--batch-array-index"}, paramLabel = "READER", - description = "The index of the batch in the array.") - private int batchArrayIndex = 0; - @Option(names = {"--format"}, paramLabel = "FORMAT", description = "The format of the repository.") private ExportVectorTiles.Format format = ExportVectorTiles.Format.file; @@ -65,7 +57,7 @@ public class Export implements Callable { @Override public Integer call() throws Exception { new ExportVectorTiles(tileset.toAbsolutePath(), - repository.toAbsolutePath(), batchArraySize, batchArrayIndex, format) + repository.toAbsolutePath(), format) .execute(new WorkflowContext()); return 0; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index 2a7b077e7..dc0b47324 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -59,16 +59,11 @@ public enum Format { private Path tileset; private Path repository; - private Integer batchArraySize; - private Integer batchArrayIndex; private Format format; - public ExportVectorTiles(Path tileset, Path repository, Integer batchArraySize, - Integer batchArrayIndex, Format format) { + public ExportVectorTiles(Path tileset, Path repository, Format format) { this.tileset = tileset; this.repository = repository; - this.batchArraySize = batchArraySize; - this.batchArrayIndex = batchArrayIndex; this.format = format; } From b794ff27064f4be4e0a7a9ed5bbb8e069548dcdd Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sat, 18 Nov 2023 21:57:36 +0100 Subject: [PATCH 14/21] Improve workflow and naming --- .../baremaps/cli/database/ImportOsm.java | 2 - .../apache/baremaps/utils/GeometryUtils.java | 10 +-- .../baremaps/workflow/WorkflowContext.java | 84 +++++++++++++++++++ .../workflow/tasks/CleanContextCache.java | 31 +++++++ .../workflow/tasks/CleanContextData.java | 31 +++++++ .../workflow/tasks/ImportGeoPackage.java | 16 ++-- .../baremaps/workflow/tasks/ImportOsmOsc.java | 48 ++--------- .../baremaps/workflow/tasks/ImportOsmPbf.java | 66 +++------------ .../workflow/tasks/ImportShapefile.java | 16 ++-- .../baremaps/workflow/ObjectMapperTest.java | 2 - .../baremaps/workflow/WorkflowTest.java | 2 +- .../workflow/tasks/ImportOsmPbfTest.java | 2 +- .../src/test/resources/workflow.json | 4 +- basemap/import.js | 13 ++- daylight/workflow.js | 12 +-- examples/naturalearth/workflow.json | 4 +- 16 files changed, 200 insertions(+), 143 deletions(-) create mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java create mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java index 2f3ede49e..ad962fcc8 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java @@ -49,8 +49,6 @@ public class ImportOsm implements Callable { public Integer call() throws Exception { new org.apache.baremaps.workflow.tasks.ImportOsmPbf( file.toAbsolutePath(), - null, - true, database, srid, true).execute(new WorkflowContext()); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/GeometryUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/GeometryUtils.java index 62df82105..c0d95a3bc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/utils/GeometryUtils.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/GeometryUtils.java @@ -73,16 +73,16 @@ public static Geometry deserialize(byte[] wkb) { /** * Creates a coordinate transform with the provided SRIDs. * - * @param sourceSRID the source SRID - * @param targetSRID the target SRID + * @param sourceSrid the source SRID + * @param targetSrid the target SRID * @return the coordinate transform */ - public static CoordinateTransform coordinateTransform(Integer sourceSRID, Integer targetSRID) { + public static CoordinateTransform coordinateTransform(Integer sourceSrid, Integer targetSrid) { CRSFactory crsFactory = new CRSFactory(); CoordinateReferenceSystem sourceCRS = - crsFactory.createFromName(String.format("EPSG:%d", sourceSRID)); + crsFactory.createFromName(String.format("EPSG:%d", sourceSrid)); CoordinateReferenceSystem targetCRS = - crsFactory.createFromName(String.format("EPSG:%d", targetSRID)); + crsFactory.createFromName(String.format("EPSG:%d", targetSrid)); CoordinateTransformFactory coordinateTransformFactory = new CoordinateTransformFactory(); return coordinateTransformFactory.createTransform(sourceCRS, targetCRS); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java index f4a720f7d..1404039f6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java @@ -19,16 +19,50 @@ +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.sql.DataSource; +import org.apache.baremaps.database.collection.*; +import org.apache.baremaps.database.memory.MemoryMappedDirectory; +import org.apache.baremaps.database.type.LongDataType; +import org.apache.baremaps.database.type.LongListDataType; +import org.apache.baremaps.database.type.PairDataType; +import org.apache.baremaps.database.type.geometry.LonLatDataType; +import org.apache.baremaps.openstreetmap.postgres.PostgresHeaderRepository; +import org.apache.baremaps.openstreetmap.postgres.PostgresNodeRepository; +import org.apache.baremaps.openstreetmap.postgres.PostgresRelationRepository; +import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository; +import org.apache.baremaps.openstreetmap.repository.HeaderRepository; +import org.apache.baremaps.openstreetmap.repository.NodeRepository; +import org.apache.baremaps.openstreetmap.repository.RelationRepository; +import org.apache.baremaps.openstreetmap.repository.WayRepository; +import org.apache.baremaps.utils.FileUtils; import org.apache.baremaps.utils.PostgresUtils; +import org.locationtech.jts.geom.Coordinate; /** * A context that is passed to the tasks of a workflow and used to share data between tasks. */ public class WorkflowContext { + private final Path dataDir; + + private final Path cacheDir; + + public WorkflowContext() { + this(Paths.get("./data"), Paths.get("./cache")); + } + + public WorkflowContext(Path dataDir, Path cacheDir) { + this.dataDir = dataDir; + this.cacheDir = cacheDir; + } + private Map dataSources = new ConcurrentHashMap<>() {}; /** @@ -41,4 +75,54 @@ public DataSource getDataSource(Object database) { return dataSources.computeIfAbsent(database, PostgresUtils::createDataSourceFromObject); } + public DataMap getCoordinateMap(Path path) throws IOException { + if (Files.size(path) > 1 << 30) { + return getAlignedCoordinateMap(); + } else { + return getMonotonicCoordinateMap(); + } + } + + public DataMap getAlignedCoordinateMap() { + var coordinateDir = cacheDir.resolve("coordinates"); + return new MemoryAlignedDataMap<>( + new LonLatDataType(), + new MemoryMappedDirectory(coordinateDir)); + } + + public DataMap getMonotonicCoordinateMap() { + var coordinateKeysDir = cacheDir.resolve("coordinate_keys"); + var coordinateValuesDir = cacheDir.resolve("coordinate_values"); + return new MonotonicDataMap<>( + new MemoryAlignedDataList<>( + new PairDataType<>(new LongDataType(), new LongDataType()), + new MemoryMappedDirectory(coordinateKeysDir)), + new AppendOnlyBuffer<>( + new LonLatDataType(), + new MemoryMappedDirectory(coordinateValuesDir))); + } + + public DataMap> getReferenceMap(Path path) throws IOException { + return getMonotonicReferenceMap(); + } + + public DataMap> getMonotonicReferenceMap() { + var referenceKeysDir = cacheDir.resolve("reference_keys"); + var referenceValuesDir = cacheDir.resolve("reference_vals"); + return new MonotonicDataMap<>( + new MemoryAlignedDataList<>( + new PairDataType<>(new LongDataType(), new LongDataType()), + new MemoryMappedDirectory(referenceKeysDir)), + new AppendOnlyBuffer<>( + new LongListDataType(), + new MemoryMappedDirectory(referenceValuesDir))); + } + + public void cleanCache() throws IOException { + FileUtils.deleteRecursively(cacheDir); + } + + public void cleanData() throws IOException { + FileUtils.deleteRecursively(dataDir); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java new file mode 100644 index 000000000..f459f0ebf --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; + +@JsonTypeName("CleanContextCache") +public class CleanContextCache implements Task { + + @Override + public void execute(WorkflowContext context) throws Exception { + context.cleanCache(); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java new file mode 100644 index 000000000..3f5af41e7 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; + +@JsonTypeName("CleanContextData") +public class CleanContextData implements Task { + + @Override + public void execute(WorkflowContext context) throws Exception { + context.cleanData(); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index 9bfab836d..5e2b8eef0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -40,22 +40,22 @@ public class ImportGeoPackage implements Task { private Path file; private Object database; - private Integer sourceSRID; - private Integer targetSRID; + private Integer fileSrid; + private Integer databaseSrid; /** * Constructs an {@code ImportGeoPackage}. * * @param file the GeoPackage file * @param database the database - * @param sourceSRID the source SRID - * @param targetSRID the target SRID + * @param fileSrid the source SRID + * @param databaseSrid the target SRID */ - public ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer targetSRID) { + public ImportGeoPackage(Path file, Object database, Integer fileSrid, Integer databaseSrid) { this.file = file; this.database = database; - this.sourceSRID = sourceSRID; - this.targetSRID = targetSRID; + this.fileSrid = fileSrid; + this.databaseSrid = databaseSrid; } /** @@ -69,7 +69,7 @@ public void execute(WorkflowContext context) throws Exception { var postgresDataStore = new PostgresDataSchema(dataSource); for (var name : geoPackageDataStore.list()) { var geoPackageTable = geoPackageDataStore.get(name); - var projectionTransformer = new ProjectionTransformer(sourceSRID, targetSRID); + var projectionTransformer = new ProjectionTransformer(fileSrid, databaseSrid); var rowTransformer = new DataTableGeometryTransformer(geoPackageTable, projectionTransformer); var transformedDataTable = diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index b6a731d1e..54ff7b678 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -23,22 +23,14 @@ import java.io.BufferedInputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import org.apache.baremaps.database.collection.*; -import org.apache.baremaps.database.memory.MemoryMappedDirectory; -import org.apache.baremaps.database.type.LongDataType; -import org.apache.baremaps.database.type.LongListDataType; -import org.apache.baremaps.database.type.PairDataType; -import org.apache.baremaps.database.type.geometry.LonLatDataType; import org.apache.baremaps.openstreetmap.function.*; import org.apache.baremaps.openstreetmap.postgres.*; import org.apache.baremaps.openstreetmap.repository.CopyChangeImporter; import org.apache.baremaps.openstreetmap.xml.XmlChangeReader; import org.apache.baremaps.utils.Compression; -import org.apache.baremaps.utils.FileUtils; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; -import org.locationtech.jts.geom.Coordinate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,45 +71,17 @@ public ImportOsmOsc(Path file, Path cache, Object database, Integer srid, */ @Override public void execute(WorkflowContext context) throws Exception { - var datasource = context.getDataSource(database); var path = file.toAbsolutePath(); - var cacheDir = cache != null ? cache : Files.createTempDirectory(Paths.get("."), "cache_"); - - DataMap coordinateMap; - if (Files.size(path) > 1 << 30) { - var coordinateDir = Files.createDirectories(cacheDir.resolve("coordinates")); - coordinateMap = new MemoryAlignedDataMap<>( - new LonLatDataType(), - new MemoryMappedDirectory(coordinateDir)); - } else { - var coordinateKeysDir = Files.createDirectories(cacheDir.resolve("coordinate_keys")); - var coordinateValuesDir = Files.createDirectories(cacheDir.resolve("coordinate_vals")); - coordinateMap = - new MonotonicDataMap<>( - new MemoryAlignedDataList<>( - new PairDataType<>(new LongDataType(), new LongDataType()), - new MemoryMappedDirectory(coordinateKeysDir)), - new AppendOnlyBuffer<>( - new LonLatDataType(), - new MemoryMappedDirectory(coordinateValuesDir))); - } - - var referenceKeysDir = Files.createDirectories(cacheDir.resolve("reference_keys")); - var referenceValuesDir = Files.createDirectories(cacheDir.resolve("reference_vals")); - var referenceMap = - new MonotonicDataMap<>( - new MemoryAlignedDataList<>( - new PairDataType<>(new LongDataType(), new LongDataType()), - new MemoryMappedDirectory(referenceKeysDir)), - new AppendOnlyBuffer<>( - new LongListDataType(), - new MemoryMappedDirectory(referenceValuesDir))); - + // Initialize the repositories + var datasource = context.getDataSource(database); var nodeRepository = new PostgresNodeRepository(datasource); var wayRepository = new PostgresWayRepository(datasource); var relationRepository = new PostgresRelationRepository(datasource); + var coordinateMap = context.getCoordinateMap(path); + var referenceMap = context.getReferenceMap(path); + var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap); var referenceMapBuilder = new ReferenceMapBuilder(referenceMap); var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); @@ -133,7 +97,5 @@ public void execute(WorkflowContext context) throws Exception { new BufferedInputStream(compression.decompress(Files.newInputStream(path)))) { new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(importChange); } - - FileUtils.deleteRecursively(cacheDir); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index 34364056a..e20ec8b22 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -21,14 +21,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import org.apache.baremaps.database.collection.*; -import org.apache.baremaps.database.memory.MemoryMappedDirectory; -import org.apache.baremaps.database.type.LongDataType; -import org.apache.baremaps.database.type.LongListDataType; -import org.apache.baremaps.database.type.PairDataType; -import org.apache.baremaps.database.type.geometry.LonLatDataType; import org.apache.baremaps.openstreetmap.model.Node; import org.apache.baremaps.openstreetmap.model.Relation; import org.apache.baremaps.openstreetmap.model.Way; @@ -40,7 +34,6 @@ import org.apache.baremaps.openstreetmap.repository.*; import org.apache.baremaps.openstreetmap.repository.BlockImporter; import org.apache.baremaps.stream.StreamUtils; -import org.apache.baremaps.utils.FileUtils; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.locationtech.jts.geom.Coordinate; @@ -56,8 +49,6 @@ public class ImportOsmPbf implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class); private Path file; - private Path cache; - private Boolean cleanCache; private Object database; private Integer databaseSrid; private Boolean replaceExisting; @@ -66,17 +57,13 @@ public class ImportOsmPbf implements Task { * Constructs an {@code ImportOsmPbf}. * * @param file the OSM PBF file - * @param cache the cache directory - * @param cleanCache whether to clean the cache directory * @param database the database * @param databaseSrid the database SRID * @param replaceExisting whether to replace the existing tables */ - public ImportOsmPbf(Path file, Path cache, Boolean cleanCache, Object database, + public ImportOsmPbf(Path file, Object database, Integer databaseSrid, Boolean replaceExisting) { this.file = file; - this.cache = cache; - this.cleanCache = cleanCache; this.database = database; this.databaseSrid = databaseSrid; this.replaceExisting = replaceExisting; @@ -87,56 +74,31 @@ public ImportOsmPbf(Path file, Path cache, Boolean cleanCache, Object database, */ @Override public void execute(WorkflowContext context) throws Exception { - var dataSource = context.getDataSource(database); var path = file.toAbsolutePath(); - var headerRepository = new PostgresHeaderRepository(dataSource); - var nodeRepository = new PostgresNodeRepository(dataSource); - var wayRepository = new PostgresWayRepository(dataSource); - var relationRepository = new PostgresRelationRepository(dataSource); + // Initialize the repositories + var datasource = context.getDataSource(database); + var headerRepository = new PostgresHeaderRepository(datasource); + var nodeRepository = new PostgresNodeRepository(datasource); + var wayRepository = new PostgresWayRepository(datasource); + var relationRepository = new PostgresRelationRepository(datasource); if (replaceExisting) { + // Drop the existing tables headerRepository.drop(); nodeRepository.drop(); wayRepository.drop(); relationRepository.drop(); + + // Create the new tables headerRepository.create(); nodeRepository.create(); wayRepository.create(); relationRepository.create(); } - var cacheDir = cache != null ? cache : Files.createTempDirectory(Paths.get("."), "cache_"); - - DataMap coordinateMap; - if (Files.size(path) > 1 << 30) { - var coordinateDir = Files.createDirectories(cacheDir.resolve("coordinates")); - coordinateMap = new MemoryAlignedDataMap<>( - new LonLatDataType(), - new MemoryMappedDirectory(coordinateDir)); - } else { - var coordinateKeysDir = Files.createDirectories(cacheDir.resolve("coordinate_keys")); - var coordinateValuesDir = Files.createDirectories(cacheDir.resolve("coordinate_vals")); - coordinateMap = - new MonotonicDataMap<>( - new MemoryAlignedDataList<>( - new PairDataType<>(new LongDataType(), new LongDataType()), - new MemoryMappedDirectory(coordinateKeysDir)), - new AppendOnlyBuffer<>( - new LonLatDataType(), - new MemoryMappedDirectory(coordinateValuesDir))); - } - - var referenceKeysDir = Files.createDirectories(cacheDir.resolve("reference_keys")); - var referenceValuesDir = Files.createDirectories(cacheDir.resolve("reference_vals")); - var referenceMap = - new MonotonicDataMap<>( - new MemoryAlignedDataList<>( - new PairDataType<>(new LongDataType(), new LongDataType()), - new MemoryMappedDirectory(referenceKeysDir)), - new AppendOnlyBuffer<>( - new LongListDataType(), - new MemoryMappedDirectory(referenceValuesDir))); + var coordinateMap = context.getCoordinateMap(path); + var referenceMap = context.getReferenceMap(path); execute( path, @@ -147,10 +109,6 @@ public void execute(WorkflowContext context) throws Exception { wayRepository, relationRepository, databaseSrid); - - if (cleanCache) { - FileUtils.deleteRecursively(cacheDir); - } } /** diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index 864fd220d..40a15f5c7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -40,22 +40,22 @@ public class ImportShapefile implements Task { private Path file; private Object database; - private Integer sourceSRID; - private Integer targetSRID; + private Integer fileSrid; + private Integer databaseSrid; /** * Constructs an {@code ImportShapefile}. * * @param file the shapefile file * @param database the database - * @param sourceSRID the source SRID - * @param targetSRID the target SRID + * @param fileSrid the source SRID + * @param databaseSrid the target SRID */ - public ImportShapefile(Path file, Object database, Integer sourceSRID, Integer targetSRID) { + public ImportShapefile(Path file, Object database, Integer fileSrid, Integer databaseSrid) { this.file = file; this.database = database; - this.sourceSRID = sourceSRID; - this.targetSRID = targetSRID; + this.fileSrid = fileSrid; + this.databaseSrid = databaseSrid; } /** @@ -69,7 +69,7 @@ public void execute(WorkflowContext context) throws Exception { var dataSource = context.getDataSource(database); var postgresDataStore = new PostgresDataSchema(dataSource); var rowTransformer = new DataTableGeometryTransformer(shapefileDataTable, - new ProjectionTransformer(sourceSRID, targetSRID)); + new ProjectionTransformer(fileSrid, databaseSrid)); var transformedDataTable = new DataTableAdapter(shapefileDataTable, rowTransformer); postgresDataStore.add(transformedDataTable); } catch (Exception e) { diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java index a754c92dc..09f1d4a84 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java @@ -42,8 +42,6 @@ public void test() throws IOException { Paths.get("liechtenstein-latest.osm.pbf"), false))), new Step("import", List.of("download"), List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"), - null, - true, "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", 3857, true))))); var json = mapper.writeValueAsString(workflow1); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index 6e7d220f4..1f75b73b6 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -100,7 +100,7 @@ void execute() { List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf", Paths.get("downloads/liechtenstein.osm.pbf"), false))), new Step("import-osmpbf", List.of("fetch-osmpbf"), - List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), null, true, + List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(), 3857, true))), new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl( diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOsmPbfTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOsmPbfTest.java index 9684c6df1..4a913bc59 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOsmPbfTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOsmPbfTest.java @@ -33,7 +33,7 @@ void execute() throws Exception { var file = TestFiles.resolve("data.osm.pbf"); var jdbcUrl = jdbcUrl(); var srid = 3857; - var task = new ImportOsmPbf(file, null, true, jdbcUrl, srid, true); + var task = new ImportOsmPbf(file, jdbcUrl, srid, true); task.execute(new WorkflowContext()); } } diff --git a/baremaps-core/src/test/resources/workflow.json b/baremaps-core/src/test/resources/workflow.json index 3bfb651f6..3f68cb8f5 100644 --- a/baremaps-core/src/test/resources/workflow.json +++ b/baremaps-core/src/test/resources/workflow.json @@ -13,8 +13,8 @@ "username": "%s", "password": "%s" }, - "sourceSRID": 4326, - "targetSRID": 3857 + "fileSrid": 4326, + "databaseSrid": 3857 } ] } \ No newline at end of file diff --git a/basemap/import.js b/basemap/import.js index cc868eba3..472f04a6f 100644 --- a/basemap/import.js +++ b/basemap/import.js @@ -53,8 +53,8 @@ export default { "type": "ImportGeoPackage", "file": "data/natural_earth_vector/packages/natural_earth_vector.gpkg", "database": config.database, - "sourceSRID": 4326, - "targetSRID": 3857 + "fileSrid": 4326, + "databaseSrid": 3857 }, { "type": "ExecuteSql", @@ -83,8 +83,8 @@ export default { "type": "ImportShapefile", "file": "data/water-polygons-split-3857/water_polygons.shp", "database": config.database, - "sourceSRID": 3857, - "targetSRID": 3857 + "fileSrid": 3857, + "databaseSrid": 3857 }, ] }, @@ -107,8 +107,8 @@ export default { "type": "ImportShapefile", "file": "data/simplified-water-polygons-split-3857/simplified_water_polygons.shp", "database": config.database, - "sourceSRID": 3857, - "targetSRID": 3857 + "fileSrid": 3857, + "databaseSrid": 3857 }, ] }, @@ -151,7 +151,6 @@ export default { "database": config.database, "databaseSrid": 3857, "replaceExisting": true, - "cleanCache": true, }, ] }, diff --git a/daylight/workflow.js b/daylight/workflow.js index 78a8ea648..6bf76fbbd 100644 --- a/daylight/workflow.js +++ b/daylight/workflow.js @@ -27,7 +27,6 @@ export default { "type": "ImportOsmPbf", "file": "data/data.osm.pbf", "cache": "cache/", - "cleanCache": false, "database": config.database, "databaseSrid": 3857, "replaceExisting": true, @@ -41,7 +40,6 @@ export default { "type": "ImportOsmPbf", "file": "data/buildings.osm.pbf", "cache": "building_cache/", - "cleanCache": true, "database": config.database, "databaseSrid": 3857, "replaceExisting": false, @@ -55,7 +53,6 @@ export default { "type": "ImportOsmOsc", "file": "data/roads.osc.gz", "cache": "cache/", - "cleanCache": false, "compression": "gzip", "database": config.database, "srid": 3857 @@ -69,7 +66,6 @@ export default { "type": "ImportOsmOsc", "file": "data/admin.osc.gz", "cache": "cache/", - "cleanCache": false, "compression": "gzip", "database": config.database, "srid": 3857 @@ -89,8 +85,8 @@ export default { "type": "ImportShapefile", "file": "data/coastlines/water_polygons.shp", "database": config.database, - "sourceSRID": 4326, - "targetSRID": 3857 + "fileSrid": 4326, + "databaseSrid": 3857 }, { "type": "ExecuteSql", @@ -141,8 +137,8 @@ export default { "type": "ImportShapefile", "file": "data/landcover/low.shp", "database": config.database, - "sourceSRID": 4326, - "targetSRID": 3857 + "fileSrid": 4326, + "databaseSrid": 3857 }, ] }, diff --git a/examples/naturalearth/workflow.json b/examples/naturalearth/workflow.json index 96071919b..d3b6ee04b 100644 --- a/examples/naturalearth/workflow.json +++ b/examples/naturalearth/workflow.json @@ -19,8 +19,8 @@ "type": "ImportGeoPackage", "file": "natural_earth_vector/packages/natural_earth_vector.gpkg", "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", - "sourceSRID": 4326, - "targetSRID": 3857 + "fileSrid": 4326, + "databaseSrid": 3857 }, { "type": "ExecuteSql", From 88225689a35dd72c3a2e23be944e4e8d3f44777a Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sat, 18 Nov 2023 23:40:39 +0100 Subject: [PATCH 15/21] Some refactoring --- .../baremaps/workflow/WorkflowContext.java | 8 ------- .../workflow/tasks/CleanContextCache.java | 2 -- .../workflow/tasks/CleanContextData.java | 2 -- .../tasks/CreateGeocoderOpenStreetMap.java | 2 -- .../workflow/tasks/CreateGeonamesIndex.java | 2 -- .../workflow/tasks/CreateIplocIndex.java | 2 -- .../workflow/tasks/DecompressFile.java | 2 -- .../baremaps/workflow/tasks/DownloadUrl.java | 2 -- .../workflow/tasks/ExecuteCommand.java | 2 -- .../baremaps/workflow/tasks/ExecuteSql.java | 2 -- .../workflow/tasks/ExecuteSqlScript.java | 2 -- .../workflow/tasks/ExportVectorTiles.java | 2 -- .../tasks/ImportDaylightFeatures.java | 2 -- .../tasks/ImportDaylightTranslations.java | 2 -- .../workflow/tasks/ImportGeoPackage.java | 10 ++++---- .../baremaps/workflow/tasks/ImportOsmOsc.java | 23 +++++++------------ .../baremaps/workflow/tasks/ImportOsmPbf.java | 2 -- .../workflow/tasks/ImportShapefile.java | 10 ++++---- .../baremaps/workflow/tasks/LogMessage.java | 2 -- .../workflow/tasks/UpdateOsmDatabase.java | 10 ++++---- .../baremaps/workflow/WorkflowTest.java | 16 ++++++------- .../workflow/tasks/ImportGeoPackageTest.java | 2 +- .../workflow/tasks/ImportShapefileTest.java | 2 +- daylight/workflow.js | 6 ++--- 24 files changed, 33 insertions(+), 84 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java index 1404039f6..1e8ea59c2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java @@ -33,14 +33,6 @@ import org.apache.baremaps.database.type.LongListDataType; import org.apache.baremaps.database.type.PairDataType; import org.apache.baremaps.database.type.geometry.LonLatDataType; -import org.apache.baremaps.openstreetmap.postgres.PostgresHeaderRepository; -import org.apache.baremaps.openstreetmap.postgres.PostgresNodeRepository; -import org.apache.baremaps.openstreetmap.postgres.PostgresRelationRepository; -import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository; -import org.apache.baremaps.openstreetmap.repository.HeaderRepository; -import org.apache.baremaps.openstreetmap.repository.NodeRepository; -import org.apache.baremaps.openstreetmap.repository.RelationRepository; -import org.apache.baremaps.openstreetmap.repository.WayRepository; import org.apache.baremaps.utils.FileUtils; import org.apache.baremaps.utils.PostgresUtils; import org.locationtech.jts.geom.Coordinate; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java index f459f0ebf..b4acbf0db 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextCache.java @@ -17,11 +17,9 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; -@JsonTypeName("CleanContextCache") public class CleanContextCache implements Task { @Override diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java index 3f5af41e7..498b3840a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CleanContextData.java @@ -17,11 +17,9 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; -@JsonTypeName("CleanContextData") public class CleanContextData implements Task { @Override diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java index c45cc0861..f7285375a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -52,7 +51,6 @@ * * @see org.apache.baremaps.geocoderosm */ -@JsonTypeName("CreateGeocoderOpenStreetMap") public class CreateGeocoderOpenStreetMap implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateGeocoderOpenStreetMap.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java index 49fc25ccb..fb37fdee0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -36,7 +35,6 @@ /** * A task that creates a geonames index. */ -@JsonTypeName("CreateGeonamesIndex") public class CreateGeonamesIndex implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateGeonamesIndex.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java index 01cedbfb8..7d7d26785 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -37,7 +36,6 @@ import org.sqlite.SQLiteConfig; import org.sqlite.SQLiteDataSource; -@JsonTypeName("CreateIplocIndex") public class CreateIplocIndex implements Task { private static final Logger logger = LoggerFactory.getLogger(CreateIplocIndex.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index f9398307d..18bef4d78 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.*; import java.nio.file.Files; import java.nio.file.Path; @@ -37,7 +36,6 @@ * Decompresses a file based on a given compression format. The supported formats are zip, targz, * tarbz2, gzip and bzip2. */ -@JsonTypeName("DecompressFile") public class DecompressFile implements Task { private static final Logger logger = LoggerFactory.getLogger(DecompressFile.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java index 14bed10dc..554e87b13 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; @@ -33,7 +32,6 @@ /** * Downloads a file from a URL. */ -@JsonTypeName("DownloadUrl") public class DownloadUrl implements Task { private static final Logger logger = LoggerFactory.getLogger(DownloadUrl.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java index a19b23d6a..f9783f75a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteCommand.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.slf4j.Logger; @@ -26,7 +25,6 @@ /** * Execute a bash command. */ -@JsonTypeName("ExecuteCommand") public class ExecuteCommand implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteCommand.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java index 7cd78544b..13423ab16 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; @@ -33,7 +32,6 @@ /** * Execute a SQL query (single statement). */ -@JsonTypeName("ExecuteSql") public class ExecuteSql implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteSql.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java index efd3b3c29..7ed4ae053 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; @@ -30,7 +29,6 @@ /** * Execute a SQL script (multiple statements). */ -@JsonTypeName("ExecuteSqlScript") public class ExecuteSqlScript implements Task { private static final Logger logger = LoggerFactory.getLogger(ExecuteSqlScript.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index dc0b47324..2e032980d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -19,7 +19,6 @@ import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper; -import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -46,7 +45,6 @@ import org.slf4j.LoggerFactory; -@JsonTypeName("ExportVectorTiles") public class ExportVectorTiles implements Task { private static final Logger logger = LoggerFactory.getLogger(ExportVectorTiles.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java index af05f9f7f..7c5e32843 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java @@ -18,7 +18,6 @@ package org.apache.baremaps.workflow.tasks; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Path; import java.util.HashMap; @@ -35,7 +34,6 @@ /** * Import daylight features. */ -@JsonTypeName("ImportDaylightFeatures") public class ImportDaylightFeatures implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportDaylightFeatures.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java index f747f31fa..16ef9fec7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; @@ -33,7 +32,6 @@ /** * Import daylight translations. */ -@JsonTypeName("ImportDaylightTranslations") public class ImportDaylightTranslations implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportDaylightTranslations.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index 5e2b8eef0..1bfa66c59 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Path; import org.apache.baremaps.database.schema.DataTableAdapter; import org.apache.baremaps.database.schema.DataTableGeometryTransformer; @@ -33,28 +32,27 @@ /** * Import a GeoPackage into a database. */ -@JsonTypeName("ImportGeoPackage") public class ImportGeoPackage implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportGeoPackage.class); private Path file; - private Object database; private Integer fileSrid; + private Object database; private Integer databaseSrid; /** * Constructs an {@code ImportGeoPackage}. * * @param file the GeoPackage file - * @param database the database * @param fileSrid the source SRID + * @param database the database * @param databaseSrid the target SRID */ - public ImportGeoPackage(Path file, Object database, Integer fileSrid, Integer databaseSrid) { + public ImportGeoPackage(Path file, Integer fileSrid, Object database, Integer databaseSrid) { this.file = file; - this.database = database; this.fileSrid = fileSrid; + this.database = database; this.databaseSrid = databaseSrid; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java index 54ff7b678..7e21c3cbb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java @@ -19,11 +19,9 @@ import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.nio.file.Files; import java.nio.file.Path; -import org.apache.baremaps.database.collection.*; import org.apache.baremaps.openstreetmap.function.*; import org.apache.baremaps.openstreetmap.postgres.*; import org.apache.baremaps.openstreetmap.repository.CopyChangeImporter; @@ -37,33 +35,28 @@ /** * Import an OSM OSC file into a database. */ -@JsonTypeName("ImportOsmOsc") public class ImportOsmOsc implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmOsc.class); private Path file; - private Path cache; - private Object database; - private Integer srid; private Compression compression; + private Object database; + private Integer databaseSrid; /** * Constructs an {@code ImportOsmOsc}. * * @param file the OSM OSC file - * @param cache the cache directory - * @param database the database - * @param srid the database SRID * @param compression the compression + * @param database the database + * @param databaseSrid the database SRID */ - public ImportOsmOsc(Path file, Path cache, Object database, Integer srid, - Compression compression) { + public ImportOsmOsc(Path file, Compression compression, Object database, Integer databaseSrid) { this.file = file; - this.cache = cache; - this.database = database; - this.srid = srid; this.compression = compression; + this.database = database; + this.databaseSrid = databaseSrid; } /** @@ -85,7 +78,7 @@ public void execute(WorkflowContext context) throws Exception { var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap); var referenceMapBuilder = new ReferenceMapBuilder(referenceMap); var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); - var reprojectGeometry = new EntityProjectionTransformer(4326, srid); + var reprojectGeometry = new EntityProjectionTransformer(4326, databaseSrid); var prepareGeometries = coordinateMapBuilder .andThen(referenceMapBuilder) .andThen(buildGeometry) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index e20ec8b22..102d24fc6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -43,7 +42,6 @@ /** * Import an OSM PBF file into a database. */ -@JsonTypeName("ImportOsmPbf") public class ImportOsmPbf implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index 40a15f5c7..bfb9a10a5 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.nio.file.Path; import org.apache.baremaps.database.schema.DataTableAdapter; import org.apache.baremaps.database.schema.DataTableGeometryTransformer; @@ -33,28 +32,27 @@ /** * Import a shapefile into a database. */ -@JsonTypeName("ImportShapefile") public class ImportShapefile implements Task { private static final Logger logger = LoggerFactory.getLogger(ImportShapefile.class); private Path file; - private Object database; private Integer fileSrid; + private Object database; private Integer databaseSrid; /** * Constructs an {@code ImportShapefile}. * * @param file the shapefile file - * @param database the database * @param fileSrid the source SRID + * @param database the database * @param databaseSrid the target SRID */ - public ImportShapefile(Path file, Object database, Integer fileSrid, Integer databaseSrid) { + public ImportShapefile(Path file, Integer fileSrid, Object database, Integer databaseSrid) { this.file = file; - this.database = database; this.fileSrid = fileSrid; + this.database = database; this.databaseSrid = databaseSrid; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java index e0de8e00b..86591f742 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/LogMessage.java @@ -17,7 +17,6 @@ package org.apache.baremaps.workflow.tasks; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.slf4j.Logger; @@ -26,7 +25,6 @@ /** * Log a message. */ -@JsonTypeName("LogMessage") public class LogMessage implements Task { private static final Logger logger = LoggerFactory.getLogger(LogMessage.class); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java index a2aa11bd3..99e95e447 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java @@ -19,7 +19,6 @@ import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; -import com.fasterxml.jackson.annotation.JsonTypeName; import java.io.BufferedInputStream; import java.util.List; import java.util.zip.GZIPInputStream; @@ -50,15 +49,12 @@ /** * Update an OSM database based on the header data stored in the database. */ -@JsonTypeName("UpdateOsmDatabase") public class UpdateOsmDatabase implements Task { private static final Logger logger = LoggerFactory.getLogger(UpdateOsmDatabase.class); private Object database; - private Integer databaseSrid; - private String replicationUrl; /** @@ -115,6 +111,7 @@ public static void execute(DataMap coordinateMap, Integer databaseSrid, String replicationUrl) throws Exception { + // Get the latest header from the database var header = headerRepository.selectLatest(); // If the replicationUrl is not provided, use the one from the latest header. @@ -122,6 +119,7 @@ public static void execute(DataMap coordinateMap, replicationUrl = header.getReplicationUrl(); } + // Get the sequence number of the latest header var stateReader = new StateReader(replicationUrl, true); var sequenceNumber = header.getReplicationSequenceNumber(); @@ -134,21 +132,23 @@ public static void execute(DataMap coordinateMap, } } + // Increment the sequence number and get the changeset url var nextSequenceNumber = sequenceNumber + 1; var changeUrl = stateReader.getUrl(replicationUrl, nextSequenceNumber, "osc.gz"); logger.info("Updating the database with the changeset: {}", changeUrl); + // Process the changeset and update the database var createGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); var reprojectGeometry = new EntityProjectionTransformer(4326, databaseSrid); var prepareGeometries = new ChangeEntitiesHandler(createGeometry.andThen(reprojectGeometry)); var prepareChange = consumeThenReturn(prepareGeometries); var importChange = new PutChangeImporter(nodeRepository, wayRepository, relationRepository); - try (var changeInputStream = new GZIPInputStream(new BufferedInputStream(changeUrl.openStream()))) { new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(importChange); } + // Add the new header to the database var stateUrl = stateReader.getUrl(replicationUrl, nextSequenceNumber, "state.txt"); try (var stateInputStream = new BufferedInputStream(stateUrl.openStream())) { var state = new StateReader().readState(stateInputStream); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index 1f75b73b6..dbc0c8556 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -38,8 +38,8 @@ void naturalearthGeoPackage() { new DecompressFile(Paths.get("natural_earth_vector.gpkg.zip"), Paths.get("natural_earth_vector"), Compression.zip), new ImportGeoPackage(Paths.get("natural_earth_vector/packages/natural_earth_vector.gpkg"), - jdbcUrl(), - 4326, 3857))))); + 4326, jdbcUrl(), + 3857))))); new WorkflowExecutor(workflow).execute(); } @@ -53,8 +53,8 @@ void coastlineShapefile() { new DecompressFile(Paths.get("coastlines-split-4326.zip"), Paths.get("coastlines-split-4326"), Compression.zip), new ImportShapefile(Paths.get("coastlines-split-4326/coastlines-split-4326/lines.shp"), - jdbcUrl(), - 4326, 3857))))); + 4326, jdbcUrl(), + 3857))))); new WorkflowExecutor(workflow).execute(); } @@ -71,7 +71,7 @@ void simplifiedWaterPolygonsShapefile() { new ImportShapefile( Paths.get( "simplified-water-polygons-split-3857/simplified-water-polygons-split-3857/simplified_water_polygons.shp"), - "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", 3857, + 3857, "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", 3857))))); new WorkflowExecutor(workflow).execute(); } @@ -82,7 +82,7 @@ void workflow() { var workflow = new Workflow(List.of(new Step("fetch-geopackage", List.of(), List.of( new DownloadUrl("https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip", Paths.get("downloads/import_db.gpkg"), false), - new ImportShapefile(Paths.get("downloads/import_db.gpkg"), jdbcUrl(), 4326, 3857))))); + new ImportShapefile(Paths.get("downloads/import_db.gpkg"), 4326, jdbcUrl(), 3857))))); new WorkflowExecutor(workflow).execute(); } @@ -94,7 +94,7 @@ void execute() { List.of(new DownloadUrl("https://tiles.baremaps.com/samples/import_db.gpkg", Paths.get("downloads/import_db.gpkg"), false))), new Step("import-geopackage", List.of("fetch-geopackage"), - List.of(new ImportGeoPackage(Paths.get("downloads/import_db.gpkg"), jdbcUrl(), 4326, + List.of(new ImportGeoPackage(Paths.get("downloads/import_db.gpkg"), 4326, jdbcUrl(), 3857))), new Step("fetch-osmpbf", List.of(), List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf", @@ -119,7 +119,7 @@ void execute() { List.of(new ImportShapefile( Paths.get( "archives/simplified-water-polygons-split-3857/simplified_water_polygons.shp"), - jdbcUrl(), 3857, 3857))))); + 3857, jdbcUrl(), 3857))))); new WorkflowExecutor(workflow).execute(); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportGeoPackageTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportGeoPackageTest.java index c8473c467..e633a13ef 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportGeoPackageTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportGeoPackageTest.java @@ -31,7 +31,7 @@ class ImportGeoPackageTest extends PostgresContainerTest { @Tag("integration") void execute() throws Exception { var task = - new ImportGeoPackage(TestFiles.resolve("data.gpkg"), jdbcUrl(), 4326, 3857); + new ImportGeoPackage(TestFiles.resolve("data.gpkg"), 4326, jdbcUrl(), 3857); task.execute(new WorkflowContext()); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java index 1f63b9864..443e829e5 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java @@ -38,7 +38,7 @@ void execute() throws Exception { var decompressFile = new DecompressFile(source, target, Compression.zip); decompressFile.execute(new WorkflowContext()); var importShapefile = new ImportShapefile(target.resolve("gis_osm_buildings_a_free_1.shp"), - jdbcUrl(), 4326, 3857); + 4326, jdbcUrl(), 3857); importShapefile.execute(new WorkflowContext()); FileUtils.deleteRecursively(target); } diff --git a/daylight/workflow.js b/daylight/workflow.js index 6bf76fbbd..9df4a0def 100644 --- a/daylight/workflow.js +++ b/daylight/workflow.js @@ -52,10 +52,9 @@ export default { { "type": "ImportOsmOsc", "file": "data/roads.osc.gz", - "cache": "cache/", "compression": "gzip", "database": config.database, - "srid": 3857 + "databaseSrid": 3857 }, { "type": "DownloadUrl", @@ -65,10 +64,9 @@ export default { { "type": "ImportOsmOsc", "file": "data/admin.osc.gz", - "cache": "cache/", "compression": "gzip", "database": config.database, - "srid": 3857 + "databaseSrid": 3857 }, { "type": "DownloadUrl", From b1de4f01392766422ead3a68094e0980718d93b5 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sun, 19 Nov 2023 00:24:55 +0100 Subject: [PATCH 16/21] Fix documentation and constructors --- .run/basemap-workflow.run.xml | 5 ++ .../org/apache/baremaps/workflow/Task.java | 23 ++++++++ .../baremaps/workflow/WorkflowContext.java | 16 ++--- .../workflow/tasks/CleanContextCache.java | 11 ++++ .../workflow/tasks/CleanContextData.java | 8 +++ .../tasks/CreateGeocoderOpenStreetMap.java | 59 ++++--------------- .../workflow/tasks/CreateGeonamesIndex.java | 17 +++++- .../workflow/tasks/CreateIplocIndex.java | 17 ++++++ .../workflow/tasks/DecompressFile.java | 7 +++ .../baremaps/workflow/tasks/DownloadUrl.java | 9 ++- .../workflow/tasks/ExecuteCommand.java | 7 +++ .../baremaps/workflow/tasks/ExecuteSql.java | 7 +++ .../workflow/tasks/ExecuteSqlScript.java | 7 +++ .../workflow/tasks/ExportVectorTiles.java | 21 ++++++- .../tasks/ImportDaylightFeatures.java | 7 +++ .../tasks/ImportDaylightTranslations.java | 8 ++- .../workflow/tasks/ImportGeoPackage.java | 7 +++ .../baremaps/workflow/tasks/ImportOsmOsc.java | 7 +++ .../baremaps/workflow/tasks/ImportOsmPbf.java | 7 +++ .../workflow/tasks/ImportShapefile.java | 7 +++ .../baremaps/workflow/tasks/LogMessage.java | 7 +++ .../workflow/tasks/UpdateOsmDatabase.java | 7 +++ basemap/import.js | 30 ---------- 23 files changed, 213 insertions(+), 88 deletions(-) diff --git a/.run/basemap-workflow.run.xml b/.run/basemap-workflow.run.xml index 663c874a9..c10d068a7 100644 --- a/.run/basemap-workflow.run.xml +++ b/.run/basemap-workflow.run.xml @@ -4,6 +4,11 @@ - - org.glassfish.hk2 - hk2-api - - - org.glassfish.hk2 - hk2-locator - org.graalvm.js js diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java index b759c4d55..8ac8b76c2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -113,7 +113,8 @@ protected static void decompressBzip2(Path source, Path target) throws IOExcepti * @throws IOException if an I/O error occurs */ protected static void decompressGzip(Path source, Path target) throws IOException { - try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(source)))) { + try (var bufferedInputStream = new BufferedInputStream(Files.newInputStream(source)); + var zis = new GZIPInputStream(bufferedInputStream)) { Files.copy(zis, target, StandardCopyOption.REPLACE_EXISTING); } } From 22a39aff80032bab9aa305a73b7c1dec1a72f45e Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Wed, 13 Dec 2023 08:32:18 +0100 Subject: [PATCH 20/21] Fix test script --- scripts/test-basemap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-basemap.sh b/scripts/test-basemap.sh index d9ae8a7f9..46554bccc 100755 --- a/scripts/test-basemap.sh +++ b/scripts/test-basemap.sh @@ -29,7 +29,7 @@ echo "" rm -fr data tiles tiles.mbtiles -baremaps workflow execute --file workflow.js +baremaps workflow execute --file import.js echo "" echo "--------------------------------------------------------------------" From 1d86ee96e9fb0058fd694fa75d0da1cd6a6f3f5e Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Wed, 13 Dec 2023 11:33:25 +0100 Subject: [PATCH 21/21] Minor refactoring --- .../baremaps/workflow/WorkflowExecutor.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java index 6d37e270e..36d3c38b9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java @@ -107,21 +107,24 @@ public WorkflowExecutor(Workflow workflow, ExecutorService executorService) { */ public void execute() { try { - // Create futures for each end step - var endSteps = graph.nodes().stream() - .filter(this::isEndStep) - .map(this::getStep) - .toArray(CompletableFuture[]::new); - - // Wait for all the end steps to complete - CompletableFuture.allOf(endSteps).join(); + executeAsync().join(); logStepMeasures(); - } catch (Exception e) { logger.error("Error while executing the workflow", e); } } + public CompletableFuture executeAsync() { + // Create futures for each end step + var endSteps = graph.nodes().stream() + .filter(this::isEndStep) + .map(this::getStep) + .toArray(CompletableFuture[]::new); + + // Wait for all the end steps to complete + return CompletableFuture.allOf(endSteps); + } + /** * Returns the future step associated to the step id. If the future step does not exist, it is * created.