Add rpsl data store and simplify package structure

apache · Nov 8, 2024 · e35ec06 · e35ec06
1 parent 21d67cf
commit e35ec06
Show file tree

Hide file tree

Showing 165 changed files with 720 additions and 419 deletions.
diff --git a/...king/src/main/java/org/apache/baremaps/benchmarking/geoparquet/OvertureMapsBenchmark.java b/...king/src/main/java/org/apache/baremaps/benchmarking/geoparquet/OvertureMapsBenchmark.java
@@ -21,7 +21,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.concurrent.TimeUnit;
-import org.apache.baremaps.geoparquet.format.GeoParquetReader;
+import org.apache.baremaps.geoparquet.GeoParquetReader;
 import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider;
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.runner.Runner;

diff --git a/...marking/src/main/java/org/apache/baremaps/benchmarking/geoparquet/SmallFileBenchmark.java b/...marking/src/main/java/org/apache/baremaps/benchmarking/geoparquet/SmallFileBenchmark.java
@@ -22,7 +22,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.concurrent.TimeUnit;
-import org.apache.baremaps.geoparquet.format.GeoParquetReader;
+import org.apache.baremaps.geoparquet.GeoParquetReader;
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.runner.Runner;
 import org.openjdk.jmh.runner.RunnerException;

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/VectorTileContours.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/VectorTileContours.java
@@ -24,8 +24,8 @@
 import java.util.concurrent.Callable;
 import org.apache.baremaps.maplibre.tileset.Tileset;
 import org.apache.baremaps.maplibre.tileset.TilesetLayer;
-import org.apache.baremaps.openstreetmap.format.stream.ProgressLogger;
-import org.apache.baremaps.openstreetmap.format.stream.StreamUtils;
+import org.apache.baremaps.openstreetmap.stream.ProgressLogger;
+import org.apache.baremaps.openstreetmap.stream.StreamUtils;
 import org.apache.baremaps.tasks.ExportVectorTiles;
 import org.apache.baremaps.tilestore.TileCoord;
 import org.apache.baremaps.tilestore.TileEntry;

diff --git a/...src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapDocumentMapper.java b/...src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapDocumentMapper.java
@@ -20,8 +20,8 @@
 
 
 import java.util.function.Function;
-import org.apache.baremaps.openstreetmap.format.model.Element;
-import org.apache.baremaps.openstreetmap.format.model.Node;
+import org.apache.baremaps.openstreetmap.model.Element;
+import org.apache.baremaps.openstreetmap.model.Node;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.LatLonShape;

diff --git a/...src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapEntityConsumer.java b/...src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapEntityConsumer.java
@@ -18,8 +18,8 @@
 package org.apache.baremaps.geocoder.openstreetmap;
 
 import java.util.function.Consumer;
-import org.apache.baremaps.openstreetmap.format.model.Element;
-import org.apache.baremaps.openstreetmap.format.model.Entity;
+import org.apache.baremaps.openstreetmap.model.Element;
+import org.apache.baremaps.openstreetmap.model.Entity;
 import org.apache.lucene.index.IndexWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java
@@ -85,19 +85,22 @@ public Optional<IpLocObject> apply(RpslObject nicObject) {
       var attributes = nicObject.asMap();
 
       // Use a default name if there is no netname
-      var network = attributes.getOrDefault("netname", "unknown");
+      var network = String.join(", ", attributes.getOrDefault("netname", List.of("unknown")));
+      var geoloc = String.join(", ", attributes.getOrDefault("geoloc", List.of()));
+      var country = String.join(", ", attributes.getOrDefault("country", List.of()));
+      var source = String.join(", ", attributes.getOrDefault("source", List.of()));
 
       // If there is a geoloc field, we use the latitude and longitude provided
       if (attributes.containsKey("geoloc")) {
-        var location = stringToCoordinate(attributes.get("geoloc"));
+        var location = stringToCoordinate(geoloc);
         if (location.isPresent()) {
           return Optional.of(new IpLocObject(
-              attributes.get("geoloc"),
+              geoloc,
               inetRange,
               location.get(),
               network,
-              attributes.get("country"),
-              attributes.get("source"),
+              country,
+              source,
               IpLocPrecision.GEOLOC));
         }
       }
@@ -112,36 +115,37 @@ public Optional<IpLocObject> apply(RpslObject nicObject) {
         // build a query text string out of the cherry-picked fields
         var queryTextBuilder = new StringBuilder();
         for (String field : searchedFields) {
-          if (!Strings.isNullOrEmpty(attributes.get(field))) {
+          var fieldValue = String.join(", ", attributes.get(field));
+          if (!Strings.isNullOrEmpty(fieldValue)) {
             queryTextBuilder.append(attributes.get(field)).append(" ");
           }
         }
 
         String queryText = queryTextBuilder.toString();
-        var location = findLocationInCountry(queryText, attributes.get("country"));
+        var location = findLocationInCountry(queryText, country);
         if (location.isPresent()) {
           return Optional.of(new IpLocObject(
               queryText,
               inetRange,
               location.get(),
               network,
-              attributes.get("country"),
-              attributes.get("source"),
+              country,
+              source,
               IpLocPrecision.GEOCODER));
         }
       }
 
       // If there is a country get the location of country
       if (attributes.containsKey("country")) {
-        var location = findCountryLocation(attributes.get("country"));
+        var location = findCountryLocation(country);
         if (location.isPresent()) {
           return Optional.of(new IpLocObject(
-              attributes.get("country"),
+              country,
               inetRange,
               location.get(),
               network,
-              attributes.get("country"),
-              attributes.get("source"),
+              country,
+              source,
               IpLocPrecision.COUNTRY));
         }
       }
@@ -152,7 +156,7 @@ public Optional<IpLocObject> apply(RpslObject nicObject) {
           new Coordinate(),
           network,
           null,
-          attributes.get("source"),
+          source,
           IpLocPrecision.WORLD));
     } catch (Exception e) {
       logger.warn("Error while mapping nic object to ip loc object", e);

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocRepository.java
@@ -28,7 +28,7 @@
 import java.util.List;
 import java.util.stream.Stream;
 import javax.sql.DataSource;
-import org.apache.baremaps.openstreetmap.format.stream.StreamUtils;
+import org.apache.baremaps.openstreetmap.stream.StreamUtils;
 import org.locationtech.jts.geom.Coordinate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/CreateGeocoderOpenStreetMap.java
@@ -25,8 +25,8 @@
 import java.util.StringJoiner;
 import org.apache.baremaps.geocoder.GeocoderConstants;
 import org.apache.baremaps.geocoder.openstreetmap.OpenStreetMapEntityConsumer;
-import org.apache.baremaps.openstreetmap.format.pbf.PbfEntityReader;
-import org.apache.baremaps.openstreetmap.format.stream.StreamUtils;
+import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
+import org.apache.baremaps.openstreetmap.stream.StreamUtils;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;
 import org.apache.lucene.index.IndexWriter;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/CreateIplocIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/CreateIplocIndex.java
@@ -26,7 +26,7 @@
 import java.util.StringJoiner;
 import org.apache.baremaps.iploc.IpLocReader;
 import org.apache.baremaps.iploc.IpLocRepository;
-import org.apache.baremaps.openstreetmap.format.stream.StreamException;
+import org.apache.baremaps.openstreetmap.stream.StreamException;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;
 import org.apache.lucene.search.SearcherFactory;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/DiffService.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/DiffService.java
@@ -17,7 +17,7 @@
 
 package org.apache.baremaps.tasks;
 
-import static org.apache.baremaps.openstreetmap.format.stream.ConsumerUtils.consumeThenReturn;
+import static org.apache.baremaps.openstreetmap.stream.ConsumerUtils.consumeThenReturn;
 
 import java.io.BufferedInputStream;
 import java.net.MalformedURLException;
@@ -28,12 +28,12 @@
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import java.util.zip.GZIPInputStream;
-import org.apache.baremaps.openstreetmap.format.function.EntityGeometryBuilder;
-import org.apache.baremaps.openstreetmap.format.function.EntityToGeometryMapper;
-import org.apache.baremaps.openstreetmap.format.function.ProjectionTransformer;
-import org.apache.baremaps.openstreetmap.format.model.*;
-import org.apache.baremaps.openstreetmap.format.stream.StreamException;
-import org.apache.baremaps.openstreetmap.format.xml.XmlChangeReader;
+import org.apache.baremaps.openstreetmap.function.EntityGeometryBuilder;
+import org.apache.baremaps.openstreetmap.function.EntityToGeometryMapper;
+import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
+import org.apache.baremaps.openstreetmap.model.*;
+import org.apache.baremaps.openstreetmap.stream.StreamException;
+import org.apache.baremaps.openstreetmap.xml.XmlChangeReader;
 import org.apache.baremaps.postgres.openstreetmap.HeaderRepository;
 import org.apache.baremaps.postgres.openstreetmap.Repository;
 import org.apache.baremaps.tilestore.TileCoord;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ExportVectorTiles.java
@@ -32,8 +32,8 @@
 import org.apache.baremaps.maplibre.style.Style;
 import org.apache.baremaps.maplibre.tileset.Tileset;
 import org.apache.baremaps.maplibre.tileset.TilesetQuery;
-import org.apache.baremaps.openstreetmap.format.stream.ProgressLogger;
-import org.apache.baremaps.openstreetmap.format.stream.StreamUtils;
+import org.apache.baremaps.openstreetmap.stream.ProgressLogger;
+import org.apache.baremaps.openstreetmap.stream.StreamUtils;
 import org.apache.baremaps.tilestore.*;
 import org.apache.baremaps.tilestore.file.FileTileStore;
 import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportDaylightFeatures.java
@@ -24,7 +24,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.StringJoiner;
-import org.apache.baremaps.openstreetmap.format.model.Element;
+import org.apache.baremaps.openstreetmap.model.Element;
 import org.apache.baremaps.postgres.openstreetmap.*;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportDaylightTranslations.java
@@ -23,7 +23,7 @@
 import java.util.List;
 import java.util.StringJoiner;
 import java.util.stream.Collectors;
-import org.apache.baremaps.openstreetmap.format.model.Element;
+import org.apache.baremaps.openstreetmap.model.Element;
 import org.apache.baremaps.postgres.openstreetmap.*;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoPackage.java
@@ -19,8 +19,8 @@
 
 import java.nio.file.Path;
 import java.util.StringJoiner;
-import org.apache.baremaps.geopackage.store.GeoPackageDataStore;
-import org.apache.baremaps.openstreetmap.format.function.ProjectionTransformer;
+import org.apache.baremaps.geopackage.GeoPackageDataStore;
+import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
 import org.apache.baremaps.postgres.store.PostgresDataStore;
 import org.apache.baremaps.store.DataTableGeometryMapper;
 import org.apache.baremaps.store.DataTableMapper;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoParquet.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoParquet.java
@@ -19,9 +19,9 @@
 
 import java.net.URI;
 import java.util.StringJoiner;
-import org.apache.baremaps.geoparquet.store.GeoParquetDataStore;
-import org.apache.baremaps.geoparquet.store.GeoParquetDataTable;
-import org.apache.baremaps.openstreetmap.format.function.ProjectionTransformer;
+import org.apache.baremaps.geoparquet.GeoParquetDataStore;
+import org.apache.baremaps.geoparquet.GeoParquetDataTable;
+import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
 import org.apache.baremaps.postgres.store.PostgresDataStore;
 import org.apache.baremaps.store.DataTableGeometryMapper;
 import org.apache.baremaps.store.DataTableMapper;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportOsmOsc.java
@@ -17,14 +17,14 @@
 
 package org.apache.baremaps.tasks;
 
-import static org.apache.baremaps.openstreetmap.format.stream.ConsumerUtils.consumeThenReturn;
+import static org.apache.baremaps.openstreetmap.stream.ConsumerUtils.consumeThenReturn;
 
 import java.io.BufferedInputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.StringJoiner;
-import org.apache.baremaps.openstreetmap.format.function.*;
-import org.apache.baremaps.openstreetmap.format.xml.XmlChangeReader;
+import org.apache.baremaps.openstreetmap.function.*;
+import org.apache.baremaps.openstreetmap.xml.XmlChangeReader;
 import org.apache.baremaps.postgres.openstreetmap.CopyChangeImporter;
 import org.apache.baremaps.postgres.openstreetmap.NodeRepository;
 import org.apache.baremaps.postgres.openstreetmap.RelationRepository;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportOsmPbf.java
@@ -25,11 +25,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.StringJoiner;
-import org.apache.baremaps.openstreetmap.format.model.Node;
-import org.apache.baremaps.openstreetmap.format.model.Relation;
-import org.apache.baremaps.openstreetmap.format.model.Way;
-import org.apache.baremaps.openstreetmap.format.pbf.PbfBlockReader;
-import org.apache.baremaps.openstreetmap.format.stream.StreamUtils;
+import org.apache.baremaps.openstreetmap.model.Node;
+import org.apache.baremaps.openstreetmap.model.Relation;
+import org.apache.baremaps.openstreetmap.model.Way;
+import org.apache.baremaps.openstreetmap.pbf.PbfBlockReader;
+import org.apache.baremaps.openstreetmap.stream.StreamUtils;
 import org.apache.baremaps.postgres.openstreetmap.*;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportShapefile.java
@@ -19,9 +19,9 @@
 
 import java.nio.file.Path;
 import java.util.StringJoiner;
-import org.apache.baremaps.openstreetmap.format.function.ProjectionTransformer;
+import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
 import org.apache.baremaps.postgres.store.PostgresDataStore;
-import org.apache.baremaps.shapefile.store.ShapefileDataTable;
+import org.apache.baremaps.shapefile.ShapefileDataTable;
 import org.apache.baremaps.store.DataTableGeometryMapper;
 import org.apache.baremaps.store.DataTableMapper;
 import org.apache.baremaps.workflow.Task;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/UpdateOsmDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/UpdateOsmDatabase.java
@@ -23,13 +23,13 @@
 import java.util.Map;
 import java.util.StringJoiner;
 import java.util.zip.GZIPInputStream;
-import org.apache.baremaps.openstreetmap.format.function.*;
-import org.apache.baremaps.openstreetmap.format.model.Header;
-import org.apache.baremaps.openstreetmap.format.model.Node;
-import org.apache.baremaps.openstreetmap.format.model.Relation;
-import org.apache.baremaps.openstreetmap.format.model.Way;
-import org.apache.baremaps.openstreetmap.format.state.StateReader;
-import org.apache.baremaps.openstreetmap.format.xml.XmlChangeReader;
+import org.apache.baremaps.openstreetmap.function.*;
+import org.apache.baremaps.openstreetmap.model.Header;
+import org.apache.baremaps.openstreetmap.model.Node;
+import org.apache.baremaps.openstreetmap.model.Relation;
+import org.apache.baremaps.openstreetmap.model.Way;
+import org.apache.baremaps.openstreetmap.state.StateReader;
+import org.apache.baremaps.openstreetmap.xml.XmlChangeReader;
 import org.apache.baremaps.postgres.openstreetmap.*;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;

diff --git a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/DataTableIndexTest.java b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/DataTableIndexTest.java
@@ -24,7 +24,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import org.apache.baremaps.geoparquet.store.GeoParquetDataTable;
+import org.apache.baremaps.geoparquet.GeoParquetDataTable;
 import org.apache.baremaps.testing.TestFiles;
 import org.apache.baremaps.utils.FileUtils;
 import org.apache.lucene.document.Document;

diff --git a/baremaps-core/src/test/java/org/apache/baremaps/integration/GeoPackageToPostgresTest.java b/baremaps-core/src/test/java/org/apache/baremaps/integration/GeoPackageToPostgresTest.java
@@ -17,7 +17,7 @@
 
 package org.apache.baremaps.integration;
 
-import org.apache.baremaps.geopackage.store.GeoPackageDataStore;
+import org.apache.baremaps.geopackage.GeoPackageDataStore;
 import org.apache.baremaps.postgres.store.PostgresDataStore;
 import org.apache.baremaps.testing.PostgresContainerTest;
 import org.apache.baremaps.testing.TestFiles;

diff --git a/baremaps-core/src/test/java/org/apache/baremaps/integration/GeoParquetToPostgresTest.java b/baremaps-core/src/test/java/org/apache/baremaps/integration/GeoParquetToPostgresTest.java
@@ -19,7 +19,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-import org.apache.baremaps.geoparquet.store.GeoParquetDataStore;
+import org.apache.baremaps.geoparquet.GeoParquetDataStore;
 import org.apache.baremaps.postgres.store.PostgresDataStore;
 import org.apache.baremaps.testing.PostgresContainerTest;
 import org.apache.baremaps.testing.TestFiles;

diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tasks/ImportUpdateSampleTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tasks/ImportUpdateSampleTest.java
@@ -31,8 +31,8 @@
 import org.apache.baremaps.data.memory.OnHeapMemory;
 import org.apache.baremaps.data.type.CoordinateDataType;
 import org.apache.baremaps.data.type.LongListDataType;
-import org.apache.baremaps.openstreetmap.format.model.Header;
-import org.apache.baremaps.openstreetmap.format.state.StateReader;
+import org.apache.baremaps.openstreetmap.model.Header;
+import org.apache.baremaps.openstreetmap.state.StateReader;
 import org.apache.baremaps.postgres.openstreetmap.*;
 import org.apache.baremaps.testing.PostgresRepositoryTest;
 import org.apache.baremaps.testing.TestFiles;

diff --git a/baremaps-core/src/test/java/org/apache/baremaps/utils/ProjectionTransformerTest.java b/baremaps-core/src/test/java/org/apache/baremaps/utils/ProjectionTransformerTest.java
@@ -19,7 +19,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-import org.apache.baremaps.openstreetmap.format.function.ProjectionTransformer;
+import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
 import org.junit.jupiter.api.Test;
 import org.locationtech.jts.geom.Coordinate;
 import org.locationtech.jts.geom.GeometryFactory;

diff --git a/...ache/baremaps/csv/store/CsvDataStore.java → ...org/apache/baremaps/csv/CsvDataStore.java b/...ache/baremaps/csv/store/CsvDataStore.java → ...org/apache/baremaps/csv/CsvDataStore.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.baremaps.csv.store;
+package org.apache.baremaps.csv;
 
 import java.io.File;
 import java.io.IOException;

diff --git a/...ache/baremaps/csv/store/CsvDataTable.java → ...org/apache/baremaps/csv/CsvDataTable.java b/...ache/baremaps/csv/store/CsvDataTable.java → ...org/apache/baremaps/csv/CsvDataTable.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.baremaps.csv.store;
+package org.apache.baremaps.csv;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;