Skip to content

Commit

Permalink
Refactor and clean the storage package (#857)
Browse files Browse the repository at this point in the history 
- The DataFrame class becomes the DataTable class
- The data.schema package becomes the data.storage package
- The method and variable names in the unit tests have been improved
- The overall architecture is documented in the README.md of the module
  • Loading branch information
@bchapuis
bchapuis authored Jun 1, 2024
1 parent db7ef92 commit ee7aed7
Show file tree
Hide file tree
Showing 46 changed files with 539 additions and 476 deletions.
38 changes: 19 additions & 19 deletions ...rage/flatgeobuf/FlatGeoBufDataSchema.java → ...orage/flatgeobuf/FlatGeoBufDataStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,41 +21,41 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.apache.baremaps.data.schema.DataSchema;
import org.apache.baremaps.data.schema.DataTable;
import org.apache.baremaps.data.schema.DataTableException;
import org.apache.baremaps.data.storage.DataStore;
import org.apache.baremaps.data.storage.DataStoreException;
import org.apache.baremaps.data.storage.DataTable;

/**
* A schema corresponding to the flatgeobuf files of a directory.
* A {@link DataStore} corresponding to the flatgeobuf files of a directory.
*/
public class FlatGeoBufDataSchema implements DataSchema {
public class FlatGeoBufDataStore implements DataStore {

private final Path directory;

public FlatGeoBufDataSchema(Path directory) {
public FlatGeoBufDataStore(Path directory) {
this.directory = directory;
}

/**
* {@inheritDoc}
*/
@Override
public List<String> list() throws DataTableException {
public List<String> list() throws DataStoreException {
try (var files = Files.list(directory)) {
return files
.filter(file -> file.toString().toLowerCase().endsWith(".fgb"))
.map(file -> file.getFileName().toString())
.toList();
} catch (IOException e) {
throw new DataTableException(e);
throw new DataStoreException(e);
}
}

/**
* {@inheritDoc}
*/
@Override
public DataTable get(String name) throws DataTableException {
public DataTable get(String name) throws DataStoreException {
var path = directory.resolve(name);
return new FlatGeoBufDataTable(path);
}
Expand All @@ -64,39 +64,39 @@ public DataTable get(String name) throws DataTableException {
* {@inheritDoc}
*/
@Override
public void add(DataTable table) throws DataTableException {
var filename = table.rowType().name();
filename = filename.endsWith(".fgb") ? filename : filename + ".fgb";
add(filename, table);
public void add(DataTable table) throws DataStoreException {
var fileName = table.schema().name();
fileName = fileName.endsWith(".fgb") ? fileName : fileName + ".fgb";
add(fileName, table);
}

@Override
public void add(String name, DataTable table) throws DataTableException {
public void add(String name, DataTable table) throws DataStoreException {
var path = directory.resolve(name);
try {
Files.deleteIfExists(path);
Files.createFile(path);
var flatGeoBufTable = new FlatGeoBufDataTable(path, table.rowType());
var flatGeoBufTable = new FlatGeoBufDataTable(path, table.schema());
flatGeoBufTable.write(table);
} catch (IOException e) {
throw new DataTableException(e);
throw new DataStoreException(e);
}
}

/**
* {@inheritDoc}
*/
@Override
public void remove(String name) throws DataTableException {
public void remove(String name) throws DataStoreException {
var path = directory.resolve(name);
if (name.equals(path.getFileName().toString())) {
try {
Files.delete(path);
} catch (IOException e) {
throw new DataTableException(e);
throw new DataStoreException(e);
}
} else {
throw new DataTableException("Table not found");
throw new DataStoreException("Table not found");
}
}
}
66 changes: 32 additions & 34 deletions baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.apache.baremaps.data.collection.DataCollection;
import org.apache.baremaps.data.schema.DataRow;
import org.apache.baremaps.data.schema.DataRowType;
import org.apache.baremaps.data.schema.DataTable;
import org.apache.baremaps.data.storage.DataRow;
import org.apache.baremaps.data.storage.DataSchema;
import org.apache.baremaps.data.storage.DataTable;
import org.locationtech.jts.geom.*;
import org.wololo.flatgeobuf.Constants;
import org.wololo.flatgeobuf.GeometryConversions;
Expand All @@ -39,13 +39,13 @@
import org.wololo.flatgeobuf.generated.GeometryType;

/**
* A table that stores rows in a flatgeobuf file.
* A {@link DataTable} that stores rows in a flatgeobuf file.
*/
public class FlatGeoBufDataTable implements DataTable {

private final Path file;

private DataRowType rowType;
private DataSchema schema;

/**
* Constructs a table from a flatgeobuf file (used for reading).
Expand All @@ -54,40 +54,38 @@ public class FlatGeoBufDataTable implements DataTable {
*/
public FlatGeoBufDataTable(Path file) {
this.file = file;
this.rowType = readRowType(file);
this.schema = readSchema(file);
}


private static DataSchema readSchema(Path file) {
try (var channel = FileChannel.open(file, StandardOpenOption.READ)) {
// try to read the schema from the file
var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN);
HeaderMeta headerMeta = readHeaderMeta(channel, buffer);
return FlatGeoBufTypeConversion.asSchema(headerMeta);
} catch (IOException e) {
return null;
}
}

/**
* Constructs a table from a flatgeobuf file and a row type (used for writing).
* Constructs a table from a flatgeobuf file and a schema (used for writing).
*
* @param file the path to the flatgeobuf file
* @param rowType the row type of the table
* @param schema the schema of the table
*/
public FlatGeoBufDataTable(Path file, DataRowType rowType) {
public FlatGeoBufDataTable(Path file, DataSchema schema) {
this.file = file;
this.rowType = rowType;
this.schema = schema;
}

/**
* {@inheritDoc}
*/
@Override
public DataRowType rowType() {
return rowType;
}

/**
* {@inheritDoc}
*/
public static DataRowType readRowType(Path file) {
try (var channel = FileChannel.open(file, StandardOpenOption.READ)) {
// try to read the row type from the file
var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN);
HeaderMeta headerMeta = readHeaderMeta(channel, buffer);
return FlatGeoBufTypeConversion.asRowType(headerMeta);
} catch (IOException e) {
return null;
}
public DataSchema schema() {
return schema;
}

/**
Expand All @@ -111,7 +109,7 @@ public Iterator<DataRow> iterator() {
buffer.clear();

// create the feature stream
return new RowIterator(channel, headerMeta, rowType, buffer);
return new RowIterator(channel, headerMeta, schema, buffer);
} catch (IOException e) {
throw new RuntimeException(e);
}
Expand Down Expand Up @@ -173,8 +171,8 @@ public void write(DataCollection<DataRow> features) throws IOException {
headerMeta.indexNodeSize = 16;
headerMeta.srid = 3857;
headerMeta.featuresCount = features.size();
headerMeta.name = rowType.name();
headerMeta.columns = FlatGeoBufTypeConversion.asColumns(rowType.columns());
headerMeta.name = schema.name();
headerMeta.columns = FlatGeoBufTypeConversion.asColumns(schema.columns());
HeaderMeta.write(headerMeta, outputStream, bufferBuilder);

var indexSize =
Expand Down Expand Up @@ -235,7 +233,7 @@ public static class RowIterator implements Iterator<DataRow> {

private final HeaderMeta headerMeta;

private final DataRowType rowType;
private final DataSchema schema;

private final SeekableByteChannel channel;

Expand All @@ -248,14 +246,14 @@ public static class RowIterator implements Iterator<DataRow> {
*
* @param channel the channel to read from
* @param headerMeta the header meta
* @param rowType the row type of the table
* @param schema the schema of the table
* @param buffer the buffer to use
*/
public RowIterator(SeekableByteChannel channel, HeaderMeta headerMeta,
DataRowType rowType, ByteBuffer buffer) {
DataSchema schema, ByteBuffer buffer) {
this.channel = channel;
this.headerMeta = headerMeta;
this.rowType = rowType;
this.schema = schema;
this.buffer = buffer;
}

Expand All @@ -278,7 +276,7 @@ public DataRow next() {

var featureSize = buffer.getInt();
var row =
FlatGeoBufTypeConversion.asRow(headerMeta, rowType, Feature.getRootAsFeature(buffer));
FlatGeoBufTypeConversion.asRow(headerMeta, schema, Feature.getRootAsFeature(buffer));

buffer.position(Integer.BYTES + featureSize);
buffer.compact();
Expand Down
10 changes: 5 additions & 5 deletions ...s-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.baremaps.data.schema.*;
import org.apache.baremaps.data.schema.DataColumn.Type;
import org.apache.baremaps.data.storage.*;
import org.apache.baremaps.data.storage.DataColumn.Type;
import org.wololo.flatgeobuf.ColumnMeta;
import org.wololo.flatgeobuf.GeometryConversions;
import org.wololo.flatgeobuf.HeaderMeta;
Expand All @@ -50,16 +50,16 @@ public class FlatGeoBufTypeConversion {
types.put(Type.STRING, ColumnType.String);
}

public static DataRowType asRowType(HeaderMeta headerMeta) {
public static DataSchema asSchema(HeaderMeta headerMeta) {
var name = headerMeta.name;
var columns = headerMeta.columns.stream()
.map(column -> new DataColumnImpl(column.name, Type.fromBinding(column.getBinding())))
.map(DataColumn.class::cast)
.toList();
return new DataRowTypeImpl(name, columns);
return new DataSchemaImpl(name, columns);
}

public static DataRow asRow(HeaderMeta headerMeta, DataRowType dataType, Feature feature) {
public static DataRow asRow(HeaderMeta headerMeta, DataSchema dataType, Feature feature) {
var values = new ArrayList();

var geometryBuffer = feature.geometry();
Expand Down
26 changes: 13 additions & 13 deletions ...rage/geopackage/GeoPackageDataSchema.java → ...orage/geopackage/GeoPackageDataStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,23 @@
import java.util.List;
import mil.nga.geopackage.GeoPackage;
import mil.nga.geopackage.GeoPackageManager;
import org.apache.baremaps.data.schema.DataSchema;
import org.apache.baremaps.data.schema.DataTable;
import org.apache.baremaps.data.schema.DataTableException;
import org.apache.baremaps.data.storage.DataStore;
import org.apache.baremaps.data.storage.DataStoreException;
import org.apache.baremaps.data.storage.DataTable;

/**
* A schema corresponding to a GeoPackage database.
* A {@link DataStore} corresponding to a GeoPackage file.
*/
public class GeoPackageDataSchema implements DataSchema, AutoCloseable {
public class GeoPackageDataStore implements DataStore, AutoCloseable {

private final GeoPackage geoPackage;

/**
* Constructs a schema from a GeoPackage database.
* Constructs a {@link GeoPackageDataStore} from a GeoPackage file.
*
* @param file the path to the GeoPackage database
* @param file the path to the GeoPackage file
*/
public GeoPackageDataSchema(Path file) {
public GeoPackageDataStore(Path file) {
this.geoPackage = GeoPackageManager.open(file.toFile());
}

Expand All @@ -54,36 +54,36 @@ public void close() throws Exception {
* {@inheritDoc}
*/
@Override
public List<String> list() throws DataTableException {
public List<String> list() throws DataStoreException {
return geoPackage.getFeatureTables();
}

/**
* {@inheritDoc}
*/
@Override
public DataTable get(String name) throws DataTableException {
public DataTable get(String name) throws DataStoreException {
return new GeoPackageDataTable(geoPackage.getFeatureDao(name));
}

/**
* {@inheritDoc}
*/
@Override
public void add(DataTable table) throws DataTableException {
public void add(DataTable table) throws DataStoreException {
throw new UnsupportedOperationException();
}

@Override
public void add(String name, DataTable table) throws DataTableException {
public void add(String name, DataTable table) throws DataStoreException {
throw new UnsupportedOperationException();
}

/**
* {@inheritDoc}
*/
@Override
public void remove(String name) throws DataTableException {
public void remove(String name) throws DataStoreException {
throw new UnsupportedOperationException();
}
}
Loading

0 comments on commit ee7aed7

Please sign in to comment.