Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PARQUET-2417: Add support for geometry logical type #2971

Open
wants to merge 46 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
6a2e051
PARQUET-2471: Add support for geometry logical type
zhangfengcdt Jul 23, 2024
d354012
fix types
zhangfengcdt Jul 23, 2024
969b696
Refactor BoundingBox and GeometryTypes
zhangfengcdt Jul 23, 2024
87ee8ea
revert naming changes
zhangfengcdt Jul 24, 2024
d67e03b
revert TestDecimalUtils
zhangfengcdt Jul 24, 2024
ccf1c4a
revert more
zhangfengcdt Jul 24, 2024
35342c2
refactor statistics
zhangfengcdt Jul 24, 2024
bcfefb7
modify EnvelopeCovering
zhangfengcdt Jul 24, 2024
cf615f6
add more unit tests
zhangfengcdt Jul 25, 2024
c6ae733
update comments
zhangfengcdt Jul 25, 2024
80a629e
add comment for envelope converging expand calculation
zhangfengcdt Jul 26, 2024
e0ec9ef
Fix the boundingbox initial values in constructor
zhangfengcdt Jul 29, 2024
7c728b1
Update the poc implementation for the changes to the spec
zhangfengcdt Aug 7, 2024
fa36d06
remove print
zhangfengcdt Aug 7, 2024
2a1c62a
implement evelop covering for spherical coordinates
zhangfengcdt Aug 8, 2024
d0e7d3d
throw a not-implemented exception for the covering statistics when th…
zhangfengcdt Aug 12, 2024
30d64be
Merge branch 'master' of github.com:apache/parquet-java into feature-…
zhangfengcdt Aug 12, 2024
29a86b5
remove unused comment codes
zhangfengcdt Aug 12, 2024
0c4b8b4
address some review comments
zhangfengcdt Aug 19, 2024
a56e9ad
revert changes that are not desired
zhangfengcdt Aug 20, 2024
1ae3d99
refactor toString and remove test scope of jts-core in parquet-hadoop…
zhangfengcdt Aug 20, 2024
198642d
refactor the converings to be map to avoid ordering issues in stats m…
zhangfengcdt Aug 23, 2024
0cf2de9
address review comments
zhangfengcdt Sep 13, 2024
4f0f7ed
fix formating
zhangfengcdt Sep 13, 2024
a378a9a
address comments to remove string parsing (to be consistent with spec)
zhangfengcdt Sep 23, 2024
698325a
update according to the changes to the upstream pqrquet-format pr
zhangfengcdt Oct 17, 2024
d65ba8e
remove coverings
zhangfengcdt Oct 17, 2024
1b0f5b9
add GeometryStatistics to ColumnMetaData
zhangfengcdt Oct 17, 2024
342e400
more code cleanup for covering
zhangfengcdt Oct 17, 2024
3cc9b68
add toParquetGeometryStatistics
zhangfengcdt Oct 18, 2024
dc05cfd
fix check errors
zhangfengcdt Oct 18, 2024
6f1d586
Merge branch 'master' of github.com:apache/parquet-java into feature-…
zhangfengcdt Nov 13, 2024
e4e3cae
Merge branch 'master' of github.com:apache/parquet-java into feature-…
zhangfengcdt Feb 7, 2025
69d950f
change and remove the encoding and edges from geometry type (spec cha…
zhangfengcdt Feb 7, 2025
d296c6a
fix unit tests
zhangfengcdt Feb 7, 2025
93e28b7
handle the wraparound case for X values
zhangfengcdt Feb 8, 2025
e688d05
support GEOGRAPHY type
zhangfengcdt Feb 8, 2025
01ac560
revert import changes
zhangfengcdt Feb 8, 2025
f9585cd
address pr review comments
zhangfengcdt Feb 19, 2025
b805cf4
fix formatting issue
zhangfengcdt Feb 19, 2025
6aa7028
refactor geography logic type
zhangfengcdt Feb 19, 2025
4536e5f
revert the edge algorithm to use string to avoid loop dependency
zhangfengcdt Feb 19, 2025
f188341
add enum EdgeInterpolationAlgorithm for the geography edge algorithm
zhangfengcdt Feb 21, 2025
214fd39
address pr comments: rename, hash, and equal
zhangfengcdt Feb 21, 2025
907acce
remove column index changes
zhangfengcdt Feb 21, 2025
ef83a2e
revert unnecessary changes
zhangfengcdt Feb 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update the poc implementation for the changes to the spec
zhangfengcdt committed Aug 7, 2024
commit 7c728b109aeebf472bce507c964a55d267f05717
Original file line number Diff line number Diff line change
@@ -62,6 +62,7 @@ private BinaryStatistics(BinaryStatistics other) {
if (other.geometryStatistics != null) {
geometryStatistics = other.geometryStatistics.copy();
}
System.out.println("BinaryStatistics.BinaryStatistics");
}

@Override
Original file line number Diff line number Diff line change
@@ -20,29 +20,30 @@

import java.nio.ByteBuffer;
import org.apache.parquet.Preconditions;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKBReader;

public class Covering {
public static final String DEFAULT_COVERING_KIND = "WKB";

protected final LogicalTypeAnnotation.Edges edges;
protected ByteBuffer geometry;
protected final String kind;
protected ByteBuffer value;

public Covering(ByteBuffer geometry, LogicalTypeAnnotation.Edges edges) {
Preconditions.checkArgument(geometry != null, "Geometry cannot be null");
Preconditions.checkArgument(edges != null, "Edges cannot be null");
this.geometry = geometry;
this.edges = edges;
public Covering(ByteBuffer value, String kind) {
Preconditions.checkArgument(kind != null, "kind cannot be null");
Preconditions.checkArgument(kind.equalsIgnoreCase(DEFAULT_COVERING_KIND), "kind only accepts WKB");
Preconditions.checkArgument(value != null, "value cannot be null");
this.value = value;
this.kind = kind;
}

public ByteBuffer getGeometry() {
return geometry;
public ByteBuffer getValue() {
return value;
}

public LogicalTypeAnnotation.Edges getEdges() {
return edges;
public String getKind() {
return kind;
}

void update(Geometry geom) {
@@ -66,18 +67,18 @@ public void abort() {
}

public Covering copy() {
return new Covering(geometry.duplicate(), edges);
return new Covering(value.duplicate(), kind);
}

@Override
public String toString() {
String geomText;
try {
geomText = new WKBReader().read(geometry.array()).toText();
geomText = new WKBReader().read(value.array()).toText();
} catch (ParseException e) {
geomText = "Invalid Geometry";
}

return "Covering{" + "geometry=" + geomText + ", edges=" + edges + '}';
return "Covering{" + "geometry=" + geomText + ", kind=" + kind + '}';
}
}
Original file line number Diff line number Diff line change
@@ -19,7 +19,6 @@
package org.apache.parquet.column.statistics.geometry;

import java.nio.ByteBuffer;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
@@ -36,7 +35,7 @@ public class EnvelopeCovering extends Covering {
private final GeometryFactory factory = new GeometryFactory();

public EnvelopeCovering() {
super(EMPTY, LogicalTypeAnnotation.Edges.SPHERICAL);
super(EMPTY, DEFAULT_COVERING_KIND);
}

@Override
@@ -45,8 +44,8 @@ void update(Geometry geom) {
return;
}
try {
if (geometry != EMPTY) {
Geometry existingGeometry = reader.read(geometry.array());
if (value != EMPTY) {
Geometry existingGeometry = reader.read(value.array());
Envelope existingEnvelope = existingGeometry.getEnvelopeInternal();
Envelope newEnvelope = geom.getEnvelopeInternal();

@@ -62,19 +61,19 @@ void update(Geometry geom) {

Geometry envelopePolygon = createPolygonFromEnvelope(existingEnvelope);

geometry = ByteBuffer.wrap(writer.write(envelopePolygon));
value = ByteBuffer.wrap(writer.write(envelopePolygon));
} else {
Geometry envelopePolygon = createPolygonFromEnvelope(geom.getEnvelopeInternal());
geometry = ByteBuffer.wrap(writer.write(envelopePolygon));
value = ByteBuffer.wrap(writer.write(envelopePolygon));
}
} catch (ParseException e) {
geometry = null;
value = null;
}
}

// Create a polygon from an envelope
// Assume we are using the Standard WKB format, that no Z and M dimension is supported
// https://libgeos.org/specifications/wkb/#standard-wkb
// We only supports WKB as covering kind and WKB polygon can only safely represent
// a covering in 2 dimension.
// Enhancement is to do post POC phase to support Z and M dimension.
private Geometry createPolygonFromEnvelope(Envelope envelope) {
return factory.createPolygon(new Coordinate[] {
@@ -90,9 +89,9 @@ private Geometry createPolygonFromEnvelope(Envelope envelope) {
public void merge(Covering other) {
if (other instanceof EnvelopeCovering) {
try {
update(reader.read(other.geometry.array()));
update(reader.read(other.value.array()));
} catch (ParseException e) {
geometry = null;
value = null;
}
} else {
throw new UnsupportedOperationException("Cannot merge " + this.getClass() + " with "
@@ -102,18 +101,18 @@ public void merge(Covering other) {

@Override
public void reset() {
geometry = EMPTY;
value = EMPTY;
}

@Override
public void abort() {
geometry = null;
value = null;
}

@Override
public EnvelopeCovering copy() {
EnvelopeCovering copy = new EnvelopeCovering();
copy.geometry = geometry == null ? null : ByteBuffer.wrap(geometry.array());
copy.value = value == null ? null : ByteBuffer.wrap(value.array());
return copy;
}
}
Original file line number Diff line number Diff line change
@@ -19,6 +19,8 @@
package org.apache.parquet.column.statistics.geometry;

import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import org.apache.parquet.Preconditions;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.LogicalTypeAnnotation;
@@ -34,7 +36,7 @@ public class GeometryStatistics {
private final ByteBuffer metadata;

private final BoundingBox boundingBox;
private final Covering covering;
private final List<Covering> coverings;
private final GeometryTypes geometryTypes;
private final WKBReader reader = new WKBReader();

@@ -43,26 +45,26 @@ public GeometryStatistics(
String crs,
ByteBuffer metadata,
BoundingBox boundingBox,
Covering covering,
List<Covering> coverings,
GeometryTypes geometryTypes) {
this.edges = edges;
this.crs = crs;
this.metadata = metadata;
this.boundingBox = supportsBoundingBox() ? boundingBox : null;
this.covering = supportsCovering() ? covering : null;
this.coverings = supportsCovering() ? coverings : null;
this.geometryTypes = geometryTypes;
}

public GeometryStatistics(LogicalTypeAnnotation.Edges edges, String crs, ByteBuffer metadata) {
this(edges, crs, metadata, new BoundingBox(), new EnvelopeCovering(), new GeometryTypes());
this(edges, crs, metadata, new BoundingBox(), Arrays.asList(new EnvelopeCovering()), new GeometryTypes());
}

public BoundingBox getBoundingBox() {
return boundingBox;
}

public Covering getCovering() {
return covering;
public List<Covering> getCoverings() {
return coverings;
}

public GeometryTypes getGeometryTypes() {
@@ -86,7 +88,7 @@ private void update(Geometry geom) {
boundingBox.update(geom);
}
if (supportsCovering()) {
covering.update(geom);
coverings.stream().forEach(c -> c.update(geom));
}
geometryTypes.update(geom);
}
@@ -112,20 +114,24 @@ private boolean supportsCovering() {

public void merge(GeometryStatistics other) {
Preconditions.checkArgument(other != null, "Cannot merge with null GeometryStatistics");
Preconditions.checkArgument(coverings.size() == other.coverings.size(), "Coverings size must be the same");

boundingBox.merge(other.boundingBox);
covering.merge(other.covering);
for (int i = 0; i < coverings.size(); i++) {
coverings.get(i).merge(other.coverings.get(i));
}
geometryTypes.merge(other.geometryTypes);
}

public void reset() {
boundingBox.reset();
covering.reset();
coverings.stream().forEach(c -> c.reset());
geometryTypes.reset();
}

public void abort() {
boundingBox.abort();
covering.abort();
coverings.stream().forEach(c -> c.abort());
geometryTypes.abort();
}

@@ -136,15 +142,15 @@ public GeometryStatistics copy() {
crs,
metadata,
boundingBox != null ? boundingBox.copy() : null,
covering != null ? covering.copy() : null,
coverings != null ? coverings : null,
geometryTypes != null ? geometryTypes.copy() : null);
}

@Override
public String toString() {
return "GeometryStatistics{" + "boundingBox="
+ boundingBox + ", covering="
+ covering + ", geometryTypes="
+ boundingBox + ", coverings="
+ coverings + ", geometryTypes="
+ geometryTypes + '}';
}
}
Original file line number Diff line number Diff line change
@@ -158,9 +158,10 @@ protected LogicalTypeAnnotation fromString(List<String> params) {
GeometryEncoding encoding = GeometryEncoding.valueOf(params.get(0));
Edges edges = Edges.valueOf(params.get(1));
String crs = params.size() > 2 ? params.get(2) : null;
String crs_encoding = params.size() > 3 ? params.get(3) : null;
ByteBuffer metadata =
params.size() > 3 ? ByteBuffer.wrap(params.get(3).getBytes()) : null;
return geometryType(encoding, edges, crs, metadata);
params.size() > 4 ? ByteBuffer.wrap(params.get(4).getBytes()) : null;
return geometryType(encoding, edges, crs, crs_encoding, metadata);
}
};

@@ -333,8 +334,8 @@ public static Float16LogicalTypeAnnotation float16Type() {
}

public static GeometryLogicalTypeAnnotation geometryType(
GeometryEncoding encoding, Edges edges, String crs, ByteBuffer metadata) {
return new GeometryLogicalTypeAnnotation(encoding, edges, crs, metadata);
GeometryEncoding encoding, Edges edges, String crs, String crs_encoding, ByteBuffer metadata) {
return new GeometryLogicalTypeAnnotation(encoding, edges, crs, crs_encoding, metadata);
}

public static class StringLogicalTypeAnnotation extends LogicalTypeAnnotation {
@@ -1138,17 +1139,21 @@ public enum Edges {
}

public static class GeometryLogicalTypeAnnotation extends LogicalTypeAnnotation {
public static final String CRS_ENCODING_DEFAULT = "PROJJSON";
private final GeometryEncoding encoding;
private final Edges edges;
private final String crs;
private final String crs_encoding;
private final ByteBuffer metadata;

private GeometryLogicalTypeAnnotation(GeometryEncoding encoding, Edges edges, String crs, ByteBuffer metadata) {
private GeometryLogicalTypeAnnotation(
GeometryEncoding encoding, Edges edges, String crs, String crs_encoding, ByteBuffer metadata) {
Preconditions.checkArgument(encoding != null, "Geometry encoding is required");
Preconditions.checkArgument(edges != null, "Geometry edges is required");
this.encoding = encoding;
this.edges = edges;
this.crs = crs;
this.crs_encoding = crs_encoding;
this.metadata = metadata;
}

@@ -1199,6 +1204,10 @@ public String getCrs() {
return crs;
}

public String getCrs_encoding() {
return crs_encoding;
}

public ByteBuffer getMetadata() {
return metadata;
}
Loading