Skip to content

Commit

Permalink
[native] Add support for Iceberg connector in presto_protocol
Browse files Browse the repository at this point in the history
Add Java classes from the Iceberg connector in presto_protocol.yml file.
Specialize IcebergColumnHandle to support 'operator<()'.
  • Loading branch information
imjalpreet committed Feb 13, 2024
1 parent 6588bab commit 2df519d
Show file tree
Hide file tree
Showing 11 changed files with 12,681 additions and 11,474 deletions.
20,097 changes: 10,507 additions & 9,590 deletions presto-native-execution/presto_cpp/presto_protocol/presto_protocol.cpp

Large diffs are not rendered by default.

3,934 changes: 2,053 additions & 1,881 deletions presto-native-execution/presto_cpp/presto_protocol/presto_protocol.h

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ AbstractClasses:
comparable: true
subclasses:
- { name: HiveColumnHandle, key: hive }
- { name: IcebergColumnHandle, key: hive-iceberg }
- { name: TpchColumnHandle, key: tpch }

ConnectorPartitioningHandle:
Expand All @@ -63,6 +64,7 @@ AbstractClasses:
super: JsonEncodedSubclass
subclasses:
- { name: HiveTableHandle, key: hive }
- { name: IcebergTableHandle, key: hive-iceberg }
- { name: TpchTableHandle, key: tpch }

ConnectorOutputTableHandle:
Expand All @@ -85,6 +87,7 @@ AbstractClasses:
super: JsonEncodedSubclass
subclasses:
- { name: HiveTableLayoutHandle, key: hive }
- { name: IcebergTableLayoutHandle, key: hive-iceberg }
- { name: TpchTableLayoutHandle, key: tpch }

ConnectorMetadataUpdateHandle:
Expand All @@ -96,6 +99,7 @@ AbstractClasses:
super: JsonEncodedSubclass
subclasses:
- { name: HiveSplit, key: hive }
- { name: IcebergSplit, key: hive-iceberg }
- { name: TpchSplit, key: tpch }
- { name: RemoteSplit, key: $remote }
- { name: EmptySplit, key: $empty }
Expand Down Expand Up @@ -177,6 +181,7 @@ JavaClasses:
- presto-hive-metastore/src/main/java/com/facebook/presto/hive/BucketFunctionType.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/CacheQuotaRequirement.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/CacheQuotaScope.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/BaseHiveColumnHandle.java
- presto-spi/src/main/java/com/facebook/presto/spi/relation/CallExpression.java
- presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java
- presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java
Expand Down Expand Up @@ -233,6 +238,18 @@ JavaClasses:
- presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTableHandle.java
- presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java
- presto-hive/src/main/java/com/facebook/presto/hive/HiveTransactionHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileContent.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplit.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableLayoutHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergColumnHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/ColumnIdentity.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogOperation.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitInfo.java
- presto-tpch/src/main/java/com/facebook/presto/tpch/TpchSplit.java
- presto-tpch/src/main/java/com/facebook/presto/tpch/TpchTableHandle.java
- presto-tpch/src/main/java/com/facebook/presto/tpch/TpchTableLayoutHandle.java
Expand Down Expand Up @@ -321,5 +338,4 @@ JavaClasses:
- presto-spark-base/src/main/java/com/facebook/presto/spark/execution/http/BatchTaskUpdateRequest.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinType.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinDistributionType.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/BaseHiveColumnHandle.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ void to_json(json& j, const std::shared_ptr<ColumnHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergColumnHandle>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchColumnHandle>(p);
return;
Expand All @@ -46,6 +51,14 @@ void from_json(const json& j, std::shared_ptr<ColumnHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
std::shared_ptr<IcebergColumnHandle> k =
std::make_shared<IcebergColumnHandle>();
j.get_to(*k);
p = std::static_pointer_cast<ColumnHandle>(k);
return;
}

if (getConnectorKey(type) == "tpch") {
std::shared_ptr<TpchColumnHandle> k = std::make_shared<TpchColumnHandle>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ void to_json(json& j, const std::shared_ptr<ConnectorSplit>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergSplit>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchSplit>(p);
return;
Expand Down Expand Up @@ -66,6 +71,13 @@ void from_json(const json& j, std::shared_ptr<ConnectorSplit>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
auto k = std::make_shared<IcebergSplit>();
j.get_to(*k);
p = k;
return;
}

if (getConnectorKey(type) == "tpch") {
auto k = std::make_shared<TpchSplit>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ void to_json(json& j, const std::shared_ptr<ConnectorTableHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergTableHandle>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchTableHandle>(p);
return;
Expand All @@ -47,6 +52,13 @@ void from_json(const json& j, std::shared_ptr<ConnectorTableHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
auto k = std::make_shared<IcebergTableHandle>();
j.get_to(*k);
p = k;
return;
}

if (getConnectorKey(type) == "tpch") {
auto k = std::make_shared<TpchTableHandle>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ void to_json(json& j, const std::shared_ptr<ConnectorTableLayoutHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergTableLayoutHandle>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchTableLayoutHandle>(p);
return;
Expand All @@ -48,6 +53,13 @@ void from_json(const json& j, std::shared_ptr<ConnectorTableLayoutHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
auto k = std::make_shared<IcebergTableLayoutHandle>();
j.get_to(*k);
p = k;
return;
}

if (getConnectorKey(type) == "tpch") {
auto k = std::make_shared<TpchTableLayoutHandle>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ void to_json(json& j, const std::shared_ptr<ConnectorTransactionHandle>& p) {
j = *std::static_pointer_cast<HiveTransactionHandle>(p);
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<HiveTransactionHandle>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchTransactionHandle>(p);
return;
Expand Down Expand Up @@ -64,6 +70,12 @@ void from_json(const json& j, std::shared_ptr<ConnectorTransactionHandle>& p) {
p = k;
return;
}
if (getConnectorKey(type) == "hive-iceberg") {
auto k = std::make_shared<HiveTransactionHandle>();
j.get_to(*k);
p = k;
return;
}
if (getConnectorKey(type) == "tpch") {
auto k = std::make_shared<TpchTransactionHandle>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
* limitations under the License.
*/

// HiveColumnHandle is special since it needs an implementation of
// operator<().

namespace facebook::presto::protocol {

struct HiveColumnHandle : public ColumnHandle {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

// IcebergColumnHandle is special since it needs an implementation of
// operator<().

namespace facebook::presto::protocol {

struct IcebergColumnHandle : public ColumnHandle {
ColumnIdentity columnIdentity = {};
Type type = {};
std::shared_ptr<String> comment = {};
ColumnType columnType = {};
List<Subfield> requiredSubfields = {};

IcebergColumnHandle() noexcept;

bool operator<(const ColumnHandle& o) const override {
return columnIdentity.name <
dynamic_cast<const IcebergColumnHandle&>(o).columnIdentity.name;
}
};

void to_json(json& j, const IcebergColumnHandle& p);
void from_json(const json& j, IcebergColumnHandle& p);

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* limitations under the License.
*/

// TpchColumnHandle is special since we require an implementation of
// TpchColumnHandle is special since it needs an implementation of
// operator<().

namespace facebook::presto::protocol {
Expand Down

0 comments on commit 2df519d

Please sign in to comment.