Skip to content

Commit

Permalink
[native] Add Iceberg Connector to parse Java Iceberg Plan Fragment
Browse files Browse the repository at this point in the history
Introduce presto protocol changes to parse/deserialize Iceberg Plan Fragment and other Iceberg Java objects
  • Loading branch information
imjalpreet committed Feb 1, 2024
1 parent f5f930c commit 4bd39ae
Show file tree
Hide file tree
Showing 25 changed files with 13,178 additions and 11,107 deletions.
19,957 changes: 10,529 additions & 9,428 deletions presto-native-execution/presto_cpp/presto_protocol/presto_protocol.cpp

Large diffs are not rendered by default.

3,697 changes: 2,025 additions & 1,672 deletions presto-native-execution/presto_cpp/presto_protocol/presto_protocol.h

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ AbstractClasses:
comparable: true
subclasses:
- { name: HiveColumnHandle, key: hive }
- { name: IcebergColumnHandle, key: hive-iceberg }
- { name: TpchColumnHandle, key: tpch }

ConnectorPartitioningHandle:
Expand All @@ -63,6 +64,7 @@ AbstractClasses:
super: JsonEncodedSubclass
subclasses:
- { name: HiveTableHandle, key: hive }
- { name: IcebergTableHandle, key: hive-iceberg }
- { name: TpchTableHandle, key: tpch }

ConnectorOutputTableHandle:
Expand All @@ -85,6 +87,7 @@ AbstractClasses:
super: JsonEncodedSubclass
subclasses:
- { name: HiveTableLayoutHandle, key: hive }
- { name: IcebergTableLayoutHandle, key: hive-iceberg }
- { name: TpchTableLayoutHandle, key: tpch }

ConnectorMetadataUpdateHandle:
Expand All @@ -96,6 +99,7 @@ AbstractClasses:
super: JsonEncodedSubclass
subclasses:
- { name: HiveSplit, key: hive }
- { name: IcebergSplit, key: hive-iceberg }
- { name: TpchSplit, key: tpch }
- { name: RemoteSplit, key: $remote }
- { name: EmptySplit, key: $empty }
Expand Down Expand Up @@ -177,6 +181,9 @@ JavaClasses:
- presto-hive-metastore/src/main/java/com/facebook/presto/hive/BucketFunctionType.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/CacheQuotaRequirement.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/CacheQuotaScope.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/BaseHiveTableHandle.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/BaseHiveColumnHandle.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/BaseHiveTableLayoutHandle.java
- presto-spi/src/main/java/com/facebook/presto/spi/relation/CallExpression.java
- presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java
- presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java
Expand Down Expand Up @@ -233,6 +240,15 @@ JavaClasses:
- presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTableHandle.java
- presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java
- presto-hive/src/main/java/com/facebook/presto/hive/HiveTransactionHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplit.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableLayoutHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergColumnHandle.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/ColumnIdentity.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java
- presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitInfo.java
- presto-tpch/src/main/java/com/facebook/presto/tpch/TpchSplit.java
- presto-tpch/src/main/java/com/facebook/presto/tpch/TpchTableHandle.java
- presto-tpch/src/main/java/com/facebook/presto/tpch/TpchTableLayoutHandle.java
Expand Down Expand Up @@ -321,5 +337,4 @@ JavaClasses:
- presto-spark-base/src/main/java/com/facebook/presto/spark/execution/http/BatchTaskUpdateRequest.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinType.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinDistributionType.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java
- presto-hive-common/src/main/java/com/facebook/presto/hive/BaseHiveColumnHandle.java
- presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
namespace facebook::presto::protocol {

struct BaseHiveColumnHandle : public ColumnHandle {
String name = {};
std::shared_ptr<String> comment = {};
ColumnType columnType = {};
List<Subfield> requiredSubfields = {};

bool operator<(const ColumnHandle& o) const override {
return name < dynamic_cast<const BaseHiveColumnHandle&>(o).name;
}
};

void to_json(json& j, const BaseHiveColumnHandle& p);
void from_json(const json& j, BaseHiveColumnHandle& p);

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
namespace facebook::presto::protocol {
struct BaseHiveTableHandle : public ConnectorTableHandle {
String schemaName = {};
String tableName = {};
};
void to_json(json& j, const BaseHiveTableHandle& p);
void from_json(const json& j, BaseHiveTableHandle& p);
} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
namespace facebook::presto::protocol {

struct BaseHiveTableLayoutHandle : public ConnectorTableLayoutHandle {
List<BaseHiveColumnHandle> partitionColumns = {};
TupleDomain<Subfield> domainPredicate = {};
std::shared_ptr<RowExpression> remainingPredicate = {};
bool pushdownFilterEnabled = {};
TupleDomain<std::shared_ptr<ColumnHandle>> partitionColumnPredicate = {};
};

void to_json(json& j, const BaseHiveTableLayoutHandle& p);
void from_json(const json& j, BaseHiveTableLayoutHandle& p);

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

namespace facebook::presto::protocol {

static const std::pair<ChangelogOperation, json>
ChangelogOperation_enum_table[] = { // NOLINT: cert-err58-cpp
{ChangelogOperation::INSERT, "INSERT"},
{ChangelogOperation::DELETE, "DELETE"},
{ChangelogOperation::UPDATE_BEFORE, "UPDATE_BEFORE"},
{ChangelogOperation::UPDATE_AFTER, "UPDATE_AFTER"}};

void to_json(json& j, const ChangelogOperation& e) {
static_assert(
std::is_enum<ChangelogOperation>::value,
"ChangelogOperation must be an enum!");

const auto* it = std::find_if(
std::begin(ChangelogOperation_enum_table),
std::end(ChangelogOperation_enum_table),
[e](const std::pair<ChangelogOperation, json>& ej_pair) -> bool {
return ej_pair.first == e;
});
j = ((it != std::end(ChangelogOperation_enum_table))
? it
: std::begin(ChangelogOperation_enum_table))
->second;
}

void from_json(const json& j, ChangelogOperation& e) {
static_assert(
std::is_enum<ChangelogOperation>::value,
"ChangelogOperation must be an enum!");

const auto* it = std::find_if(
std::begin(ChangelogOperation_enum_table),
std::end(ChangelogOperation_enum_table),
[&j](const std::pair<ChangelogOperation, json>& ej_pair) -> bool {
return ej_pair.second == j;
});
e = ((it != std::end(ChangelogOperation_enum_table))
? it
: std::begin(ChangelogOperation_enum_table))
->first;
}

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

namespace facebook::presto::protocol {
enum class ChangelogOperation { INSERT, DELETE, UPDATE_BEFORE, UPDATE_AFTER };

void to_json(json& j, const ChangelogOperation& p);
void from_json(const json& j, ChangelogOperation& p);

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ void to_json(json& j, const std::shared_ptr<ColumnHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergColumnHandle>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchColumnHandle>(p);
return;
Expand All @@ -46,6 +51,14 @@ void from_json(const json& j, std::shared_ptr<ColumnHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
std::shared_ptr<IcebergColumnHandle> k =
std::make_shared<IcebergColumnHandle>();
j.get_to(*k);
p = std::static_pointer_cast<ColumnHandle>(k);
return;
}

if (getConnectorKey(type) == "tpch") {
std::shared_ptr<TpchColumnHandle> k = std::make_shared<TpchColumnHandle>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

namespace facebook::presto::protocol {

static const std::pair<ColumnType, json> ColumnType_enum_table[] =
{ // NOLINT: cert-err58-cpp
{ColumnType::PARTITION_KEY, "PARTITION_KEY"},
{ColumnType::REGULAR, "REGULAR"},
{ColumnType::SYNTHESIZED, "SYNTHESIZED"},
{ColumnType::AGGREGATED, "AGGREGATED"}};

void to_json(json& j, const ColumnType& e) {
static_assert(std::is_enum<ColumnType>::value, "ColumnType must be an enum!");

const auto* it = std::find_if(
std::begin(ColumnType_enum_table),
std::end(ColumnType_enum_table),
[e](const std::pair<ColumnType, json>& ej_pair) -> bool {
return ej_pair.first == e;
});
j = ((it != std::end(ColumnType_enum_table))
? it
: std::begin(ColumnType_enum_table))
->second;
}

void from_json(const json& j, ColumnType& e) {
static_assert(std::is_enum<ColumnType>::value, "ColumnType must be an enum!");

const auto* it = std::find_if(
std::begin(ColumnType_enum_table),
std::end(ColumnType_enum_table),
[&j](const std::pair<ColumnType, json>& ej_pair) -> bool {
return ej_pair.second == j;
});
e = ((it != std::end(ColumnType_enum_table))
? it
: std::begin(ColumnType_enum_table))
->first;
}

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

namespace facebook::presto::protocol {
enum class ColumnType { PARTITION_KEY, REGULAR, SYNTHESIZED, AGGREGATED };

void to_json(json& j, const ColumnType& p);
void from_json(const json& j, ColumnType& p);

} // namespace facebook::presto::protocol
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ void to_json(json& j, const std::shared_ptr<ConnectorSplit>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergSplit>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchSplit>(p);
return;
Expand Down Expand Up @@ -66,6 +71,13 @@ void from_json(const json& j, std::shared_ptr<ConnectorSplit>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
auto k = std::make_shared<IcebergSplit>();
j.get_to(*k);
p = k;
return;
}

if (getConnectorKey(type) == "tpch") {
auto k = std::make_shared<TpchSplit>();
j.get_to(*k);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ void to_json(json& j, const std::shared_ptr<ConnectorTableHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
j = *std::static_pointer_cast<IcebergTableHandle>(p);
return;
}

if (getConnectorKey(type) == "tpch") {
j = *std::static_pointer_cast<TpchTableHandle>(p);
return;
Expand All @@ -47,6 +52,13 @@ void from_json(const json& j, std::shared_ptr<ConnectorTableHandle>& p) {
return;
}

if (getConnectorKey(type) == "hive-iceberg") {
auto k = std::make_shared<IcebergTableHandle>();
j.get_to(*k);
p = k;
return;
}

if (getConnectorKey(type) == "tpch") {
auto k = std::make_shared<TpchTableHandle>();
j.get_to(*k);
Expand Down
Loading

0 comments on commit 4bd39ae

Please sign in to comment.