From 442f86b33b4c0886b6c9e43b1d28fe49536a5727 Mon Sep 17 00:00:00 2001 From: Allison Portis Date: Fri, 20 Dec 2024 22:06:57 -0800 Subject: [PATCH] Remove cc code --- .../io/delta/kernel/internal/TableConfig.java | 49 ----- .../internal/coordinatedcommits/Commit.java | 72 -------- .../CommitCoordinatorClientHandler.java | 170 ------------------ .../CommitFailedException.java | 64 ------- .../coordinatedcommits/CommitResponse.java | 43 ----- .../GetCommitsResponse.java | 60 ------- .../coordinatedcommits/UpdatedActions.java | 99 ---------- .../actions/AbstractCommitInfo.java | 33 ---- .../actions/AbstractMetadata.java | 58 ------ .../actions/AbstractProtocol.java | 42 ----- .../TableCommitCoordinatorClientHandler.java | 73 -------- .../util/CoordinatedCommitsUtils.java | 106 ----------- .../internal/SnapshotManagerSuite.scala | 1 - .../kernel/internal/TableConfigSuite.scala | 3 - .../delta/kernel/test/MockEngineUtils.scala | 4 +- .../LogReplayEngineMetricsSuite.scala | 1 - 16 files changed, 1 insertion(+), 877 deletions(-) delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/Commit.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitCoordinatorClientHandler.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitFailedException.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitResponse.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/GetCommitsResponse.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/UpdatedActions.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractCommitInfo.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractMetadata.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractProtocol.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/TableCommitCoordinatorClientHandler.java delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java index 794c016d1ed..aa82b284560 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java @@ -138,52 +138,6 @@ public class TableConfig { "needs to be a long.", true); - /* - * This table property is used to track the commit-coordinator name for this table. If this - * property is not set, the table will be considered as file system table and commits will be - * done via atomically publishing the commit file. - */ - public static final TableConfig> COORDINATED_COMMITS_COORDINATOR_NAME = - new TableConfig<>( - "delta.coordinatedCommits.commitCoordinator-preview", - null, /* default values */ - Optional::ofNullable, - value -> true, - "The commit-coordinator name for this table. This is used to determine " - + "which implementation of commit-coordinator to use when committing " - + "to this table. If this property is not set, the table will be " - + "considered as file system table and commits will be done via " - + "atomically publishing the commit file.", - true); - - /* - * This table property is used to track the configuration properties for the commit coordinator - * which is needed to build the commit coordinator client. - */ - public static final TableConfig> COORDINATED_COMMITS_COORDINATOR_CONF = - new TableConfig<>( - "delta.coordinatedCommits.commitCoordinatorConf-preview", - null, /* default values */ - JsonUtils::parseJSONKeyValueMap, - value -> true, - "A string-to-string map of configuration properties for the" - + " coordinated commits-coordinator.", - true); - - /* - * This property is used by the commit coordinator to uniquely identify and manage the table - * internally. - */ - public static final TableConfig> COORDINATED_COMMITS_TABLE_CONF = - new TableConfig<>( - "delta.coordinatedCommits.tableConf-preview", - null, /* default values */ - JsonUtils::parseJSONKeyValueMap, - value -> true, - "A string-to-string map of configuration properties for" - + " describing the table to commit-coordinator.", - true); - /** This table property is used to control the column mapping mode. */ public static final TableConfig COLUMN_MAPPING_MODE = new TableConfig<>( @@ -226,9 +180,6 @@ public class TableConfig { addConfig(this, IN_COMMIT_TIMESTAMPS_ENABLED); addConfig(this, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION); addConfig(this, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP); - addConfig(this, COORDINATED_COMMITS_COORDINATOR_NAME); - addConfig(this, COORDINATED_COMMITS_COORDINATOR_CONF); - addConfig(this, COORDINATED_COMMITS_TABLE_CONF); addConfig(this, COLUMN_MAPPING_MODE); addConfig(this, ICEBERG_COMPAT_V2_ENABLED); addConfig(this, COLUMN_MAPPING_MAX_COLUMN_ID); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/Commit.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/Commit.java deleted file mode 100644 index 841daeb0d74..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/Commit.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits; - -import io.delta.kernel.annotation.Evolving; -import io.delta.kernel.utils.FileStatus; - -/** - * Representation of a commit file. It contains the version of the commit, the file status of the - * commit, and the timestamp of the commit. This is used when we want to get the commit information - * from the {@link CommitCoordinatorClientHandler#commit} and {@link - * CommitCoordinatorClientHandler#getCommits} APIs. - * - * @since 3.3.0 - */ -@Evolving -public class Commit { - - private final long version; - - private final FileStatus fileStatus; - - private final long commitTimestamp; - - public Commit(long version, FileStatus fileStatus, long commitTimestamp) { - this.version = version; - this.fileStatus = fileStatus; - this.commitTimestamp = commitTimestamp; - } - - /** - * Get the version of the commit. - * - * @return the version of the commit. - */ - public long getVersion() { - return version; - } - - /** - * Get the file status of the commit. - * - * @return the file status of the commit. - */ - public FileStatus getFileStatus() { - return fileStatus; - } - - /** - * Get the timestamp that represents the time since epoch in milliseconds when the commit write - * was started. - * - * @return the timestamp of the commit. - */ - public long getCommitTimestamp() { - return commitTimestamp; - } -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitCoordinatorClientHandler.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitCoordinatorClientHandler.java deleted file mode 100644 index 619c1b76bbe..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitCoordinatorClientHandler.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits; - -import io.delta.kernel.annotation.Evolving; -import io.delta.kernel.data.Row; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractMetadata; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractProtocol; -import io.delta.kernel.utils.CloseableIterator; -import java.io.IOException; -import java.util.Map; - -/** - * An interface that encapsulates all the functions needed by Kernel to perform commits to a table - * owned by a Commit Coordinator. Commit coordinator is defined by the Delta Protocol. - * - * @see Coordinated - * commit protocol table feature. - * @since 3.3.0 - */ -@Evolving -public interface CommitCoordinatorClientHandler { - - /** - * API to register the table represented by the given `logPath` at the provided - * currentTableVersion with the commit coordinator this commit coordinator client represents. - * - *

This API is called when the table is being converted from a file system table to a - * coordinated-commit table. - * - *

When a new coordinated-commit table is being created, the currentTableVersion will be -1 and - * the upgrade commit needs to be a file system commit which will write the backfilled file - * directly. - * - * @param logPath The path to the delta log of the table that should be converted - * @param currentVersion The currentTableVersion is the version of the table just before - * conversion. currentTableVersion + 1 represents the commit that will do the conversion. This - * must be backfilled atomically. currentTableVersion + 2 represents the first commit after - * conversion. This will go through the CommitCoordinatorClient and the client is free to - * choose when it wants to backfill this commit. - * @param currentMetadata The metadata of the table at currentTableVersion - * @param currentProtocol The protocol of the table at currentTableVersion - * @return A map of key-value pairs which is issued by the commit coordinator to identify the - * table. This should be stored in the table's metadata. This information needs to be passed - * to the {@link #commit}, {@link #getCommits}, and {@link #backfillToVersion} APIs to - * identify the table. - */ - Map registerTable( - String logPath, - long currentVersion, - AbstractMetadata currentMetadata, - AbstractProtocol currentProtocol); - - /** - * Commits a set of actions to a specified table at a given version. - * - *

This method applies the provided actions to the table identified by {@code logPath} as given - * table version {@code commitVersion}. - * - * @param logPath The path to the delta log of the target table - * @param tableConf Configuration details returned by the commit coordinator client during table - * registration - * @param commitVersion The version of the commit that is being committed. - * @param actions The set of actions to be committed - * @param updatedActions Additional information for the commit, including: - *

    - *
  • Commit info - *
  • Metadata changes - *
  • Protocol changes - *
- * - * @return CommitResponse containing the file status of the committed file. Note: If the commit is - * already backfilled, the file status may be omitted, and the client can retrieve this - * information independently. - * @throws CommitFailedException if the commit operation fails - */ - CommitResponse commit( - String logPath, - Map tableConf, - long commitVersion, - CloseableIterator actions, - UpdatedActions updatedActions) - throws CommitFailedException; - - /** - * Retrieves unbackfilled commits for a specified table within a given version range. - * - *

This method fetches commits that have not yet been backfilled for the table identified by - * {@code logPath}. It returns commits between {@code startVersion} and {@code endVersion} - * (inclusive), ignoring commits outside this range. The returned commits are guaranteed to be - * contiguous and in ascending version order. - * - *

Note: The first returned version may not equal {@code startVersion} if some - * versions have already been backfilled and are no longer tracked by the commit coordinator. - * - *

The {@code latestTableVersion} in the response represents the highest commit version - * ratified by the commit coordinator. A value of -1 is only valid if the commit coordinator has - * never ratified any version (i.e., never accepted any unbackfilled commit). - * - * @param logPath The path to the delta log of the target table - * @param tableConf The table configuration returned by the commit coordinator during registration - * @param startVersion The minimum commit version to retrieve (inclusive, can be null) - * @param endVersion The maximum commit version to retrieve (inclusive, can be null) - * @return GetCommitsResponse containing: - *

    - *
  • A list of {@link Commit} objects - *
  • The {@code latestTableVersion} tracked by {@link CommitCoordinatorClientHandler} - *
- */ - GetCommitsResponse getCommits( - String logPath, Map tableConf, Long startVersion, Long endVersion); - - /** - * Requests the commit coordinator client to backfill commits up to a specified version. - * - *

This method instructs the commit coordinator client to backfill all commits up to the given - * {@code version} and notify the commit coordinator of the completion. A successful return from - * this method guarantees that the backfill has been completed, even if the commit coordinator - * hasn't been notified yet. - * - * @param logPath The path to the delta log of the table to be backfilled - * @param tableConf The table configuration returned by the commit coordinator during registration - * @param version The target version up to which commits should be backfilled - * @param lastKnownBackfilledVersion The most recent version known to be backfilled before this - * call. If null or invalid, the client will backfill from the table's beginning. Can be null. - * @throws IOException if the backfill operation fails - *

Note: A successful return indicates backfill completion, but the commit - * coordinator may not be immediately aware of this update. - */ - void backfillToVersion( - String logPath, Map tableConf, long version, Long lastKnownBackfilledVersion) - throws IOException; - - /** - * Compares this {@link CommitCoordinatorClientHandler} for semantic equality with another - * instance. - * - *

Semantic equality is defined by the specific implementation of {@link - * CommitCoordinatorClientHandler}. Two instances are considered semantically equal if they can be - * used interchangeably for any {@link CommitCoordinatorClientHandler} API, including but not - * limited to: - * - *

    - *
  • {@link #commit} - *
  • {@link #getCommits} - *
- * - *

For example, semantic equality might be based on both instances pointing to the same - * underlying endpoint. - * - * @param other The {@link CommitCoordinatorClientHandler} to compare with this instance - * @return boolean True if the instances are semantically equal, false otherwise - */ - boolean semanticEquals(CommitCoordinatorClientHandler other); -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitFailedException.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitFailedException.java deleted file mode 100644 index f5f06697f7f..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitFailedException.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits; - -import io.delta.kernel.annotation.Evolving; - -/** - * Exception raised by {@link CommitCoordinatorClientHandler#commit} - * - * 

- *  | retryable | conflict  | meaning                                                         |
- *  |   no      |   no      | something bad happened (e.g. auth failure)                      |
- *  |   no      |   yes     | permanent transaction conflict (e.g. multi-table commit failed) |
- *  |   yes     |   no      | transient error (e.g. network hiccup)                           |
- *  |   yes     |   yes     | physical conflict (allowed to rebase and retry)                 |
- *
- * - * @since 3.3.0 - */ -@Evolving -public class CommitFailedException extends Exception { - - private final boolean retryable; - - private final boolean conflict; - - public CommitFailedException(boolean retryable, boolean conflict, String message) { - super(message); - this.retryable = retryable; - this.conflict = conflict; - } - - /** - * Returns whether the commit attempt can be retried. - * - * @return whether the commit attempt can be retried. - */ - public boolean getRetryable() { - return retryable; - } - - /** - * Returns whether the commit failed due to a conflict. - * - * @return whether the commit failed due to a conflict. - */ - public boolean getConflict() { - return conflict; - } -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitResponse.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitResponse.java deleted file mode 100644 index ed23e99203a..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/CommitResponse.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits; - -import io.delta.kernel.annotation.Evolving; - -/** - * Response container for {@link CommitCoordinatorClientHandler#commit}. - * - * @since 3.3.0 - */ -@Evolving -public class CommitResponse { - - private final Commit commit; - - public CommitResponse(Commit commit) { - this.commit = commit; - } - - /** - * Get the commit object. - * - * @return the commit object. - */ - public Commit getCommit() { - return commit; - } -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/GetCommitsResponse.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/GetCommitsResponse.java deleted file mode 100644 index 85b4c298726..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/GetCommitsResponse.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits; - -import io.delta.kernel.annotation.Evolving; -import java.util.List; -import java.util.Map; - -/** - * Response container for {@link CommitCoordinatorClientHandler#getCommits( String, Map, Long, - * Long)}. Holds all the commits that have not been backfilled as per the commit coordinator. - * - * @since 3.3.0 - */ -@Evolving -public class GetCommitsResponse { - private final List commits; - - private final long latestTableVersion; - - public GetCommitsResponse(List commits, long latestTableVersion) { - this.commits = commits; - this.latestTableVersion = latestTableVersion; - } - - /** - * Get the list of commits that have not been backfilled as per the commit coordinator. It is - * possible that some of these commits have been physically backfilled but the commit coordinator - * is not aware of this. - * - * @return the list of commits. - */ - public List getCommits() { - return commits; - } - - /** - * Get the latest table version as per the coordinator. This can be -1 when no commit has gone - * through the coordinator even though the actual table has a non-negative version. - * - * @return the latest table version. - */ - public long getLatestTableVersion() { - return latestTableVersion; - } -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/UpdatedActions.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/UpdatedActions.java deleted file mode 100644 index 0f9841403c9..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/UpdatedActions.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits; - -import io.delta.kernel.annotation.Evolving; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractCommitInfo; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractMetadata; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractProtocol; - -/** - * A container class to inform the {@link CommitCoordinatorClientHandler} about any changes in - * Protocol/Metadata - * - * @since 3.3.0 - */ -@Evolving -public class UpdatedActions { - private final AbstractCommitInfo commitInfo; - - private final AbstractMetadata newMetadata; - - private final AbstractProtocol newProtocol; - - private final AbstractMetadata oldMetadata; - - private final AbstractProtocol oldProtocol; - - public UpdatedActions( - AbstractCommitInfo commitInfo, - AbstractMetadata newMetadata, - AbstractProtocol newProtocol, - AbstractMetadata oldMetadata, - AbstractProtocol oldProtocol) { - this.commitInfo = commitInfo; - this.newMetadata = newMetadata; - this.newProtocol = newProtocol; - this.oldMetadata = oldMetadata; - this.oldProtocol = oldProtocol; - } - - /** - * Get the commit info for this commit attempt. - * - * @return the commit info. - */ - public AbstractCommitInfo getCommitInfo() { - return commitInfo; - } - - /** - * Get the new metadata which needs to be committed. - * - * @return the new metadata. - */ - public AbstractMetadata getNewMetadata() { - return newMetadata; - } - - /** - * Get the new protocol which needs to be committed. - * - * @return the new protocol. - */ - public AbstractProtocol getNewProtocol() { - return newProtocol; - } - - /** - * Get the metadata from the read snapshot of this transaction. - * - * @return the old metadata. - */ - public AbstractMetadata getOldMetadata() { - return oldMetadata; - } - - /** - * Get the protocol from the read snapshot of this transaction. - * - * @return the old protocol. - */ - public AbstractProtocol getOldProtocol() { - return oldProtocol; - } -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractCommitInfo.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractCommitInfo.java deleted file mode 100644 index 1d2b1f1b849..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractCommitInfo.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits.actions; - -import io.delta.kernel.annotation.Evolving; - -/** - * Interface for objects that represents the base information for a commit. Commits need to provide - * an in-commit timestamp. This timestamp is used to specify the exact time the commit happened and - * determines the target version for time-based time travel queries. - * - * @since 3.3.0 - */ -@Evolving -public interface AbstractCommitInfo { - - /** Get the timestamp of the commit as millis after the epoch. */ - long getCommitTimestamp(); -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractMetadata.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractMetadata.java deleted file mode 100644 index 46352e6bbe5..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractMetadata.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits.actions; - -import io.delta.kernel.annotation.Evolving; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * Interface for metadata actions in Delta. The metadata defines the metadata of the table. - * - * @since 3.3.0 - */ -@Evolving -public interface AbstractMetadata { - - /** A unique table identifier. */ - String getId(); - - /** User-specified table identifier. */ - String getName(); - - /** User-specified table description. */ - String getDescription(); - - /** The table provider format. */ - String getProvider(); - - /** The format options */ - Map getFormatOptions(); - - /** The table schema in string representation. */ - String getSchemaString(); - - /** List of partition columns. */ - List getPartitionColumns(); - - /** The table properties defined on the table. */ - Map getConfiguration(); - - /** Timestamp for the creation of this metadata. */ - Optional getCreatedTime(); -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractProtocol.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractProtocol.java deleted file mode 100644 index 9773b007aae..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/coordinatedcommits/actions/AbstractProtocol.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.coordinatedcommits.actions; - -import io.delta.kernel.annotation.Evolving; -import java.util.Set; - -/** - * Interface for protocol actions in Delta. The protocol defines the requirements that readers and - * writers of the table need to meet. - * - * @since 3.3.0 - */ -@Evolving -public interface AbstractProtocol { - - /** The minimum reader version required to read the table. */ - int getMinReaderVersion(); - - /** The minimum writer version required to read the table. */ - int getMinWriterVersion(); - - /** The reader features that need to be supported to read the table. */ - Set getReaderFeatures(); - - /** The writer features that need to be supported to write the table. */ - Set getWriterFeatures(); -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/TableCommitCoordinatorClientHandler.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/TableCommitCoordinatorClientHandler.java deleted file mode 100644 index c379319522b..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/TableCommitCoordinatorClientHandler.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.internal.snapshot; - -import io.delta.kernel.data.Row; -import io.delta.kernel.internal.coordinatedcommits.CommitCoordinatorClientHandler; -import io.delta.kernel.internal.coordinatedcommits.CommitFailedException; -import io.delta.kernel.internal.coordinatedcommits.CommitResponse; -import io.delta.kernel.internal.coordinatedcommits.GetCommitsResponse; -import io.delta.kernel.internal.coordinatedcommits.UpdatedActions; -import io.delta.kernel.utils.CloseableIterator; -import java.io.IOException; -import java.util.Map; - -/** - * A wrapper around {@link CommitCoordinatorClientHandler} that provides a more user-friendly API - * for committing/ accessing commits to a specific table. This class takes care of passing the table - * specific configuration to the underlying {@link CommitCoordinatorClientHandler} e.g. logPath / - * coordinatedCommitsTableConf. - */ -public class TableCommitCoordinatorClientHandler { - private final CommitCoordinatorClientHandler commitCoordinatorClientHandler; - private final String logPath; - private final Map tableConf; - - public TableCommitCoordinatorClientHandler( - CommitCoordinatorClientHandler commitCoordinatorClientHandler, - String logPath, - Map tableConf) { - this.commitCoordinatorClientHandler = commitCoordinatorClientHandler; - this.logPath = logPath; - this.tableConf = tableConf; - } - - public CommitResponse commit( - long commitVersion, CloseableIterator actions, UpdatedActions updatedActions) - throws CommitFailedException { - return commitCoordinatorClientHandler.commit( - logPath, tableConf, commitVersion, actions, updatedActions); - } - - public GetCommitsResponse getCommits(Long startVersion, Long endVersion) { - return commitCoordinatorClientHandler.getCommits(logPath, tableConf, startVersion, endVersion); - } - - public void backfillToVersion(long version, Long lastKnownBackfilledVersion) throws IOException { - commitCoordinatorClientHandler.backfillToVersion( - logPath, tableConf, version, lastKnownBackfilledVersion); - } - - public boolean semanticEquals( - CommitCoordinatorClientHandler otherCommitCoordinatorClientHandler) { - return commitCoordinatorClientHandler.semanticEquals(otherCommitCoordinatorClientHandler); - } - - public boolean semanticEquals(TableCommitCoordinatorClientHandler otherCommitCoordinatorClient) { - return semanticEquals(otherCommitCoordinatorClient.commitCoordinatorClientHandler); - } -} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java deleted file mode 100644 index d6c73925738..00000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.internal.util; - -import io.delta.kernel.internal.actions.CommitInfo; -import io.delta.kernel.internal.actions.Metadata; -import io.delta.kernel.internal.actions.Protocol; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractCommitInfo; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractMetadata; -import io.delta.kernel.internal.coordinatedcommits.actions.AbstractProtocol; -import java.util.*; - -public class CoordinatedCommitsUtils { - public static AbstractMetadata convertMetadataToAbstractMetadata(Metadata metadata) { - return new AbstractMetadata() { - @Override - public String getId() { - return metadata.getId(); - } - - @Override - public String getName() { - return metadata.getName().orElse(null); - } - - @Override - public String getDescription() { - return metadata.getDescription().orElse(null); - } - - @Override - public String getProvider() { - return metadata.getFormat().getProvider(); - } - - @Override - public Map getFormatOptions() { - // Assuming Format class has a method to get format options - return metadata.getFormat().getOptions(); - } - - @Override - public String getSchemaString() { - // Assuming Metadata class has a method to get schema string - return metadata.getSchemaString(); - } - - @Override - public List getPartitionColumns() { - // Assuming Metadata class has a method to get partition columns - return VectorUtils.toJavaList(metadata.getPartitionColumns()); - } - - @Override - public Map getConfiguration() { - return metadata.getConfiguration(); - } - - @Override - public Optional getCreatedTime() { - return metadata.getCreatedTime(); - } - }; - } - - public static AbstractProtocol convertProtocolToAbstractProtocol(Protocol protocol) { - return new AbstractProtocol() { - @Override - public int getMinReaderVersion() { - return protocol.getMinReaderVersion(); - } - - @Override - public int getMinWriterVersion() { - return protocol.getMinWriterVersion(); - } - - @Override - public Set getReaderFeatures() { - return new HashSet<>(protocol.getReaderFeatures()); - } - - @Override - public Set getWriterFeatures() { - return new HashSet<>(protocol.getWriterFeatures()); - } - }; - } - - public static AbstractCommitInfo convertCommitInfoToAbstractCommitInfo(CommitInfo commitInfo) { - return () -> commitInfo.getInCommitTimestamp().orElse(commitInfo.getTimestamp()); - } -} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala index 0a728b7ab02..1cca46d51f7 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala @@ -24,7 +24,6 @@ import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.exceptions.InvalidTableException import io.delta.kernel.expressions.Predicate import io.delta.kernel.internal.checkpoints.{CheckpointInstance, SidecarFile} -import io.delta.kernel.internal.coordinatedcommits.{Commit, CommitCoordinatorClientHandler, CommitResponse, GetCommitsResponse} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.snapshot.{LogSegment, SnapshotManager} import io.delta.kernel.internal.util.{FileNames, Utils} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TableConfigSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TableConfigSuite.scala index 2dafb5b82a9..65ac2fe4d05 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TableConfigSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TableConfigSuite.scala @@ -30,9 +30,6 @@ class TableConfigSuite extends AnyFunSuite { TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> "1", TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey -> "1", - TableConfig.COORDINATED_COMMITS_COORDINATOR_NAME.getKey -> "{in-memory}", - TableConfig.COORDINATED_COMMITS_COORDINATOR_CONF.getKey -> "{\"1\": \"1\"}", - TableConfig.COORDINATED_COMMITS_TABLE_CONF.getKey -> "{\"1\": \"1\"}", TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true").asJava) } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockEngineUtils.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockEngineUtils.scala index ea095a94adb..be6dcb4b192 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockEngineUtils.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockEngineUtils.scala @@ -18,7 +18,6 @@ package io.delta.kernel.test import io.delta.kernel.engine._ import io.delta.kernel.data.{ColumnVector, ColumnarBatch, FilteredColumnarBatch, Row} import io.delta.kernel.expressions.{Column, Expression, ExpressionEvaluator, Predicate, PredicateEvaluator} -import io.delta.kernel.internal.coordinatedcommits.CommitCoordinatorClientHandler import io.delta.kernel.types.{DataType, StructType} import io.delta.kernel.utils.{CloseableIterator, DataFileStatus, FileStatus} @@ -53,8 +52,7 @@ trait MockEngineUtils { fileSystemClient: FileSystemClient = null, jsonHandler: JsonHandler = null, parquetHandler: ParquetHandler = null, - expressionHandler: ExpressionHandler = null, - commitCoordinatorClientHandler: CommitCoordinatorClientHandler = null): Engine = { + expressionHandler: ExpressionHandler = null): Engine = { new Engine() { override def getExpressionHandler: ExpressionHandler = Option(expressionHandler).getOrElse( diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala index d30d5d1f92a..cd43dc11838 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala @@ -23,7 +23,6 @@ import io.delta.kernel.data.ColumnarBatch import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler, DefaultParquetHandler} import io.delta.kernel.expressions.Predicate import io.delta.kernel.internal.checkpoints.Checkpointer -import io.delta.kernel.internal.coordinatedcommits.CommitCoordinatorClientHandler import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.internal.util.Utils.toCloseableIterator