Skip to content

Commit

Permalink
[590] Add interfaces for CatalogSyncClient and CatalogSync
Browse files Browse the repository at this point in the history
  • Loading branch information
vinishjail97 committed Dec 26, 2024
1 parent a08ca24 commit fb724f0
Show file tree
Hide file tree
Showing 34 changed files with 1,148 additions and 269 deletions.
15 changes: 14 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
<maven-gpg-plugin.version>3.2.4</maven-gpg-plugin.version>
<maven-deploy-plugin.version>3.1.1</maven-deploy-plugin.version>
<maven-release-plugin.version>2.5.3</maven-release-plugin.version>
<mockito.version>5.2.0</mockito.version>
<parquet.version>1.12.2</parquet.version>
<protobuf.version>3.25.5</protobuf.version>
<scala12.version>2.12.20</scala12.version>
Expand Down Expand Up @@ -438,7 +439,19 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>4.8.0</version>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
<version>${mockito.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>

Expand Down
55 changes: 0 additions & 55 deletions rfc/template.md

This file was deleted.

4 changes: 4 additions & 0 deletions xtable-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -84,5 +84,9 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
@EqualsAndHashCode(callSuper = true)
@Getter
public class SourceTable extends ExternalTable {
/** The path to the data files, defaults to the metadataPath */
/** The path to the data files, defaults to the basePath */
@NonNull private final String dataPath;

@Builder(toBuilder = true)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.xtable.model.catalog;

/** Represents a catalog table identifier in a multi-level catalog system. */
public interface CatalogTableIdentifier {
/** Returns the string identifier for the table within its catalog context */
String getId();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.xtable.model.catalog;

/**
* Represents a hierarchical table identifier, often including catalog, database (or schema), and
* table names. Some catalogs may omit the catalog name.
*/
public interface HierarchicalTableIdentifier extends CatalogTableIdentifier {
/** @return the catalog name if present, otherwise null */
String getCatalogName();

/** @return the database (or schema) name; required */
String getDatabaseName();

/** @return the table name; required */
String getTableName();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.xtable.model.catalog;

import lombok.NonNull;
import lombok.Value;

/**
* An internal representation of a fully qualified table identifier within a catalog. The naming
* convention follows a three level hierarchy, few examples can be found below.
*
* <ul>
* <li>1. catalog.database.table
* <li>2. catalog.schema.table
* <li>3. database.schema.table
* </ul>
*
* We have selected the first naming convention and will interoperate among other catalogs following
* a different convention.
*/
@Value
public class ThreePartHierarchicalTableIdentifier implements HierarchicalTableIdentifier {
/**
* The top level hierarchy/namespace for organizing tables. Each catalog can have multiple
* databases/schemas. This is an optional field as many catalogs have a "default" catalog whose
* name varies depending on the catalogType.
*/
String catalogName;
/**
* Catalogs have the ability to group tables logically, databaseName is the identifier for such
* logical classification. The alternate names for this field include namespace, schemaName etc.
*/
@NonNull String databaseName;

/**
* The table name used when syncing the table to the catalog. NOTE: This name can be different
* from the table name in storage.
*/
@NonNull String tableName;

public ThreePartHierarchicalTableIdentifier(
String catalogName, @NonNull String databaseName, @NonNull String tableName) {
this.catalogName = catalogName;
this.databaseName = databaseName;
this.tableName = tableName;
}

public ThreePartHierarchicalTableIdentifier(String databaseName, String tableName) {
this(null, databaseName, tableName);
}

/**
* Constructs a new {@code CatalogTableIdentifier} from a hierarchical string identifier.
*
* <p>The identifier is expected to be in one of the following formats:
*
* <ul>
* <li>{@code database.table} (two parts, no catalog)
* <li>{@code catalog.database.table} (three parts)
* </ul>
*
* If the identifier does not match one of these formats, an {@link IllegalArgumentException} is
* thrown.
*
* @param hierarchicalTableIdentifier The hierarchical string identifier (e.g.,
* "myCatalog.myDatabase.myTable" or "myDatabase.myTable").
* @throws IllegalArgumentException If the provided string does not match a valid two-part or
* three-part identifier.
*/
public static ThreePartHierarchicalTableIdentifier fromDotSeparatedIdentifier(
String hierarchicalTableIdentifier) {
String[] parts = hierarchicalTableIdentifier.split("\\.");
if (parts.length == 2) {
return new ThreePartHierarchicalTableIdentifier(parts[0], parts[1]);
} else if (parts.length == 3) {
return new ThreePartHierarchicalTableIdentifier(parts[0], parts[1], parts[2]);
} else {
throw new IllegalArgumentException("Invalid table identifier " + hierarchicalTableIdentifier);
}
}

@Override
public String getId() {
if (catalogName != null && !catalogName.isEmpty()) {
return catalogName + "." + databaseName + "." + tableName;
}
return databaseName + "." + tableName;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.xtable.model.exception;

/** Exception thrown when refresh operation (updating table format metadata) in catalog fails. */
public class CatalogRefreshException extends InternalException {

public CatalogRefreshException(String message, Throwable e) {
super(ErrorCode.CATALOG_REFRESH_EXCEPTION, message, e);
}

public CatalogRefreshException(String message) {
super(ErrorCode.CATALOG_REFRESH_EXCEPTION, message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ public enum ErrorCode {
INVALID_SCHEMA(10006),
UNSUPPORTED_SCHEMA_TYPE(10007),
UNSUPPORTED_FEATURE(10008),
PARSE_EXCEPTION(10009);
PARSE_EXCEPTION(10009),
CATALOG_REFRESH_EXCEPTION(10010);

private final int errorCode;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import java.time.Duration;
import java.time.Instant;
import java.util.List;

import lombok.Builder;
import lombok.Value;
Expand All @@ -30,18 +31,20 @@
* @since 0.1
*/
@Value
@Builder
@Builder(toBuilder = true)
public class SyncResult {
// Mode used for the sync
SyncMode mode;
Instant lastInstantSynced;
Instant syncStartTime;
// Duration
Duration syncDuration;
// Status of the sync
SyncStatus status;
// Status of the tableFormat sync
SyncStatus tableFormatSyncStatus;
// The Sync Mode recommended for the next sync (Usually filled on an error)
SyncMode recommendedSyncMode;
// The sync status for each catalog.
List<CatalogSyncStatus> catalogSyncStatusList;

public enum SyncStatusCode {
SUCCESS,
Expand All @@ -57,6 +60,25 @@ public static class SyncStatus {
SyncStatus.builder().statusCode(SyncStatusCode.SUCCESS).build();
// Status code
SyncStatusCode statusCode;
// errorDetails if any
ErrorDetails errorDetails;
}

/** Represents status for catalog sync status operation */
@Value
@Builder
public static class CatalogSyncStatus {
// A user defined unique catalog identifier.
String catalogId;
// Status code
SyncStatusCode statusCode;
// errorDetails if any
ErrorDetails errorDetails;
}

@Value
@Builder
public static class ErrorDetails {
// error Message if any
String errorMessage;
// Readable description of the error
Expand Down
Loading

0 comments on commit fb724f0

Please sign in to comment.