Skip to content

Commit

Permalink
[#4599][#4809] feat(core): Add tag support for column (#5186)
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

This PR proposes to add tag support for columns.

### Why are the changes needed?

With this PR, user could add and remove tags in the column level.

Fix: #4599 
Fix: #4809 

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Add new UT and IT to test.

---------

Co-authored-by: Qi Yu <[email protected]>
  • Loading branch information
jerryshao and yuqi1129 authored Oct 25, 2024
1 parent 13c4625 commit a89a1ca
Show file tree
Hide file tree
Showing 28 changed files with 929 additions and 145 deletions.
9 changes: 9 additions & 0 deletions api/src/main/java/org/apache/gravitino/rel/Column.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.gravitino.rel.expressions.Expression;
import org.apache.gravitino.rel.expressions.FunctionExpression;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.tag.SupportsTags;

/**
* An interface representing a column of a {@link Table}. It defines basic properties of a column,
Expand Down Expand Up @@ -71,6 +72,14 @@ public interface Column {
*/
Expression defaultValue();

/**
* @return the {@link SupportsTags} if the column supports tag operations.
* @throws UnsupportedOperationException if the column does not support tag operations.
*/
default SupportsTags supportsTags() {
throw new UnsupportedOperationException("Column does not support tag operations.");
}

/**
* Create a {@link Column} instance.
*
Expand Down
87 changes: 87 additions & 0 deletions api/src/test/java/org/apache/gravitino/TestMetadataObjects.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino;

import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

public class TestMetadataObjects {

@Test
public void testColumnObject() {
MetadataObject columnObject =
MetadataObjects.of("catalog.schema.table", "c1", MetadataObject.Type.COLUMN);
Assertions.assertEquals("catalog.schema.table", columnObject.parent());
Assertions.assertEquals("c1", columnObject.name());
Assertions.assertEquals(MetadataObject.Type.COLUMN, columnObject.type());
Assertions.assertEquals("catalog.schema.table.c1", columnObject.fullName());

MetadataObject columnObject2 =
MetadataObjects.of(
Lists.newArrayList("catalog", "schema", "table", "c2"), MetadataObject.Type.COLUMN);
Assertions.assertEquals("catalog.schema.table", columnObject2.parent());
Assertions.assertEquals("c2", columnObject2.name());
Assertions.assertEquals(MetadataObject.Type.COLUMN, columnObject2.type());
Assertions.assertEquals("catalog.schema.table.c2", columnObject2.fullName());

MetadataObject columnObject3 =
MetadataObjects.parse("catalog.schema.table.c3", MetadataObject.Type.COLUMN);
Assertions.assertEquals("catalog.schema.table", columnObject3.parent());
Assertions.assertEquals("c3", columnObject3.name());
Assertions.assertEquals(MetadataObject.Type.COLUMN, columnObject3.type());
Assertions.assertEquals("catalog.schema.table.c3", columnObject3.fullName());

// Test parent
MetadataObject parent = MetadataObjects.parent(columnObject);
Assertions.assertEquals("catalog.schema.table", parent.fullName());
Assertions.assertEquals("catalog.schema", parent.parent());
Assertions.assertEquals("table", parent.name());
Assertions.assertEquals(MetadataObject.Type.TABLE, parent.type());

// Test incomplete name
Assertions.assertThrows(
IllegalArgumentException.class,
() -> MetadataObjects.parse("c1", MetadataObject.Type.COLUMN));
Assertions.assertThrows(
IllegalArgumentException.class,
() -> MetadataObjects.parse("catalog", MetadataObject.Type.COLUMN));
Assertions.assertThrows(
IllegalArgumentException.class,
() -> MetadataObjects.parse("catalog.schema", MetadataObject.Type.COLUMN));
Assertions.assertThrows(
IllegalArgumentException.class,
() -> MetadataObjects.parse("catalog.schema.table", MetadataObject.Type.COLUMN));

// Test incomplete name list
Assertions.assertThrows(
IllegalArgumentException.class,
() -> MetadataObjects.of(Lists.newArrayList("catalog"), MetadataObject.Type.COLUMN));
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
MetadataObjects.of(
Lists.newArrayList("catalog", "schema"), MetadataObject.Type.COLUMN));
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
MetadataObjects.of(
Lists.newArrayList("catalog", "schema", "table"), MetadataObject.Type.COLUMN));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -291,135 +291,144 @@ private void assertColumns(Table table) {
Column[] columns = table.columns();
Assertions.assertEquals(11, columns.length);
if (table.name().endsWith("_rt") || table.name().endsWith("_ro")) {
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_commit_time")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[0]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_commit_seqno")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[1]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_record_key")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[2]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_partition_path")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[3]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_file_name")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[4]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("ts")
.withDataType(Types.LongType.get())
.withComment("")
.build(),
columns[5]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("uuid")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[6]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("rider")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[7]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("driver")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[8]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("fare")
.withDataType(Types.DoubleType.get())
.withComment("")
.build(),
columns[9]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("city")
.withDataType(Types.StringType.get())
.withComment("")
.build(),
columns[10]);
} else {
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_commit_time")
.withDataType(Types.StringType.get())
.build(),
columns[0]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_commit_seqno")
.withDataType(Types.StringType.get())
.build(),
columns[1]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_record_key")
.withDataType(Types.StringType.get())
.build(),
columns[2]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_partition_path")
.withDataType(Types.StringType.get())
.build(),
columns[3]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder()
.withName("_hoodie_file_name")
.withDataType(Types.StringType.get())
.build(),
columns[4]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder().withName("ts").withDataType(Types.LongType.get()).build(),
columns[5]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder().withName("uuid").withDataType(Types.StringType.get()).build(),
columns[6]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder().withName("rider").withDataType(Types.StringType.get()).build(),
columns[7]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder().withName("driver").withDataType(Types.StringType.get()).build(),
columns[8]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder().withName("fare").withDataType(Types.DoubleType.get()).build(),
columns[9]);
Assertions.assertEquals(
assertColumn(
ColumnDTO.builder().withName("city").withDataType(Types.StringType.get()).build(),
columns[10]);
}
}

private void assertColumn(ColumnDTO columnDTO, Column column) {
Assertions.assertEquals(columnDTO.name(), column.name());
Assertions.assertEquals(columnDTO.dataType(), column.dataType());
Assertions.assertEquals(columnDTO.comment(), column.comment());
Assertions.assertEquals(columnDTO.nullable(), column.nullable());
Assertions.assertEquals(columnDTO.autoIncrement(), column.autoIncrement());
Assertions.assertEquals(columnDTO.defaultValue(), column.defaultValue());
}

private static void createHudiTables() {
sparkSession =
SparkSession.builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTable;
import org.apache.gravitino.catalog.lakehouse.iceberg.ops.IcebergCatalogWrapperHelper;
import org.apache.gravitino.client.GravitinoMetalake;
import org.apache.gravitino.dto.util.DTOConverters;
import org.apache.gravitino.exceptions.NoSuchSchemaException;
import org.apache.gravitino.exceptions.SchemaAlreadyExistsException;
import org.apache.gravitino.exceptions.TableAlreadyExistsException;
Expand Down Expand Up @@ -417,7 +416,7 @@ void testCreateAndLoadIcebergTable() {
Assertions.assertEquals(createdTable.columns().length, columns.length);

for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), createdTable.columns()[i]);
assertColumn(columns[i], createdTable.columns()[i]);
}

// TODO add partitioning and sort order check
Expand All @@ -434,7 +433,7 @@ void testCreateAndLoadIcebergTable() {
}
Assertions.assertEquals(loadTable.columns().length, columns.length);
for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), loadTable.columns()[i]);
assertColumn(columns[i], loadTable.columns()[i]);
}

Assertions.assertEquals(partitioning.length, loadTable.partitioning().length);
Expand Down Expand Up @@ -1257,4 +1256,13 @@ protected static void assertionsTableInfo(
Assertions.assertEquals(entry.getValue(), table.properties().get(entry.getKey()));
}
}

protected void assertColumn(Column expectedColumn, Column actualColumn) {
Assertions.assertEquals(expectedColumn.name(), actualColumn.name());
Assertions.assertEquals(expectedColumn.dataType(), actualColumn.dataType());
Assertions.assertEquals(expectedColumn.comment(), actualColumn.comment());
Assertions.assertEquals(expectedColumn.nullable(), actualColumn.nullable());
Assertions.assertEquals(expectedColumn.autoIncrement(), actualColumn.autoIncrement());
Assertions.assertEquals(expectedColumn.defaultValue(), actualColumn.defaultValue());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import org.apache.gravitino.catalog.lakehouse.paimon.ops.PaimonBackendCatalogWrapper;
import org.apache.gravitino.catalog.lakehouse.paimon.utils.CatalogUtils;
import org.apache.gravitino.client.GravitinoMetalake;
import org.apache.gravitino.dto.util.DTOConverters;
import org.apache.gravitino.exceptions.NoSuchSchemaException;
import org.apache.gravitino.exceptions.SchemaAlreadyExistsException;
import org.apache.gravitino.exceptions.TableAlreadyExistsException;
Expand Down Expand Up @@ -256,7 +255,7 @@ void testCreateAndLoadPaimonTable()
Assertions.assertEquals(createdTable.columns().length, columns.length);

for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), createdTable.columns()[i]);
assertColumn(columns[i], createdTable.columns()[i]);
}

Table loadTable = tableCatalog.loadTable(tableIdentifier);
Expand All @@ -269,7 +268,7 @@ void testCreateAndLoadPaimonTable()
}
Assertions.assertEquals(loadTable.columns().length, columns.length);
for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), loadTable.columns()[i]);
assertColumn(columns[i], loadTable.columns()[i]);
}

// catalog load check
Expand Down Expand Up @@ -346,7 +345,7 @@ void testCreateAndLoadPaimonPartitionedTable()
Assertions.assertEquals(createdTable.columns().length, columns.length);

for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), createdTable.columns()[i]);
assertColumn(columns[i], createdTable.columns()[i]);
}

Table loadTable = tableCatalog.loadTable(tableIdentifier);
Expand Down Expand Up @@ -374,7 +373,7 @@ void testCreateAndLoadPaimonPartitionedTable()
Assertions.assertArrayEquals(partitionKeys, loadedPartitionKeys);
Assertions.assertEquals(loadTable.columns().length, columns.length);
for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), loadTable.columns()[i]);
assertColumn(columns[i], loadTable.columns()[i]);
}

// catalog load check
Expand Down Expand Up @@ -459,7 +458,7 @@ void testCreateAndLoadPaimonPrimaryKeyTable()
}
Assertions.assertEquals(createdTable.columns().length, columns.length);
for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), createdTable.columns()[i]);
assertColumn(columns[i], createdTable.columns()[i]);
}

Table loadTable = tableCatalog.loadTable(tableIdentifier);
Expand Down Expand Up @@ -488,7 +487,7 @@ void testCreateAndLoadPaimonPrimaryKeyTable()
}
Assertions.assertEquals(loadTable.columns().length, columns.length);
for (int i = 0; i < columns.length; i++) {
Assertions.assertEquals(DTOConverters.toDTO(columns[i]), loadTable.columns()[i]);
assertColumn(columns[i], loadTable.columns()[i]);
}

// catalog load check
Expand Down Expand Up @@ -969,4 +968,13 @@ protected void initSparkEnv() {
.enableHiveSupport()
.getOrCreate();
}

protected void assertColumn(Column expectedColumn, Column actualColumn) {
Assertions.assertEquals(expectedColumn.name(), actualColumn.name());
Assertions.assertEquals(expectedColumn.dataType(), actualColumn.dataType());
Assertions.assertEquals(expectedColumn.comment(), actualColumn.comment());
Assertions.assertEquals(expectedColumn.nullable(), actualColumn.nullable());
Assertions.assertEquals(expectedColumn.autoIncrement(), actualColumn.autoIncrement());
Assertions.assertEquals(expectedColumn.defaultValue(), actualColumn.defaultValue());
}
}
Loading

0 comments on commit a89a1ca

Please sign in to comment.