Skip to content

Commit

Permalink
Add tag support for model
Browse files Browse the repository at this point in the history
  • Loading branch information
jerryshao committed Jan 14, 2025
1 parent c6476b8 commit 30d53be
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ public static MetadataObject parent(MetadataObject object) {
case TABLE:
case FILESET:
case TOPIC:
case MODEL:
parentType = MetadataObject.Type.SCHEMA;
break;
case SCHEMA:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,35 @@
*/
package org.apache.gravitino.client;

import com.google.common.collect.Lists;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.Audit;
import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.MetadataObjects;
import org.apache.gravitino.Namespace;
import org.apache.gravitino.authorization.SupportsRoles;
import org.apache.gravitino.dto.model.ModelDTO;
import org.apache.gravitino.exceptions.NoSuchTagException;
import org.apache.gravitino.exceptions.TagAlreadyAssociatedException;
import org.apache.gravitino.model.Model;
import org.apache.gravitino.tag.SupportsTags;
import org.apache.gravitino.tag.Tag;

/** Represents a generic model. */
class GenericModel implements Model {
class GenericModel implements Model, SupportsTags {

private final ModelDTO modelDTO;

GenericModel(ModelDTO modelDTO) {
private final MetadataObjectTagOperations objectTagOperations;

GenericModel(ModelDTO modelDTO, RESTClient restClient, Namespace modelNs) {
this.modelDTO = modelDTO;
List<String> modelFullName =
Lists.newArrayList(modelNs.level(1), modelNs.level(2), modelDTO.name());
MetadataObject modelObject = MetadataObjects.of(modelFullName, MetadataObject.Type.MODEL);
this.objectTagOperations =
new MetadataObjectTagOperations(modelNs.level(0), modelObject, restClient);
}

@Override
Expand Down Expand Up @@ -61,7 +76,7 @@ public int latestVersion() {

@Override
public SupportsTags supportsTags() {
throw new UnsupportedOperationException("Not supported yet.");
return this;
}

@Override
Expand All @@ -86,4 +101,25 @@ public boolean equals(Object o) {
public int hashCode() {
return modelDTO.hashCode();
}

@Override
public String[] listTags() {
return objectTagOperations.listTags();
}

@Override
public Tag[] listTagsInfo() {
return objectTagOperations.listTagsInfo();
}

@Override
public Tag getTag(String name) throws NoSuchTagException {
return objectTagOperations.getTag(name);
}

@Override
public String[] associateTags(String[] tagsToAdd, String[] tagsToRemove)
throws TagAlreadyAssociatedException {
return objectTagOperations.associateTags(tagsToAdd, tagsToRemove);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public Model getModel(NameIdentifier ident) throws NoSuchModelException {
ErrorHandlers.modelErrorHandler());
resp.validate();

return new GenericModel(resp.getModel());
return new GenericModel(resp.getModel(), restClient, modelFullNs);
}

@Override
Expand All @@ -118,7 +118,7 @@ public Model registerModel(NameIdentifier ident, String comment, Map<String, Str
ErrorHandlers.modelErrorHandler());

resp.validate();
return new GenericModel(resp.getModel());
return new GenericModel(resp.getModel(), restClient, modelFullNs);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.gravitino.integration.test.container.HiveContainer;
import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.model.Model;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.types.Types;
Expand All @@ -60,6 +61,10 @@ public class TagIT extends BaseIT {
private static Schema schema;
private static Table table;

private static Catalog modelCatalog;
private static Schema modelSchema;
private static Model model;

private static Column column;

@BeforeAll
Expand Down Expand Up @@ -108,13 +113,41 @@ public void setUp() {
"comment",
Collections.emptyMap());
column = Arrays.stream(table.columns()).filter(c -> c.name().equals("col1")).findFirst().get();

// Create model catalog
String modelCatalogName = GravitinoITUtils.genRandomName("tag_it_model_catalog");
Assertions.assertFalse(metalake.catalogExists(modelCatalogName));
modelCatalog =
metalake.createCatalog(
modelCatalogName, Catalog.Type.MODEL, "comment", Collections.emptyMap());

// Create model schema
String modelSchemaName = GravitinoITUtils.genRandomName("tag_it_model_schema");
Assertions.assertFalse(modelCatalog.asSchemas().schemaExists(modelSchemaName));
modelSchema =
modelCatalog.asSchemas().createSchema(modelSchemaName, "comment", Collections.emptyMap());

// Create model
String modelName = GravitinoITUtils.genRandomName("tag_it_model");
Assertions.assertFalse(
modelCatalog.asModelCatalog().modelExists(NameIdentifier.of(modelSchemaName, modelName)));
model =
modelCatalog
.asModelCatalog()
.registerModel(
NameIdentifier.of(modelSchemaName, modelName), "comment", Collections.emptyMap());
}

@AfterAll
public void tearDown() {
relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name()));
relationalCatalog.asSchemas().dropSchema(schema.name(), true);
metalake.dropCatalog(relationalCatalog.name(), true);

modelCatalog.asModelCatalog().deleteModel(NameIdentifier.of(modelSchema.name(), model.name()));
modelCatalog.asSchemas().dropSchema(modelSchema.name(), true);
metalake.dropCatalog(modelCatalog.name(), true);

client.dropMetalake(metalakeName, true);

if (client != null) {
Expand Down Expand Up @@ -653,4 +686,84 @@ public void testAssociateAndDeleteTags() {
String[] associatedTags1 = relationalCatalog.supportsTags().listTags();
Assertions.assertArrayEquals(new String[] {tag2.name()}, associatedTags1);
}

@Test
public void testAssociateTagsToModel() {
Tag tag1 =
metalake.createTag(
GravitinoITUtils.genRandomName("tag_it_model_tag1"),
"comment1",
Collections.emptyMap());
Tag tag2 =
metalake.createTag(
GravitinoITUtils.genRandomName("tag_it_model_tag2"),
"comment2",
Collections.emptyMap());
Tag tag3 =
metalake.createTag(
GravitinoITUtils.genRandomName("tag_it_model_tag3"),
"comment3",
Collections.emptyMap());

// Associate tags to catalog
modelCatalog.supportsTags().associateTags(new String[] {tag1.name()}, null);

// Associate tags to schema
modelSchema.supportsTags().associateTags(new String[] {tag2.name()}, null);

// Associate tags to model
model.supportsTags().associateTags(new String[] {tag3.name()}, null);

// Test list associated tags for model
String[] tags1 = model.supportsTags().listTags();
Assertions.assertEquals(3, tags1.length);
Set<String> tagNames = Sets.newHashSet(tags1);
Assertions.assertTrue(tagNames.contains(tag1.name()));
Assertions.assertTrue(tagNames.contains(tag2.name()));
Assertions.assertTrue(tagNames.contains(tag3.name()));

// Test list associated tags with details for model
Tag[] tags2 = model.supportsTags().listTagsInfo();
Assertions.assertEquals(3, tags2.length);

Set<Tag> nonInheritedTags =
Arrays.stream(tags2).filter(tag -> !tag.inherited().get()).collect(Collectors.toSet());
Set<Tag> inheritedTags =
Arrays.stream(tags2).filter(tag -> tag.inherited().get()).collect(Collectors.toSet());

Assertions.assertEquals(1, nonInheritedTags.size());
Assertions.assertEquals(2, inheritedTags.size());
Assertions.assertTrue(nonInheritedTags.contains(tag3));
Assertions.assertTrue(inheritedTags.contains(tag1));
Assertions.assertTrue(inheritedTags.contains(tag2));

// Test get associated tag for model
Tag resultTag1 = model.supportsTags().getTag(tag1.name());
Assertions.assertEquals(tag1, resultTag1);
Assertions.assertTrue(resultTag1.inherited().get());

Tag resultTag2 = model.supportsTags().getTag(tag2.name());
Assertions.assertEquals(tag2, resultTag2);
Assertions.assertTrue(resultTag2.inherited().get());

Tag resultTag3 = model.supportsTags().getTag(tag3.name());
Assertions.assertEquals(tag3, resultTag3);
Assertions.assertFalse(resultTag3.inherited().get());

// Test get objects associated with tag
Assertions.assertEquals(1, tag1.associatedObjects().count());
Assertions.assertEquals(modelCatalog.name(), tag1.associatedObjects().objects()[0].name());
Assertions.assertEquals(
MetadataObject.Type.CATALOG, tag1.associatedObjects().objects()[0].type());

Assertions.assertEquals(1, tag2.associatedObjects().count());
Assertions.assertEquals(modelSchema.name(), tag2.associatedObjects().objects()[0].name());
Assertions.assertEquals(
MetadataObject.Type.SCHEMA, tag2.associatedObjects().objects()[0].type());

Assertions.assertEquals(1, tag3.associatedObjects().count());
Assertions.assertEquals(model.name(), tag3.associatedObjects().objects()[0].name());
Assertions.assertEquals(
MetadataObject.Type.MODEL, tag3.associatedObjects().objects()[0].type());
}
}
8 changes: 4 additions & 4 deletions docs/manage-tags-in-gravitino.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ the future versions.

:::info
1. Metadata objects are objects that are managed in Gravitino, such as `CATALOG`, `SCHEMA`, `TABLE`,
`COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, etc. A metadata object is combined by a `type` and a
`COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, `MODEL`, etc. A metadata object is combined by a `type` and a
comma-separated `name`. For example, a `CATAGLOG` object has a name "catalog1" with type
"CATALOG", a `SCHEMA` object has a name "catalog1.schema1" with type "SCHEMA", a `TABLE`
object has a name "catalog1.schema1.table1" with type "TABLE", a `COLUMN` object has a name
"catalog1.schema1.table1.column1" with type "COLUMN".
2. Currently, `CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, and `COLUMN` objects can be tagged.
2. Currently, `CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, `MODEL`, and `COLUMN` objects can be tagged.
3. Tags in Gravitino is inheritable, so listing tags of a metadata object will also list the
tags of its parent metadata objects. For example, listing tags of a `Table` will also list
the tags of its parent `Schema` and `Catalog`.
Expand Down Expand Up @@ -204,8 +204,8 @@ client.deleteTag("tag2");

## Tag associations

Gravitino allows you to associate and disassociate tags with metadata objects. Currently, only
`CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC` objects can be tagged.
Gravitino allows you to associate and disassociate tags with metadata objects. Currently,
`CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, `MODEL`, and `COLUMN` objects can be tagged.

### Associate and disassociate tags with a metadata object

Expand Down
4 changes: 2 additions & 2 deletions docs/open-api/tags.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,8 @@ components:
- "TABLE"
- "FILESET"
- "TOPIC"
- "ROLE"
- "METALAKE"
- "MODEL"
- "COLUMN"


requests:
Expand Down

0 comments on commit 30d53be

Please sign in to comment.