diff --git a/datacatalog/pkg/manager/impl/artifact_manager.go b/datacatalog/pkg/manager/impl/artifact_manager.go index 40f3f40538..577df39dbc 100644 --- a/datacatalog/pkg/manager/impl/artifact_manager.go +++ b/datacatalog/pkg/manager/impl/artifact_manager.go @@ -67,12 +67,10 @@ func (m *artifactManager) CreateArtifact(ctx context.Context, request *datacatal } ctx = contextutils.WithProjectDomain(ctx, artifact.Dataset.Project, artifact.Dataset.Domain) - datasetKey := transformers.FromDatasetID(artifact.Dataset) - // The dataset must exist for the artifact, let's verify that first - dataset, err := m.repo.DatasetRepo().Get(ctx, datasetKey) + dataset, err := m.repo.DatasetRepo().Get(ctx, request.GetArtifact().GetDataset()) if err != nil { - logger.Warnf(ctx, "Failed to get dataset for artifact creation %v, err: %v", datasetKey, err) + logger.Warnf(ctx, "Failed to get dataset for artifact creation %v, err: %v", request.GetArtifact().GetDataset(), err) m.systemMetrics.createFailureCounter.Inc(ctx) return nil, err } @@ -111,7 +109,7 @@ func (m *artifactManager) CreateArtifact(ctx context.Context, request *datacatal return nil, err } - err = m.repo.ArtifactRepo().Create(ctx, artifactModel) + err = m.repo.ArtifactRepo().Create(ctx, request.GetArtifact().GetDataset(), artifactModel) if err != nil { if errors.IsAlreadyExistsError(err) { logger.Warnf(ctx, "Artifact already exists key: %+v, err %v", artifact.Id, err) @@ -182,9 +180,8 @@ func (m *artifactManager) findArtifact(ctx context.Context, datasetID *datacatal key := queryHandle.GetArtifactId() if len(key) > 0 { logger.Debugf(ctx, "Get artifact by id %v", key) - artifactKey := transformers.ToArtifactKey(datasetID, key) var err error - artifactModel, err = m.repo.ArtifactRepo().Get(ctx, artifactKey) + artifactModel, err = m.repo.ArtifactRepo().Get(ctx, datasetID, key) if err != nil { if errors.IsDoesNotExistError(err) { @@ -249,10 +246,9 @@ func (m *artifactManager) ListArtifacts(ctx context.Context, request *datacatalo } // Verify the dataset exists before listing artifacts - datasetKey := transformers.FromDatasetID(request.Dataset) - dataset, err := m.repo.DatasetRepo().Get(ctx, datasetKey) + _, err = m.repo.DatasetRepo().Get(ctx, request.GetDataset()) if err != nil { - logger.Warnf(ctx, "Failed to get dataset for listing artifacts %v, err: %v", datasetKey, err) + logger.Warnf(ctx, "Failed to get dataset for listing artifacts %v, err: %v", request.GetDataset(), err) m.systemMetrics.listFailureCounter.Inc(ctx) return nil, err } @@ -273,7 +269,7 @@ func (m *artifactManager) ListArtifacts(ctx context.Context, request *datacatalo } // Perform the list with the dataset and listInput filters - artifactModels, err := m.repo.ArtifactRepo().List(ctx, dataset.DatasetKey, listInput) + artifactModels, err := m.repo.ArtifactRepo().List(ctx, request.GetDataset(), listInput) if err != nil { logger.Errorf(ctx, "Unable to list Artifacts err: %v", err) m.systemMetrics.listFailureCounter.Inc(ctx) @@ -381,7 +377,7 @@ func (m *artifactManager) UpdateArtifact(ctx context.Context, request *datacatal artifactModel.ArtifactData = artifactDataModels logger.Debugf(ctx, "Updating ArtifactModel with %+v", artifactModel) - err = m.repo.ArtifactRepo().Update(ctx, artifactModel) + err = m.repo.ArtifactRepo().Update(ctx, request.GetDataset(), artifactModel) if err != nil { if errors.IsDoesNotExistError(err) { logger.Warnf(ctx, "Artifact does not exist key: %+v, err %v", artifact.Id, err) diff --git a/datacatalog/pkg/manager/impl/artifact_manager_test.go b/datacatalog/pkg/manager/impl/artifact_manager_test.go index 420c9bd4fd..73931ea72b 100644 --- a/datacatalog/pkg/manager/impl/artifact_manager_test.go +++ b/datacatalog/pkg/manager/impl/artifact_manager_test.go @@ -5,9 +5,9 @@ import ( stdErrors "errors" "fmt" "os" + "reflect" "testing" "time" - "reflect" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" @@ -15,12 +15,13 @@ import ( "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/flyteorg/flyte/datacatalog/pkg/repositories/transformers" + "github.com/flyteorg/flyte/datacatalog/pkg/common" "github.com/flyteorg/flyte/datacatalog/pkg/errors" repoErrors "github.com/flyteorg/flyte/datacatalog/pkg/repositories/errors" "github.com/flyteorg/flyte/datacatalog/pkg/repositories/mocks" "github.com/flyteorg/flyte/datacatalog/pkg/repositories/models" + "github.com/flyteorg/flyte/datacatalog/pkg/repositories/transformers" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flyte/flytestdlib/contextutils" @@ -33,6 +34,14 @@ func init() { labeled.SetMetricKeys(contextutils.AppNameKey) } +var testDatasetID = &datacatalog.DatasetID{ + Project: "test-project", + Domain: "test-domain", + Name: "test-name", + Version: "test-version", + UUID: "test-uuid", +} + func createInmemoryDataStore(t testing.TB, scope mockScope.Scope) *storage.DataStore { cfg := storage.Config{ Type: storage.TypeMemory, @@ -64,18 +73,11 @@ func getTestTimestamp() time.Time { } func getTestArtifact() *datacatalog.Artifact { - datasetID := &datacatalog.DatasetID{ - Project: "test-project", - Domain: "test-domain", - Name: "test-name", - Version: "test-version", - UUID: "test-uuid", - } createdAt, _ := ptypes.TimestampProto(getTestTimestamp()) return &datacatalog.Artifact{ Id: "test-id", - Dataset: datasetID, + Dataset: testDatasetID, Metadata: &datacatalog.Metadata{ KeyMap: map[string]string{"key1": "value1"}, }, @@ -90,7 +92,7 @@ func getTestArtifact() *datacatalog.Artifact { {Key: "key2", Value: "value2"}, }, Tags: []*datacatalog.Tag{ - {Name: "test-tag", Dataset: datasetID, ArtifactId: "test-id"}, + {Name: "test-tag", Dataset: testDatasetID, ArtifactId: "test-id"}, }, CreatedAt: createdAt, } @@ -197,18 +199,18 @@ func TestCreateArtifact(t *testing.T) { ctx := context.Background() dcRepo := newMockDataCatalogRepo() + expectedArtifact := getTestArtifact() dcRepo.MockDatasetRepo.On("Get", mock.Anything, - mock.MatchedBy(func(dataset models.DatasetKey) bool { - return dataset.Project == expectedDataset.Id.Project && - dataset.Domain == expectedDataset.Id.Domain && - dataset.Name == expectedDataset.Id.Name && - dataset.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) })).Return(mockDatasetModel, nil) dcRepo.MockArtifactRepo.On("Create", mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.MatchedBy(func(artifact models.Artifact) bool { - expectedArtifact := getTestArtifact() return artifact.ArtifactID == expectedArtifact.Id && artifact.SerializedMetadata != nil && len(artifact.ArtifactData) == len(expectedArtifact.Data) && @@ -289,10 +291,13 @@ func TestCreateArtifact(t *testing.T) { dcRepo.MockDatasetRepo.On("Get", mock.Anything, mock.Anything).Return(mockDatasetModel, nil) + expectedArtifact := getTestArtifact() dcRepo.MockArtifactRepo.On("Create", mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.MatchedBy(func(artifact models.Artifact) bool { - expectedArtifact := getTestArtifact() return artifact.ArtifactID == expectedArtifact.Id && artifact.SerializedMetadata != nil && len(artifact.ArtifactData) == len(expectedArtifact.Data) && @@ -346,7 +351,7 @@ func TestCreateArtifact(t *testing.T) { dcRepo.MockDatasetRepo.On("Get", mock.Anything, mock.Anything).Return(mockDatasetModel, nil) artifact := getTestArtifact() artifact.Partitions = []*datacatalog.Partition{} - dcRepo.MockArtifactRepo.On("Create", mock.Anything, mock.Anything).Return(nil) + dcRepo.MockArtifactRepo.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(nil) request := &datacatalog.CreateArtifactRequest{Artifact: artifact} artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) @@ -390,13 +395,9 @@ func TestGetArtifact(t *testing.T) { t.Run("Get by Id", func(t *testing.T) { dcRepo.MockArtifactRepo.On("Get", mock.Anything, - mock.MatchedBy(func(artifactKey models.ArtifactKey) bool { - return artifactKey.ArtifactID == expectedArtifact.Id && - artifactKey.DatasetProject == expectedArtifact.Dataset.Project && - artifactKey.DatasetDomain == expectedArtifact.Dataset.Domain && - artifactKey.DatasetVersion == expectedArtifact.Dataset.Version && - artifactKey.DatasetName == expectedArtifact.Dataset.Name - })).Return(mockArtifactModel, nil) + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), expectedArtifact.Id).Return(mockArtifactModel, nil) artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) artifactResponse, err := artifactManager.GetArtifact(ctx, &datacatalog.GetArtifactRequest{ @@ -541,11 +542,8 @@ func TestListArtifact(t *testing.T) { } dcRepo.MockDatasetRepo.On("Get", mock.Anything, - mock.MatchedBy(func(dataset models.DatasetKey) bool { - return dataset.Project == expectedDataset.Id.Project && - dataset.Domain == expectedDataset.Id.Domain && - dataset.Name == expectedDataset.Id.Name && - dataset.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) })).Return(mockDatasetModel, nil) mockArtifacts := []models.Artifact{ @@ -554,11 +552,8 @@ func TestListArtifact(t *testing.T) { } dcRepo.MockArtifactRepo.On("List", mock.Anything, - mock.MatchedBy(func(dataset models.DatasetKey) bool { - return dataset.Project == expectedDataset.Id.Project && - dataset.Domain == expectedDataset.Id.Domain && - dataset.Name == expectedDataset.Id.Name && - dataset.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) }), mock.MatchedBy(func(listInput models.ListModelsInput) bool { return len(listInput.ModelFilters) == 3 && @@ -582,11 +577,8 @@ func TestListArtifact(t *testing.T) { filter := &datacatalog.FilterExpression{Filters: nil} dcRepo.MockDatasetRepo.On("Get", mock.Anything, - mock.MatchedBy(func(dataset models.DatasetKey) bool { - return dataset.Project == expectedDataset.Id.Project && - dataset.Domain == expectedDataset.Id.Domain && - dataset.Name == expectedDataset.Id.Name && - dataset.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) })).Return(mockDatasetModel, nil) mockArtifacts := []models.Artifact{ @@ -594,11 +586,8 @@ func TestListArtifact(t *testing.T) { mockArtifactModel, } dcRepo.MockArtifactRepo.On("List", mock.Anything, - mock.MatchedBy(func(dataset models.DatasetKey) bool { - return dataset.Project == expectedDataset.Id.Project && - dataset.Domain == expectedDataset.Id.Domain && - dataset.Name == expectedDataset.Id.Name && - dataset.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) }), mock.MatchedBy(func(listInput models.ListModelsInput) bool { return len(listInput.ModelFilters) == 0 @@ -632,13 +621,9 @@ func TestUpdateArtifact(t *testing.T) { dcRepo := newMockDataCatalogRepo() dcRepo.MockArtifactRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(artifactKey models.ArtifactKey) bool { - return artifactKey.ArtifactID == expectedArtifact.Id && - artifactKey.DatasetProject == expectedArtifact.Dataset.Project && - artifactKey.DatasetDomain == expectedArtifact.Dataset.Domain && - artifactKey.DatasetName == expectedArtifact.Dataset.Name && - artifactKey.DatasetVersion == expectedArtifact.Dataset.Version - })).Return(mockArtifactModel, nil) + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), expectedArtifact.Id).Return(mockArtifactModel, nil) metaData := &datacatalog.Metadata{ KeyMap: map[string]string{"key2": "value2"}, @@ -648,6 +633,9 @@ func TestUpdateArtifact(t *testing.T) { dcRepo.MockArtifactRepo.On("Update", mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.MatchedBy(func(artifact models.Artifact) bool { return artifact.ArtifactID == expectedArtifact.Id && artifact.ArtifactKey.DatasetProject == expectedArtifact.Dataset.Project && @@ -657,7 +645,6 @@ func TestUpdateArtifact(t *testing.T) { reflect.DeepEqual(artifact.SerializedMetadata, serializedMetadata) })).Return(nil) - request := &datacatalog.UpdateArtifactRequest{ Dataset: expectedDataset.Id, QueryHandle: &datacatalog.UpdateArtifactRequest_ArtifactId{ @@ -723,6 +710,9 @@ func TestUpdateArtifact(t *testing.T) { dcRepo := newMockDataCatalogRepo() dcRepo.MockArtifactRepo.On("Update", mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.MatchedBy(func(artifact models.Artifact) bool { return artifact.ArtifactID == expectedArtifact.Id && artifact.ArtifactKey.DatasetProject == expectedArtifact.Dataset.Project && @@ -808,7 +798,9 @@ func TestUpdateArtifact(t *testing.T) { datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) dcRepo := newMockDataCatalogRepo() - dcRepo.MockArtifactRepo.On("Get", mock.Anything, mock.Anything).Return(models.Artifact{}, repoErrors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ + dcRepo.MockArtifactRepo.On("Get", mock.Anything, mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.Anything).Return(models.Artifact{}, repoErrors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ Dataset: expectedDataset.Id, Id: expectedArtifact.Id, })) diff --git a/datacatalog/pkg/manager/impl/dataset_manager.go b/datacatalog/pkg/manager/impl/dataset_manager.go index 0db84d6360..81ad1ee3f8 100644 --- a/datacatalog/pkg/manager/impl/dataset_manager.go +++ b/datacatalog/pkg/manager/impl/dataset_manager.go @@ -78,7 +78,7 @@ func (dm *datasetManager) CreateDataset(ctx context.Context, request *datacatalo return nil, err } - err = dm.repo.DatasetRepo().Create(ctx, *datasetModel) + err = dm.repo.DatasetRepo().Create(ctx, request.GetDataset().GetId(), *datasetModel) if err != nil { if errors.IsAlreadyExistsError(err) { logger.Warnf(ctx, "Dataset already exists key: %+v, err %v", request.Dataset, err) @@ -106,13 +106,11 @@ func (dm *datasetManager) GetDataset(ctx context.Context, request *datacatalog.G dm.systemMetrics.validationErrorCounter.Inc(ctx) return nil, err } - - datasetKey := transformers.FromDatasetID(request.Dataset) - datasetModel, err := dm.repo.DatasetRepo().Get(ctx, datasetKey) + datasetModel, err := dm.repo.DatasetRepo().Get(ctx, request.GetDataset()) if err != nil { if errors.IsDoesNotExistError(err) { - logger.Warnf(ctx, "Dataset does not exist key: %+v, err %v", datasetKey, err) + logger.Warnf(ctx, "Dataset does not exist key: %+v, err %v", request.GetDataset(), err) dm.systemMetrics.doesNotExistCounter.Inc(ctx) } else { logger.Errorf(ctx, "Unable to get dataset request %+v err: %v", request, err) diff --git a/datacatalog/pkg/manager/impl/dataset_manager_test.go b/datacatalog/pkg/manager/impl/dataset_manager_test.go index 2ebd107304..c3fa768d0a 100644 --- a/datacatalog/pkg/manager/impl/dataset_manager_test.go +++ b/datacatalog/pkg/manager/impl/dataset_manager_test.go @@ -56,6 +56,9 @@ func TestCreateDataset(t *testing.T) { datasetManager := NewDatasetManager(dcRepo, nil, mockScope.NewTestScope()) dcRepo.MockDatasetRepo.On("Create", mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.MatchedBy(func(dataset models.Dataset) bool { return dataset.Name == expectedDataset.Id.Name && @@ -77,6 +80,9 @@ func TestCreateDataset(t *testing.T) { datasetManager := NewDatasetManager(dcRepo, nil, mockScope.NewTestScope()) dcRepo.MockDatasetRepo.On("Create", mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), mock.MatchedBy(func(dataset models.Dataset) bool { return dataset.Name == expectedDataset.Id.Name && @@ -117,7 +123,7 @@ func TestCreateDataset(t *testing.T) { datasetManager := NewDatasetManager(dcRepo, nil, mockScope.NewTestScope()) dcRepo.MockDatasetRepo.On("Create", - mock.Anything, + mock.Anything, mock.Anything, mock.Anything).Return(status.Error(codes.AlreadyExists, "test already exists")) request := &datacatalog.CreateDatasetRequest{ Dataset: getTestDataset(), @@ -160,12 +166,8 @@ func TestGetDataset(t *testing.T) { dcRepo.MockDatasetRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(datasetKey models.DatasetKey) bool { - - return datasetKey.Name == expectedDataset.Id.Name && - datasetKey.Project == expectedDataset.Id.Project && - datasetKey.Domain == expectedDataset.Id.Domain && - datasetKey.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) })).Return(*datasetModelResponse, nil) request := &datacatalog.GetDatasetRequest{Dataset: getTestDataset().Id} datasetResponse, err := datasetManager.GetDataset(context.Background(), request) @@ -181,12 +183,8 @@ func TestGetDataset(t *testing.T) { dcRepo.MockDatasetRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(datasetKey models.DatasetKey) bool { - - return datasetKey.Name == expectedDataset.Id.Name && - datasetKey.Project == expectedDataset.Id.Project && - datasetKey.Domain == expectedDataset.Id.Domain && - datasetKey.Version == expectedDataset.Id.Version + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) })).Return(models.Dataset{}, errors.NewDataCatalogError(codes.NotFound, "dataset does not exist")) request := &datacatalog.GetDatasetRequest{Dataset: getTestDataset().Id} _, err := datasetManager.GetDataset(context.Background(), request) diff --git a/datacatalog/pkg/manager/impl/tag_manager.go b/datacatalog/pkg/manager/impl/tag_manager.go index 784af9164c..7ee23fd15a 100644 --- a/datacatalog/pkg/manager/impl/tag_manager.go +++ b/datacatalog/pkg/manager/impl/tag_manager.go @@ -46,16 +46,13 @@ func (m *tagManager) AddTag(ctx context.Context, request *datacatalog.AddTagRequ // verify the artifact and dataset exists before adding a tag to it datasetID := request.Tag.Dataset ctx = contextutils.WithProjectDomain(ctx, datasetID.Project, datasetID.Domain) - - datasetKey := transformers.FromDatasetID(datasetID) - dataset, err := m.repo.DatasetRepo().Get(ctx, datasetKey) + dataset, err := m.repo.DatasetRepo().Get(ctx, datasetID) if err != nil { m.systemMetrics.addTagFailureCounter.Inc(ctx) return nil, err } - artifactKey := transformers.ToArtifactKey(datasetID, request.Tag.ArtifactId) - _, err = m.repo.ArtifactRepo().Get(ctx, artifactKey) + _, err = m.repo.ArtifactRepo().Get(ctx, datasetID, request.Tag.ArtifactId) if err != nil { m.systemMetrics.addTagFailureCounter.Inc(ctx) return nil, err diff --git a/datacatalog/pkg/manager/impl/tag_manager_test.go b/datacatalog/pkg/manager/impl/tag_manager_test.go index 98e4b41dfd..8b3bfc74b8 100644 --- a/datacatalog/pkg/manager/impl/tag_manager_test.go +++ b/datacatalog/pkg/manager/impl/tag_manager_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" @@ -76,20 +77,13 @@ func TestAddTag(t *testing.T) { } dcRepo.MockArtifactRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(artifactKey models.ArtifactKey) bool { - return artifactKey.DatasetProject == expectedTag.DatasetProject && - artifactKey.DatasetDomain == expectedTag.DatasetDomain && - artifactKey.DatasetName == expectedTag.DatasetName && - artifactKey.DatasetVersion == expectedTag.DatasetVersion && - artifactKey.ArtifactID == expectedTag.ArtifactID - })).Return(artifact, nil) + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) + }), expectedTag.ArtifactID).Return(artifact, nil) dcRepo.MockDatasetRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(datasetKey models.DatasetKey) bool { - return datasetKey.Project == expectedTag.DatasetProject && - datasetKey.Domain == expectedTag.DatasetDomain && - datasetKey.Name == expectedTag.DatasetName && - datasetKey.Version == expectedTag.DatasetVersion + mock.MatchedBy(func(datasetID *datacatalog.DatasetID) bool { + return proto.Equal(testDatasetID, datasetID) })).Return(dataset, nil) tagManager := NewTagManager(dcRepo, nil, mockScope.NewTestScope()) diff --git a/datacatalog/pkg/repositories/gormimpl/artifact.go b/datacatalog/pkg/repositories/gormimpl/artifact.go index b5089123d1..46d3976291 100644 --- a/datacatalog/pkg/repositories/gormimpl/artifact.go +++ b/datacatalog/pkg/repositories/gormimpl/artifact.go @@ -29,7 +29,7 @@ func NewArtifactRepo(db *gorm.DB, errorTransformer errors.ErrorTransformer, scop } // Create the artifact in a transaction because ArtifactData will be created and associated along with it -func (h *artifactRepo) Create(ctx context.Context, artifact models.Artifact) error { +func (h *artifactRepo) Create(ctx context.Context, id *datacatalog.DatasetID, artifact models.Artifact) error { timer := h.repoMetrics.CreateDuration.Start(ctx) defer timer.Stop() @@ -50,7 +50,7 @@ func (h *artifactRepo) Create(ctx context.Context, artifact models.Artifact) err return nil } -func (h *artifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.Artifact, error) { +func (h *artifactRepo) Get(ctx context.Context, id *datacatalog.DatasetID, artifactID string) (models.Artifact, error) { timer := h.repoMetrics.GetDuration.Start(ctx) defer timer.Stop() @@ -63,19 +63,20 @@ func (h *artifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.A Order("artifacts.created_at DESC"). First( &artifact, - &models.Artifact{ArtifactKey: in}, + &models.Artifact{ArtifactKey: models.ArtifactKey{ + DatasetProject: id.Project, + DatasetDomain: id.Domain, + DatasetName: id.Name, + DatasetVersion: id.Version, + ArtifactID: artifactID, + }}, ) if result.Error != nil { if result.Error.Error() == gorm.ErrRecordNotFound.Error() { return models.Artifact{}, errors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ - Dataset: &datacatalog.DatasetID{ - Project: in.DatasetProject, - Domain: in.DatasetDomain, - Name: in.DatasetName, - Version: in.DatasetVersion, - }, - Id: in.ArtifactID, + Dataset: id, + Id: artifactID, }) } @@ -85,7 +86,7 @@ func (h *artifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.A return artifact, nil } -func (h *artifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) ([]models.Artifact, error) { +func (h *artifactRepo) List(ctx context.Context, id *datacatalog.DatasetID, in models.ListModelsInput) ([]models.Artifact, error) { timer := h.repoMetrics.ListDuration.Start(ctx) defer timer.Stop() @@ -93,7 +94,7 @@ func (h *artifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, i sourceEntity := common.Artifact // add filter for dataset - datasetUUIDFilter := NewGormValueFilter(common.Equal, "dataset_uuid", datasetKey.UUID) + datasetUUIDFilter := NewGormValueFilter(common.Equal, "dataset_uuid", id.UUID) datasetFilter := models.ModelFilter{ Entity: common.Artifact, ValueFilters: []models.ModelValueFilter{datasetUUIDFilter}, @@ -122,7 +123,7 @@ func (h *artifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, i // Update updates the given artifact and its associated ArtifactData in database. The ArtifactData entries are upserted // (ignoring conflicts, as no updates to the database model are to be expected) and any longer existing data is deleted. -func (h *artifactRepo) Update(ctx context.Context, artifact models.Artifact) error { +func (h *artifactRepo) Update(ctx context.Context, id *datacatalog.DatasetID, artifact models.Artifact) error { timer := h.repoMetrics.UpdateDuration.Start(ctx) defer timer.Stop() diff --git a/datacatalog/pkg/repositories/gormimpl/artifact_test.go b/datacatalog/pkg/repositories/gormimpl/artifact_test.go index 18b819d45f..2e56394fc2 100644 --- a/datacatalog/pkg/repositories/gormimpl/artifact_test.go +++ b/datacatalog/pkg/repositories/gormimpl/artifact_test.go @@ -24,6 +24,14 @@ func init() { labeled.SetMetricKeys(contextutils.AppNameKey) } +var testDatasetID = &datacatalog.DatasetID{ + Project: "testProject", + Domain: "testDomain", + Name: "testName", + Version: "testVersion", + UUID: "test-uuid", +} + func getTestArtifact() models.Artifact { return models.Artifact{ ArtifactKey: models.ArtifactKey{ @@ -154,7 +162,7 @@ func TestCreateArtifact(t *testing.T) { artifact.Partitions = partitions artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Create(context.Background(), artifact) + err := artifactRepo.Create(context.Background(), &datacatalog.DatasetID{}, artifact) assert.NoError(t, err) assert.True(t, artifactCreated) assert.Equal(t, 2, numArtifactDataCreated) @@ -181,16 +189,13 @@ func TestGetArtifact(t *testing.T) { `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) - getInput := models.ArtifactKey{ - DatasetProject: artifact.DatasetProject, - DatasetDomain: artifact.DatasetDomain, - DatasetName: artifact.DatasetName, - DatasetVersion: artifact.DatasetVersion, - ArtifactID: artifact.ArtifactID, - } - artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - response, err := artifactRepo.Get(context.Background(), getInput) + response, err := artifactRepo.Get(context.Background(), &datacatalog.DatasetID{ + Project: artifact.DatasetProject, + Domain: artifact.DatasetDomain, + Name: artifact.DatasetName, + Version: artifact.DatasetVersion, + }, artifact.ArtifactID) assert.NoError(t, err) assert.Equal(t, artifact.ArtifactID, response.ArtifactID) assert.Equal(t, artifact.DatasetProject, response.DatasetProject) @@ -223,12 +228,8 @@ func TestGetArtifactByID(t *testing.T) { `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) - getInput := models.ArtifactKey{ - ArtifactID: artifact.ArtifactID, - } - artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - response, err := artifactRepo.Get(context.Background(), getInput) + response, err := artifactRepo.Get(context.Background(), &datacatalog.DatasetID{}, artifact.ArtifactID) assert.NoError(t, err) assert.Equal(t, artifact.ArtifactID, response.ArtifactID) } @@ -239,17 +240,14 @@ func TestGetArtifactDoesNotExist(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true - getInput := models.ArtifactKey{ - DatasetProject: artifact.DatasetProject, - DatasetDomain: artifact.DatasetDomain, - DatasetName: artifact.DatasetName, - DatasetVersion: artifact.DatasetVersion, - ArtifactID: artifact.ArtifactID, - } - // by default mocket will return nil for any queries artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - _, err := artifactRepo.Get(context.Background(), getInput) + _, err := artifactRepo.Get(context.Background(), &datacatalog.DatasetID{ + Project: artifact.DatasetProject, + Domain: artifact.DatasetDomain, + Name: artifact.DatasetName, + Version: artifact.DatasetVersion, + }, artifact.ArtifactID) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) @@ -269,7 +267,7 @@ func TestCreateArtifactAlreadyExists(t *testing.T) { ) artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Create(context.Background(), artifact) + err := artifactRepo.Create(context.Background(), testDatasetID, artifact) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) @@ -311,7 +309,7 @@ func TestListArtifactsWithPartition(t *testing.T) { Limit: 10, SortParameter: NewGormSortParameter(datacatalog.PaginationOptions_CREATION_TIME, datacatalog.PaginationOptions_DESCENDING), } - artifacts, err := artifactRepo.List(context.Background(), dataset.DatasetKey, listInput) + artifacts, err := artifactRepo.List(context.Background(), testDatasetID, listInput) assert.NoError(t, err) assert.Len(t, artifacts, 1) assert.Equal(t, artifacts[0].ArtifactID, artifact.ArtifactID) @@ -343,7 +341,7 @@ func TestListArtifactsNoPartitions(t *testing.T) { Offset: 10, Limit: 10, } - artifacts, err := artifactRepo.List(context.Background(), dataset.DatasetKey, listInput) + artifacts, err := artifactRepo.List(context.Background(), testDatasetID, listInput) assert.NoError(t, err) assert.Len(t, artifacts, 1) assert.Equal(t, artifacts[0].ArtifactID, artifact.ArtifactID) @@ -395,7 +393,7 @@ func TestUpdateArtifact(t *testing.T) { } artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Update(ctx, updateInput) + err := artifactRepo.Update(ctx, testDatasetID, updateInput) assert.NoError(t, err) assert.True(t, artifactUpdated) assert.True(t, artifactDataDeleted) @@ -426,7 +424,7 @@ func TestUpdateArtifactDoesNotExist(t *testing.T) { } artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Update(ctx, updateInput) + err := artifactRepo.Update(ctx, testDatasetID, updateInput) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) @@ -462,7 +460,7 @@ func TestUpdateArtifactError(t *testing.T) { } artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Update(ctx, updateInput) + err := artifactRepo.Update(ctx, testDatasetID, updateInput) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) @@ -502,7 +500,7 @@ func TestUpdateArtifactError(t *testing.T) { } artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Update(ctx, updateInput) + err := artifactRepo.Update(ctx, testDatasetID, updateInput) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) @@ -549,7 +547,7 @@ func TestUpdateArtifactError(t *testing.T) { } artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := artifactRepo.Update(ctx, updateInput) + err := artifactRepo.Update(ctx, testDatasetID, updateInput) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) diff --git a/datacatalog/pkg/repositories/gormimpl/dataset.go b/datacatalog/pkg/repositories/gormimpl/dataset.go index eb560c4347..42ca687a6b 100644 --- a/datacatalog/pkg/repositories/gormimpl/dataset.go +++ b/datacatalog/pkg/repositories/gormimpl/dataset.go @@ -29,7 +29,7 @@ func NewDatasetRepo(db *gorm.DB, errorTransformer errors.ErrorTransformer, scope } // Create a Dataset model -func (h *dataSetRepo) Create(ctx context.Context, in models.Dataset) error { +func (h *dataSetRepo) Create(ctx context.Context, id *idl_datacatalog.DatasetID, in models.Dataset) error { timer := h.repoMetrics.CreateDuration.Start(ctx) defer timer.Stop() @@ -41,25 +41,26 @@ func (h *dataSetRepo) Create(ctx context.Context, in models.Dataset) error { } // Get Dataset model -func (h *dataSetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Dataset, error) { +func (h *dataSetRepo) Get(ctx context.Context, id *idl_datacatalog.DatasetID) (models.Dataset, error) { timer := h.repoMetrics.GetDuration.Start(ctx) defer timer.Stop() var ds models.Dataset result := h.db.WithContext(ctx).Preload("PartitionKeys", func(db *gorm.DB) *gorm.DB { return db.WithContext(ctx).Order("partition_keys.created_at ASC") // preserve the order in which the partitions were created - }).First(&ds, &models.Dataset{DatasetKey: in}) + }).First(&ds, &models.Dataset{DatasetKey: models.DatasetKey{ + Project: id.Project, + Domain: id.Domain, + Name: id.Name, + Version: id.Version, + UUID: id.UUID, + }}) if result.Error != nil { - logger.Debugf(ctx, "Unable to find Dataset: [%+v], err: %v", in, result.Error) + logger.Debugf(ctx, "Unable to find Dataset: [%+v], err: %v", id, result.Error) if result.Error.Error() == gorm.ErrRecordNotFound.Error() { - return models.Dataset{}, errors.GetMissingEntityError("Dataset", &idl_datacatalog.DatasetID{ - Project: in.Project, - Domain: in.Domain, - Name: in.Name, - Version: in.Version, - }) + return models.Dataset{}, errors.GetMissingEntityError("Dataset", id) } return models.Dataset{}, h.errorTransformer.ToDataCatalogError(result.Error) } diff --git a/datacatalog/pkg/repositories/gormimpl/dataset_test.go b/datacatalog/pkg/repositories/gormimpl/dataset_test.go index 01475248fb..c5d95f378a 100644 --- a/datacatalog/pkg/repositories/gormimpl/dataset_test.go +++ b/datacatalog/pkg/repositories/gormimpl/dataset_test.go @@ -101,7 +101,7 @@ func TestCreateDatasetNoPartitions(t *testing.T) { dataset.PartitionKeys = nil datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := datasetRepo.Create(context.Background(), dataset) + err := datasetRepo.Create(context.Background(), testDatasetID, dataset) assert.NoError(t, err) assert.True(t, datasetCreated) } @@ -137,7 +137,7 @@ func TestCreateDataset(t *testing.T) { ) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := datasetRepo.Create(context.Background(), getTestDataset()) + err := datasetRepo.Create(context.Background(), testDatasetID, getTestDataset()) assert.NoError(t, err) assert.True(t, datasetCreated) assert.Equal(t, insertKeyQueryNum, 2) @@ -170,7 +170,7 @@ func TestGetDataset(t *testing.T) { GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1 ORDER BY partition_keys.created_at ASC`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) + actualDataset, err := datasetRepo.Get(context.Background(), testDatasetID) assert.NoError(t, err) assert.Equal(t, dataset.Project, actualDataset.Project) assert.Equal(t, dataset.Domain, actualDataset.Domain) @@ -202,7 +202,9 @@ func TestGetDatasetWithUUID(t *testing.T) { GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."uuid" = $1 ORDER BY "datasets"."created_at" LIMIT 1%!(EXTRA string=test-uuid)`).WithReply(expectedResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) + actualDataset, err := datasetRepo.Get(context.Background(), &datacatalog.DatasetID{ + UUID: getDatasetUUID(), + }) assert.NoError(t, err) assert.Equal(t, dataset.Project, actualDataset.Project) assert.Equal(t, dataset.Domain, actualDataset.Domain) @@ -225,7 +227,7 @@ func TestGetDatasetNotFound(t *testing.T) { GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND (("datasets"."project" = testProject) AND ("datasets"."name" = testName) AND ("datasets"."domain" = testDomain) AND ("datasets"."version" = testVersion)) ORDER BY "datasets"."id" ASC LIMIT 1`).WithReply(nil) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - _, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) + _, err := datasetRepo.Get(context.Background(), testDatasetID) assert.Error(t, err) notFoundErr, ok := err.(datacatalog_error.DataCatalogError) assert.True(t, ok) @@ -243,7 +245,7 @@ func TestCreateDatasetAlreadyExists(t *testing.T) { ) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) - err := datasetRepo.Create(context.Background(), getTestDataset()) + err := datasetRepo.Create(context.Background(), testDatasetID, getTestDataset()) assert.Error(t, err) dcErr, ok := err.(datacatalog_error.DataCatalogError) assert.True(t, ok) diff --git a/datacatalog/pkg/repositories/interfaces/artifact_repo.go b/datacatalog/pkg/repositories/interfaces/artifact_repo.go index ae56b99d7f..19772d58fe 100644 --- a/datacatalog/pkg/repositories/interfaces/artifact_repo.go +++ b/datacatalog/pkg/repositories/interfaces/artifact_repo.go @@ -4,13 +4,14 @@ import ( "context" "github.com/flyteorg/flyte/datacatalog/pkg/repositories/models" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog" ) //go:generate mockery -name=ArtifactRepo -output=../mocks -case=underscore type ArtifactRepo interface { - Create(ctx context.Context, in models.Artifact) error - Get(ctx context.Context, in models.ArtifactKey) (models.Artifact, error) - List(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) ([]models.Artifact, error) - Update(ctx context.Context, artifact models.Artifact) error + Create(ctx context.Context, id *datacatalog.DatasetID, in models.Artifact) error + Get(ctx context.Context, id *datacatalog.DatasetID, artifactID string) (models.Artifact, error) + List(ctx context.Context, id *datacatalog.DatasetID, in models.ListModelsInput) ([]models.Artifact, error) + Update(ctx context.Context, id *datacatalog.DatasetID, artifact models.Artifact) error } diff --git a/datacatalog/pkg/repositories/interfaces/dataset_repo.go b/datacatalog/pkg/repositories/interfaces/dataset_repo.go index 984adadfbe..27258b5cc4 100644 --- a/datacatalog/pkg/repositories/interfaces/dataset_repo.go +++ b/datacatalog/pkg/repositories/interfaces/dataset_repo.go @@ -4,12 +4,13 @@ import ( "context" "github.com/flyteorg/flyte/datacatalog/pkg/repositories/models" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog" ) //go:generate mockery -name=DatasetRepo -output=../mocks -case=underscore type DatasetRepo interface { - Create(ctx context.Context, in models.Dataset) error - Get(ctx context.Context, in models.DatasetKey) (models.Dataset, error) + Create(ctx context.Context, id *datacatalog.DatasetID, in models.Dataset) error + Get(ctx context.Context, id *datacatalog.DatasetID) (models.Dataset, error) List(ctx context.Context, in models.ListModelsInput) ([]models.Dataset, error) } diff --git a/datacatalog/pkg/repositories/mocks/artifact_repo.go b/datacatalog/pkg/repositories/mocks/artifact_repo.go index bf939f3e4b..e956f04572 100644 --- a/datacatalog/pkg/repositories/mocks/artifact_repo.go +++ b/datacatalog/pkg/repositories/mocks/artifact_repo.go @@ -5,6 +5,8 @@ package mocks import ( context "context" + datacatalog "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog" + mock "github.com/stretchr/testify/mock" models "github.com/flyteorg/flyte/datacatalog/pkg/repositories/models" @@ -23,8 +25,8 @@ func (_m ArtifactRepo_Create) Return(_a0 error) *ArtifactRepo_Create { return &ArtifactRepo_Create{Call: _m.Call.Return(_a0)} } -func (_m *ArtifactRepo) OnCreate(ctx context.Context, in models.Artifact) *ArtifactRepo_Create { - c_call := _m.On("Create", ctx, in) +func (_m *ArtifactRepo) OnCreate(ctx context.Context, id *datacatalog.DatasetID, in models.Artifact) *ArtifactRepo_Create { + c_call := _m.On("Create", ctx, id, in) return &ArtifactRepo_Create{Call: c_call} } @@ -33,13 +35,13 @@ func (_m *ArtifactRepo) OnCreateMatch(matchers ...interface{}) *ArtifactRepo_Cre return &ArtifactRepo_Create{Call: c_call} } -// Create provides a mock function with given fields: ctx, in -func (_m *ArtifactRepo) Create(ctx context.Context, in models.Artifact) error { - ret := _m.Called(ctx, in) +// Create provides a mock function with given fields: ctx, id, in +func (_m *ArtifactRepo) Create(ctx context.Context, id *datacatalog.DatasetID, in models.Artifact) error { + ret := _m.Called(ctx, id, in) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.Artifact) error); ok { - r0 = rf(ctx, in) + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.DatasetID, models.Artifact) error); ok { + r0 = rf(ctx, id, in) } else { r0 = ret.Error(0) } @@ -55,8 +57,8 @@ func (_m ArtifactRepo_Get) Return(_a0 models.Artifact, _a1 error) *ArtifactRepo_ return &ArtifactRepo_Get{Call: _m.Call.Return(_a0, _a1)} } -func (_m *ArtifactRepo) OnGet(ctx context.Context, in models.ArtifactKey) *ArtifactRepo_Get { - c_call := _m.On("Get", ctx, in) +func (_m *ArtifactRepo) OnGet(ctx context.Context, id *datacatalog.DatasetID, artifactID string) *ArtifactRepo_Get { + c_call := _m.On("Get", ctx, id, artifactID) return &ArtifactRepo_Get{Call: c_call} } @@ -65,20 +67,20 @@ func (_m *ArtifactRepo) OnGetMatch(matchers ...interface{}) *ArtifactRepo_Get { return &ArtifactRepo_Get{Call: c_call} } -// Get provides a mock function with given fields: ctx, in -func (_m *ArtifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.Artifact, error) { - ret := _m.Called(ctx, in) +// Get provides a mock function with given fields: ctx, id, artifactID +func (_m *ArtifactRepo) Get(ctx context.Context, id *datacatalog.DatasetID, artifactID string) (models.Artifact, error) { + ret := _m.Called(ctx, id, artifactID) var r0 models.Artifact - if rf, ok := ret.Get(0).(func(context.Context, models.ArtifactKey) models.Artifact); ok { - r0 = rf(ctx, in) + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.DatasetID, string) models.Artifact); ok { + r0 = rf(ctx, id, artifactID) } else { r0 = ret.Get(0).(models.Artifact) } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.ArtifactKey) error); ok { - r1 = rf(ctx, in) + if rf, ok := ret.Get(1).(func(context.Context, *datacatalog.DatasetID, string) error); ok { + r1 = rf(ctx, id, artifactID) } else { r1 = ret.Error(1) } @@ -94,8 +96,8 @@ func (_m ArtifactRepo_List) Return(_a0 []models.Artifact, _a1 error) *ArtifactRe return &ArtifactRepo_List{Call: _m.Call.Return(_a0, _a1)} } -func (_m *ArtifactRepo) OnList(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) *ArtifactRepo_List { - c_call := _m.On("List", ctx, datasetKey, in) +func (_m *ArtifactRepo) OnList(ctx context.Context, id *datacatalog.DatasetID, in models.ListModelsInput) *ArtifactRepo_List { + c_call := _m.On("List", ctx, id, in) return &ArtifactRepo_List{Call: c_call} } @@ -104,13 +106,13 @@ func (_m *ArtifactRepo) OnListMatch(matchers ...interface{}) *ArtifactRepo_List return &ArtifactRepo_List{Call: c_call} } -// List provides a mock function with given fields: ctx, datasetKey, in -func (_m *ArtifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) ([]models.Artifact, error) { - ret := _m.Called(ctx, datasetKey, in) +// List provides a mock function with given fields: ctx, id, in +func (_m *ArtifactRepo) List(ctx context.Context, id *datacatalog.DatasetID, in models.ListModelsInput) ([]models.Artifact, error) { + ret := _m.Called(ctx, id, in) var r0 []models.Artifact - if rf, ok := ret.Get(0).(func(context.Context, models.DatasetKey, models.ListModelsInput) []models.Artifact); ok { - r0 = rf(ctx, datasetKey, in) + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.DatasetID, models.ListModelsInput) []models.Artifact); ok { + r0 = rf(ctx, id, in) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]models.Artifact) @@ -118,8 +120,8 @@ func (_m *ArtifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.DatasetKey, models.ListModelsInput) error); ok { - r1 = rf(ctx, datasetKey, in) + if rf, ok := ret.Get(1).(func(context.Context, *datacatalog.DatasetID, models.ListModelsInput) error); ok { + r1 = rf(ctx, id, in) } else { r1 = ret.Error(1) } @@ -135,8 +137,8 @@ func (_m ArtifactRepo_Update) Return(_a0 error) *ArtifactRepo_Update { return &ArtifactRepo_Update{Call: _m.Call.Return(_a0)} } -func (_m *ArtifactRepo) OnUpdate(ctx context.Context, artifact models.Artifact) *ArtifactRepo_Update { - c_call := _m.On("Update", ctx, artifact) +func (_m *ArtifactRepo) OnUpdate(ctx context.Context, id *datacatalog.DatasetID, artifact models.Artifact) *ArtifactRepo_Update { + c_call := _m.On("Update", ctx, id, artifact) return &ArtifactRepo_Update{Call: c_call} } @@ -145,13 +147,13 @@ func (_m *ArtifactRepo) OnUpdateMatch(matchers ...interface{}) *ArtifactRepo_Upd return &ArtifactRepo_Update{Call: c_call} } -// Update provides a mock function with given fields: ctx, artifact -func (_m *ArtifactRepo) Update(ctx context.Context, artifact models.Artifact) error { - ret := _m.Called(ctx, artifact) +// Update provides a mock function with given fields: ctx, id, artifact +func (_m *ArtifactRepo) Update(ctx context.Context, id *datacatalog.DatasetID, artifact models.Artifact) error { + ret := _m.Called(ctx, id, artifact) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.Artifact) error); ok { - r0 = rf(ctx, artifact) + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.DatasetID, models.Artifact) error); ok { + r0 = rf(ctx, id, artifact) } else { r0 = ret.Error(0) } diff --git a/datacatalog/pkg/repositories/mocks/dataset_repo.go b/datacatalog/pkg/repositories/mocks/dataset_repo.go index d0a2079f26..c20f8c5f5c 100644 --- a/datacatalog/pkg/repositories/mocks/dataset_repo.go +++ b/datacatalog/pkg/repositories/mocks/dataset_repo.go @@ -5,6 +5,8 @@ package mocks import ( context "context" + datacatalog "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog" + mock "github.com/stretchr/testify/mock" models "github.com/flyteorg/flyte/datacatalog/pkg/repositories/models" @@ -23,8 +25,8 @@ func (_m DatasetRepo_Create) Return(_a0 error) *DatasetRepo_Create { return &DatasetRepo_Create{Call: _m.Call.Return(_a0)} } -func (_m *DatasetRepo) OnCreate(ctx context.Context, in models.Dataset) *DatasetRepo_Create { - c_call := _m.On("Create", ctx, in) +func (_m *DatasetRepo) OnCreate(ctx context.Context, id *datacatalog.DatasetID, in models.Dataset) *DatasetRepo_Create { + c_call := _m.On("Create", ctx, id, in) return &DatasetRepo_Create{Call: c_call} } @@ -33,13 +35,13 @@ func (_m *DatasetRepo) OnCreateMatch(matchers ...interface{}) *DatasetRepo_Creat return &DatasetRepo_Create{Call: c_call} } -// Create provides a mock function with given fields: ctx, in -func (_m *DatasetRepo) Create(ctx context.Context, in models.Dataset) error { - ret := _m.Called(ctx, in) +// Create provides a mock function with given fields: ctx, id, in +func (_m *DatasetRepo) Create(ctx context.Context, id *datacatalog.DatasetID, in models.Dataset) error { + ret := _m.Called(ctx, id, in) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.Dataset) error); ok { - r0 = rf(ctx, in) + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.DatasetID, models.Dataset) error); ok { + r0 = rf(ctx, id, in) } else { r0 = ret.Error(0) } @@ -55,8 +57,8 @@ func (_m DatasetRepo_Get) Return(_a0 models.Dataset, _a1 error) *DatasetRepo_Get return &DatasetRepo_Get{Call: _m.Call.Return(_a0, _a1)} } -func (_m *DatasetRepo) OnGet(ctx context.Context, in models.DatasetKey) *DatasetRepo_Get { - c_call := _m.On("Get", ctx, in) +func (_m *DatasetRepo) OnGet(ctx context.Context, id *datacatalog.DatasetID) *DatasetRepo_Get { + c_call := _m.On("Get", ctx, id) return &DatasetRepo_Get{Call: c_call} } @@ -65,20 +67,20 @@ func (_m *DatasetRepo) OnGetMatch(matchers ...interface{}) *DatasetRepo_Get { return &DatasetRepo_Get{Call: c_call} } -// Get provides a mock function with given fields: ctx, in -func (_m *DatasetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Dataset, error) { - ret := _m.Called(ctx, in) +// Get provides a mock function with given fields: ctx, id +func (_m *DatasetRepo) Get(ctx context.Context, id *datacatalog.DatasetID) (models.Dataset, error) { + ret := _m.Called(ctx, id) var r0 models.Dataset - if rf, ok := ret.Get(0).(func(context.Context, models.DatasetKey) models.Dataset); ok { - r0 = rf(ctx, in) + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.DatasetID) models.Dataset); ok { + r0 = rf(ctx, id) } else { r0 = ret.Get(0).(models.Dataset) } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.DatasetKey) error); ok { - r1 = rf(ctx, in) + if rf, ok := ret.Get(1).(func(context.Context, *datacatalog.DatasetID) error); ok { + r1 = rf(ctx, id) } else { r1 = ret.Error(1) } diff --git a/datacatalog/pkg/repositories/models/list.go b/datacatalog/pkg/repositories/models/list.go index 09a74ac4da..4d19ec58cb 100644 --- a/datacatalog/pkg/repositories/models/list.go +++ b/datacatalog/pkg/repositories/models/list.go @@ -1,6 +1,8 @@ package models -import "github.com/flyteorg/flyte/datacatalog/pkg/common" +import ( + "github.com/flyteorg/flyte/datacatalog/pkg/common" +) // Inputs to specify to list models type ListModelsInput struct { diff --git a/flyteadmin/pkg/clusterresource/controller.go b/flyteadmin/pkg/clusterresource/controller.go index 0ca60d7bd6..a875e72e07 100644 --- a/flyteadmin/pkg/clusterresource/controller.go +++ b/flyteadmin/pkg/clusterresource/controller.go @@ -718,8 +718,7 @@ func NewClusterResourceControllerFromConfig(ctx context.Context, scope promutils } dbScope := scope.NewSubScope("db") pluginRegistry.RegisterDefault(plugins.PluginIDNewRepositoryFunction, repositories.NewGormRepo) - var newRepoFunc repoInterfaces.NewRepositoryFunc - newRepoFunc = plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) + newRepoFunc := plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) repo := newRepoFunc( db, errors2.NewPostgresErrorTransformer(dbScope.NewSubScope("errors")), dbScope) diff --git a/flyteadmin/pkg/manager/impl/task_execution_manager.go b/flyteadmin/pkg/manager/impl/task_execution_manager.go index c9a435a37f..eac40c678f 100644 --- a/flyteadmin/pkg/manager/impl/task_execution_manager.go +++ b/flyteadmin/pkg/manager/impl/task_execution_manager.go @@ -249,6 +249,9 @@ func (m *TaskExecutionManager) ListTaskExecutions( nodeIDFilter, err := util.GetSingleValueEqualityFilter( common.NodeExecution, shared.NodeID, request.GetNodeExecutionId().NodeId) + if err != nil { + return nil, err + } filters, err := util.ParseFilters(request.Filters, common.TaskExecution) if err != nil { diff --git a/flyteadmin/pkg/manager/impl/util/single_task_execution.go b/flyteadmin/pkg/manager/impl/util/single_task_execution.go index 6c8fb111dd..31276d7268 100644 --- a/flyteadmin/pkg/manager/impl/util/single_task_execution.go +++ b/flyteadmin/pkg/manager/impl/util/single_task_execution.go @@ -72,8 +72,7 @@ func CreateOrGetWorkflowModel( ctx context.Context, db repositoryInterfaces.Repository, workflowManager interfaces.WorkflowInterface, namedEntityManager interfaces.NamedEntityInterface, taskIdentifier *core.Identifier, task *admin.Task) (*models.Workflow, error) { - var workflowIdentifier *core.Identifier - workflowIdentifier = proto.Clone(taskIdentifier).(*core.Identifier) + workflowIdentifier := proto.Clone(taskIdentifier).(*core.Identifier) workflowIdentifier.Name = generateWorkflowNameFromTask(taskIdentifier.Name) workflowIdentifier.ResourceType = core.ResourceType_WORKFLOW workflowModel, err := db.WorkflowRepo().Get(ctx, workflowIdentifier) @@ -151,8 +150,7 @@ func CreateOrGetLaunchPlan(ctx context.Context, var launchPlan *admin.LaunchPlan var err error - var launchPlanIdentifier *core.Identifier - launchPlanIdentifier = proto.Clone(taskIdentifier).(*core.Identifier) + launchPlanIdentifier := proto.Clone(taskIdentifier).(*core.Identifier) launchPlanIdentifier.Name = generateWorkflowNameFromTask(taskIdentifier.Name) launchPlanIdentifier.ResourceType = core.ResourceType_LAUNCH_PLAN launchPlan, err = GetLaunchPlan(ctx, db, launchPlanIdentifier) diff --git a/flyteadmin/pkg/manager/impl/workflow_manager.go b/flyteadmin/pkg/manager/impl/workflow_manager.go index 26e702c332..ab8b83df7e 100644 --- a/flyteadmin/pkg/manager/impl/workflow_manager.go +++ b/flyteadmin/pkg/manager/impl/workflow_manager.go @@ -73,7 +73,8 @@ func (w *WorkflowManager) getCompiledWorkflow( var tasks = make([]*core.CompiledTask, len(reqs.GetRequiredTaskIds())) for idx, taskID := range reqs.GetRequiredTaskIds() { - task, err := util.GetTask(ctx, w.db, &taskID) + taskIdentifier := taskID + task, err := util.GetTask(ctx, w.db, &taskIdentifier) if err != nil { logger.Debugf(ctx, "Failed to get task with id [%+v] when compiling workflow with id [%+v] with err %v", taskID, request.Id, err) @@ -84,8 +85,9 @@ func (w *WorkflowManager) getCompiledWorkflow( var launchPlans = make([]compiler.InterfaceProvider, len(reqs.GetRequiredLaunchPlanIds())) for idx, launchPlanID := range reqs.GetRequiredLaunchPlanIds() { + launchPlanIdentifier := launchPlanID var launchPlanModel models.LaunchPlan - launchPlanModel, err = util.GetLaunchPlanModel(ctx, w.db, &launchPlanID) + launchPlanModel, err = util.GetLaunchPlanModel(ctx, w.db, &launchPlanIdentifier) if err != nil { logger.Debugf(ctx, "Failed to get launch plan with id [%+v] when compiling workflow with id [%+v] with err %v", launchPlanID, request.Id, err) diff --git a/flyteadmin/pkg/repositories/gormimpl/common.go b/flyteadmin/pkg/repositories/gormimpl/common.go index 2fb4080b97..6c8136b546 100644 --- a/flyteadmin/pkg/repositories/gormimpl/common.go +++ b/flyteadmin/pkg/repositories/gormimpl/common.go @@ -2,7 +2,6 @@ package gormimpl import ( "fmt" - "k8s.io/apimachinery/pkg/util/sets" "google.golang.org/grpc/codes" "gorm.io/gorm" @@ -88,8 +87,6 @@ func ValidateListInput(input interfaces.ListResourceInput) adminErrors.FlyteAdmi return nil } -var executionEntities = sets.New[common.Entity](common.NodeExecution, common.TaskExecution) - // Returns equality filters initialized for identifier attributes (project, domain & name) // which can be optionally specified in requests. func getIdentifierFilters(entity common.Entity, identifier *admin.NamedEntityIdentifier) ([]common.InlineFilter, error) { diff --git a/flyteadmin/pkg/repositories/gormimpl/named_entity_repo.go b/flyteadmin/pkg/repositories/gormimpl/named_entity_repo.go index ab2ed6edfa..9b60e39cc2 100644 --- a/flyteadmin/pkg/repositories/gormimpl/named_entity_repo.go +++ b/flyteadmin/pkg/repositories/gormimpl/named_entity_repo.go @@ -82,29 +82,6 @@ func getSelectForNamedEntity(tableName string, resourceType core.ResourceType) [ } } -func getNamedEntityFilters(resourceType core.ResourceType, project string, domain string, name string) ([]common.InlineFilter, error) { - entity := common.ResourceTypeToEntity[resourceType] - - filters := make([]common.InlineFilter, 0) - projectFilter, err := common.NewSingleValueFilter(entity, common.Equal, Project, project) - if err != nil { - return nil, err - } - filters = append(filters, projectFilter) - domainFilter, err := common.NewSingleValueFilter(entity, common.Equal, Domain, domain) - if err != nil { - return nil, err - } - filters = append(filters, domainFilter) - nameFilter, err := common.NewSingleValueFilter(entity, common.Equal, Name, name) - if err != nil { - return nil, err - } - filters = append(filters, nameFilter) - - return filters, nil -} - // Implementation of NamedEntityRepoInterface. type NamedEntityRepo struct { db *gorm.DB diff --git a/flyteadmin/pkg/rpc/adminservice/base.go b/flyteadmin/pkg/rpc/adminservice/base.go index c484337b62..eb4d41f609 100644 --- a/flyteadmin/pkg/rpc/adminservice/base.go +++ b/flyteadmin/pkg/rpc/adminservice/base.go @@ -80,9 +80,7 @@ func NewAdminServer(ctx context.Context, pluginRegistry *plugins.Registry, confi logger.Fatal(ctx, err) } dbScope := adminScope.NewSubScope("database") - - var newRepoFunc repoInterfaces.NewRepositoryFunc - newRepoFunc = plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) + newRepoFunc := plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) repo := newRepoFunc( db, errors.NewPostgresErrorTransformer(adminScope.NewSubScope("errors")), dbScope) diff --git a/flyteadmin/pkg/rpc/signal_service.go b/flyteadmin/pkg/rpc/signal_service.go index facf446bfc..cf5cd7a146 100644 --- a/flyteadmin/pkg/rpc/signal_service.go +++ b/flyteadmin/pkg/rpc/signal_service.go @@ -68,8 +68,7 @@ func NewSignalServer(ctx context.Context, configuration runtimeIfaces.Configurat logger.Fatal(ctx, err) } dbScope := adminScope.NewSubScope("database") - var newRepoFunc repoInterfaces.NewRepositoryFunc - newRepoFunc = plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) + newRepoFunc := plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) repo := newRepoFunc( db, errors.NewPostgresErrorTransformer(adminScope.NewSubScope("errors")), dbScope) diff --git a/flyteadmin/scheduler/dbapi/event_scheduler_impl.go b/flyteadmin/scheduler/dbapi/event_scheduler_impl.go index 38e1e42e5e..7ec2c425d4 100644 --- a/flyteadmin/scheduler/dbapi/event_scheduler_impl.go +++ b/flyteadmin/scheduler/dbapi/event_scheduler_impl.go @@ -57,7 +57,7 @@ func (s *eventScheduler) AddSchedule(ctx context.Context, input interfaces.AddSc Version: input.Identifier.Version, }, } - err := s.db.SchedulableEntityRepo().Activate(ctx, modelInput) + err := s.db.SchedulableEntityRepo().Activate(ctx, &input.Identifier, modelInput) if err != nil { return err } @@ -68,12 +68,7 @@ func (s *eventScheduler) AddSchedule(ctx context.Context, input interfaces.AddSc func (s *eventScheduler) RemoveSchedule(ctx context.Context, input interfaces.RemoveScheduleInput) error { logger.Infof(ctx, "Received call to remove schedule [%+v]. Will deactivate it in the scheduler", input.Identifier) - err := s.db.SchedulableEntityRepo().Deactivate(ctx, models.SchedulableEntityKey{ - Project: input.Identifier.Project, - Domain: input.Identifier.Domain, - Name: input.Identifier.Name, - Version: input.Identifier.Version, - }) + err := s.db.SchedulableEntityRepo().Deactivate(ctx, &input.Identifier) if err != nil { return err diff --git a/flyteadmin/scheduler/dbapi/event_scheduler_impl_test.go b/flyteadmin/scheduler/dbapi/event_scheduler_impl_test.go index fc9df8912b..2083a714d2 100644 --- a/flyteadmin/scheduler/dbapi/event_scheduler_impl_test.go +++ b/flyteadmin/scheduler/dbapi/event_scheduler_impl_test.go @@ -37,7 +37,7 @@ func TestCreateScheduleInput(t *testing.T) { addScheduleInput, err := eventScheduler.CreateScheduleInput(context.Background(), nil, core.Identifier{ Project: "project", Domain: "domain", - Name: "scheduled_wroflow", + Name: "scheduled_workflow", Version: "v1", }, schedule) assert.Nil(t, err) @@ -54,7 +54,7 @@ func TestRemoveSchedule(t *testing.T) { Identifier: core.Identifier{ Project: "project", Domain: "domain", - Name: "scheduled_wroflow", + Name: "scheduled_workflow", Version: "v1", }, }) @@ -74,16 +74,17 @@ func TestAddSchedule(t *testing.T) { KickoffTimeInputArg: "kickoff_time", } + id := core.Identifier{ + Project: "project", + Domain: "domain", + Name: "scheduled_workflow", + Version: "v1", + } scheduleEntitiesRepo := db.SchedulableEntityRepo().(*schedMocks.SchedulableEntityRepoInterface) - scheduleEntitiesRepo.OnActivateMatch(mock.Anything, mock.Anything).Return(nil) + scheduleEntitiesRepo.OnActivateMatch(mock.Anything, &id, mock.Anything).Return(nil) err := eventScheduler.AddSchedule(context.Background(), interfaces.AddScheduleInput{ - Identifier: core.Identifier{ - Project: "project", - Domain: "domain", - Name: "scheduled_wroflow", - Version: "v1", - }, + Identifier: id, ScheduleExpression: schedule, }) assert.Nil(t, err) @@ -100,16 +101,17 @@ func TestAddSchedule(t *testing.T) { KickoffTimeInputArg: "kickoff_time", } + id := core.Identifier{ + Project: "project", + Domain: "domain", + Name: "scheduled_workflow", + Version: "v1", + } scheduleEntitiesRepo := db.SchedulableEntityRepo().(*schedMocks.SchedulableEntityRepoInterface) - scheduleEntitiesRepo.OnActivateMatch(mock.Anything, mock.Anything).Return(nil) + scheduleEntitiesRepo.OnActivateMatch(mock.Anything, &id, mock.Anything).Return(nil) err := eventScheduler.AddSchedule(context.Background(), interfaces.AddScheduleInput{ - Identifier: core.Identifier{ - Project: "project", - Domain: "domain", - Name: "scheduled_wroflow", - Version: "v1", - }, + Identifier: id, ScheduleExpression: schedule, }) assert.Nil(t, err) @@ -131,7 +133,7 @@ func TestAddSchedule(t *testing.T) { Identifier: core.Identifier{ Project: "project", Domain: "domain", - Name: "scheduled_wroflow", + Name: "scheduled_workflow", Version: "v1", }, ScheduleExpression: schedule, diff --git a/flyteadmin/scheduler/repositories/gormimpl/schedulable_entity_repo.go b/flyteadmin/scheduler/repositories/gormimpl/schedulable_entity_repo.go index 81a84659b5..799184afbb 100644 --- a/flyteadmin/scheduler/repositories/gormimpl/schedulable_entity_repo.go +++ b/flyteadmin/scheduler/repositories/gormimpl/schedulable_entity_repo.go @@ -21,7 +21,7 @@ type SchedulableEntityRepo struct { metrics gormMetrics } -func (r *SchedulableEntityRepo) Create(ctx context.Context, input models.SchedulableEntity) error { +func (r *SchedulableEntityRepo) Create(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) error { timer := r.metrics.GetDuration.Start() var record models.SchedulableEntity tx := r.db.Omit("id").FirstOrCreate(&record, input) @@ -32,16 +32,16 @@ func (r *SchedulableEntityRepo) Create(ctx context.Context, input models.Schedul return nil } -func (r *SchedulableEntityRepo) Activate(ctx context.Context, input models.SchedulableEntity) error { +func (r *SchedulableEntityRepo) Activate(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) error { var schedulableEntity models.SchedulableEntity timer := r.metrics.GetDuration.Start() // Find the existence of a scheduled entity tx := r.db.Where(&models.SchedulableEntity{ SchedulableEntityKey: models.SchedulableEntityKey{ - Project: input.Project, - Domain: input.Domain, - Name: input.Name, - Version: input.Version, + Project: id.Project, + Domain: id.Domain, + Name: id.Name, + Version: id.Version, }, }).Take(&schedulableEntity) timer.Stop() @@ -49,18 +49,18 @@ func (r *SchedulableEntityRepo) Activate(ctx context.Context, input models.Sched if tx.Error != nil { if errors.Is(tx.Error, gorm.ErrRecordNotFound) { // Not found and hence create one - return r.Create(ctx, input) + return r.Create(ctx, id, input) } return r.errorTransformer.ToFlyteAdminError(tx.Error) } // Activate the already existing schedule - return activateOrDeactivate(r, input.SchedulableEntityKey, true) + return activateOrDeactivate(r, id, true) } -func (r *SchedulableEntityRepo) Deactivate(ctx context.Context, ID models.SchedulableEntityKey) error { +func (r *SchedulableEntityRepo) Deactivate(ctx context.Context, id *core.Identifier) error { // Activate the schedule - return activateOrDeactivate(r, ID, false) + return activateOrDeactivate(r, id, false) } func (r *SchedulableEntityRepo) GetAll(ctx context.Context) ([]models.SchedulableEntity, error) { @@ -82,15 +82,15 @@ func (r *SchedulableEntityRepo) GetAll(ctx context.Context) ([]models.Schedulabl return schedulableEntities, nil } -func (r *SchedulableEntityRepo) Get(ctx context.Context, ID models.SchedulableEntityKey) (models.SchedulableEntity, error) { +func (r *SchedulableEntityRepo) Get(ctx context.Context, id *core.Identifier) (models.SchedulableEntity, error) { var schedulableEntity models.SchedulableEntity timer := r.metrics.GetDuration.Start() tx := r.db.Where(&models.SchedulableEntity{ SchedulableEntityKey: models.SchedulableEntityKey{ - Project: ID.Project, - Domain: ID.Domain, - Name: ID.Name, - Version: ID.Version, + Project: id.Project, + Domain: id.Domain, + Name: id.Name, + Version: id.Version, }, }).Take(&schedulableEntity) timer.Stop() @@ -98,12 +98,7 @@ func (r *SchedulableEntityRepo) Get(ctx context.Context, ID models.SchedulableEn if tx.Error != nil { if errors.Is(tx.Error, gorm.ErrRecordNotFound) { return models.SchedulableEntity{}, - adminErrors.GetMissingEntityError("schedulable entity", &core.Identifier{ - Project: ID.Project, - Domain: ID.Domain, - Name: ID.Name, - Version: ID.Version, - }) + adminErrors.GetMissingEntityError("schedulable entity", id) } return models.SchedulableEntity{}, r.errorTransformer.ToFlyteAdminError(tx.Error) } @@ -112,25 +107,20 @@ func (r *SchedulableEntityRepo) Get(ctx context.Context, ID models.SchedulableEn } // Helper function to activate and deactivate a schedule -func activateOrDeactivate(r *SchedulableEntityRepo, ID models.SchedulableEntityKey, activate bool) error { +func activateOrDeactivate(r *SchedulableEntityRepo, id *core.Identifier, activate bool) error { timer := r.metrics.GetDuration.Start() tx := r.db.Model(&models.SchedulableEntity{}).Where(&models.SchedulableEntity{ SchedulableEntityKey: models.SchedulableEntityKey{ - Project: ID.Project, - Domain: ID.Domain, - Name: ID.Name, - Version: ID.Version, + Project: id.Project, + Domain: id.Domain, + Name: id.Name, + Version: id.Version, }, }).Update("active", activate) timer.Stop() if tx.Error != nil { if errors.Is(tx.Error, gorm.ErrRecordNotFound) { - return adminErrors.GetMissingEntityError("schedulable entity", &core.Identifier{ - Project: ID.Project, - Domain: ID.Domain, - Name: ID.Name, - Version: ID.Version, - }) + return adminErrors.GetMissingEntityError("schedulable entity", id) } return r.errorTransformer.ToFlyteAdminError(tx.Error) } diff --git a/flyteadmin/scheduler/repositories/interfaces/schedulable_entity_repo.go b/flyteadmin/scheduler/repositories/interfaces/schedulable_entity_repo.go index 2b19767587..2332827371 100644 --- a/flyteadmin/scheduler/repositories/interfaces/schedulable_entity_repo.go +++ b/flyteadmin/scheduler/repositories/interfaces/schedulable_entity_repo.go @@ -2,6 +2,7 @@ package interfaces import ( "context" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flyteadmin/scheduler/repositories/models" ) @@ -12,16 +13,16 @@ import ( type SchedulableEntityRepoInterface interface { // Create a schedulable entity in the database store - Create(ctx context.Context, input models.SchedulableEntity) error + Create(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) error // Activate a schedulable entity in the database store. - Activate(ctx context.Context, input models.SchedulableEntity) error + Activate(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) error // Deactivate a schedulable entity in the database store. - Deactivate(ctx context.Context, ID models.SchedulableEntityKey) error + Deactivate(ctx context.Context, id *core.Identifier) error // Get a schedulable entity from the database store using the schedulable entity id. - Get(ctx context.Context, ID models.SchedulableEntityKey) (models.SchedulableEntity, error) + Get(ctx context.Context, id *core.Identifier) (models.SchedulableEntity, error) // GetAll Gets all the active schedulable entities from the db GetAll(ctx context.Context) ([]models.SchedulableEntity, error) diff --git a/flyteadmin/scheduler/repositories/mocks/schedulable_entity_repo_interface.go b/flyteadmin/scheduler/repositories/mocks/schedulable_entity_repo_interface.go index c9dc1e7e56..044f2dde11 100644 --- a/flyteadmin/scheduler/repositories/mocks/schedulable_entity_repo_interface.go +++ b/flyteadmin/scheduler/repositories/mocks/schedulable_entity_repo_interface.go @@ -5,6 +5,8 @@ package mocks import ( context "context" + core "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" + mock "github.com/stretchr/testify/mock" models "github.com/flyteorg/flyte/flyteadmin/scheduler/repositories/models" @@ -23,8 +25,8 @@ func (_m SchedulableEntityRepoInterface_Activate) Return(_a0 error) *Schedulable return &SchedulableEntityRepoInterface_Activate{Call: _m.Call.Return(_a0)} } -func (_m *SchedulableEntityRepoInterface) OnActivate(ctx context.Context, input models.SchedulableEntity) *SchedulableEntityRepoInterface_Activate { - c_call := _m.On("Activate", ctx, input) +func (_m *SchedulableEntityRepoInterface) OnActivate(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) *SchedulableEntityRepoInterface_Activate { + c_call := _m.On("Activate", ctx, id, input) return &SchedulableEntityRepoInterface_Activate{Call: c_call} } @@ -33,13 +35,13 @@ func (_m *SchedulableEntityRepoInterface) OnActivateMatch(matchers ...interface{ return &SchedulableEntityRepoInterface_Activate{Call: c_call} } -// Activate provides a mock function with given fields: ctx, input -func (_m *SchedulableEntityRepoInterface) Activate(ctx context.Context, input models.SchedulableEntity) error { - ret := _m.Called(ctx, input) +// Activate provides a mock function with given fields: ctx, id, input +func (_m *SchedulableEntityRepoInterface) Activate(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) error { + ret := _m.Called(ctx, id, input) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.SchedulableEntity) error); ok { - r0 = rf(ctx, input) + if rf, ok := ret.Get(0).(func(context.Context, *core.Identifier, models.SchedulableEntity) error); ok { + r0 = rf(ctx, id, input) } else { r0 = ret.Error(0) } @@ -55,8 +57,8 @@ func (_m SchedulableEntityRepoInterface_Create) Return(_a0 error) *SchedulableEn return &SchedulableEntityRepoInterface_Create{Call: _m.Call.Return(_a0)} } -func (_m *SchedulableEntityRepoInterface) OnCreate(ctx context.Context, input models.SchedulableEntity) *SchedulableEntityRepoInterface_Create { - c_call := _m.On("Create", ctx, input) +func (_m *SchedulableEntityRepoInterface) OnCreate(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) *SchedulableEntityRepoInterface_Create { + c_call := _m.On("Create", ctx, id, input) return &SchedulableEntityRepoInterface_Create{Call: c_call} } @@ -65,13 +67,13 @@ func (_m *SchedulableEntityRepoInterface) OnCreateMatch(matchers ...interface{}) return &SchedulableEntityRepoInterface_Create{Call: c_call} } -// Create provides a mock function with given fields: ctx, input -func (_m *SchedulableEntityRepoInterface) Create(ctx context.Context, input models.SchedulableEntity) error { - ret := _m.Called(ctx, input) +// Create provides a mock function with given fields: ctx, id, input +func (_m *SchedulableEntityRepoInterface) Create(ctx context.Context, id *core.Identifier, input models.SchedulableEntity) error { + ret := _m.Called(ctx, id, input) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.SchedulableEntity) error); ok { - r0 = rf(ctx, input) + if rf, ok := ret.Get(0).(func(context.Context, *core.Identifier, models.SchedulableEntity) error); ok { + r0 = rf(ctx, id, input) } else { r0 = ret.Error(0) } @@ -87,8 +89,8 @@ func (_m SchedulableEntityRepoInterface_Deactivate) Return(_a0 error) *Schedulab return &SchedulableEntityRepoInterface_Deactivate{Call: _m.Call.Return(_a0)} } -func (_m *SchedulableEntityRepoInterface) OnDeactivate(ctx context.Context, ID models.SchedulableEntityKey) *SchedulableEntityRepoInterface_Deactivate { - c_call := _m.On("Deactivate", ctx, ID) +func (_m *SchedulableEntityRepoInterface) OnDeactivate(ctx context.Context, id *core.Identifier) *SchedulableEntityRepoInterface_Deactivate { + c_call := _m.On("Deactivate", ctx, id) return &SchedulableEntityRepoInterface_Deactivate{Call: c_call} } @@ -97,13 +99,13 @@ func (_m *SchedulableEntityRepoInterface) OnDeactivateMatch(matchers ...interfac return &SchedulableEntityRepoInterface_Deactivate{Call: c_call} } -// Deactivate provides a mock function with given fields: ctx, ID -func (_m *SchedulableEntityRepoInterface) Deactivate(ctx context.Context, ID models.SchedulableEntityKey) error { - ret := _m.Called(ctx, ID) +// Deactivate provides a mock function with given fields: ctx, id +func (_m *SchedulableEntityRepoInterface) Deactivate(ctx context.Context, id *core.Identifier) error { + ret := _m.Called(ctx, id) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.SchedulableEntityKey) error); ok { - r0 = rf(ctx, ID) + if rf, ok := ret.Get(0).(func(context.Context, *core.Identifier) error); ok { + r0 = rf(ctx, id) } else { r0 = ret.Error(0) } @@ -119,8 +121,8 @@ func (_m SchedulableEntityRepoInterface_Get) Return(_a0 models.SchedulableEntity return &SchedulableEntityRepoInterface_Get{Call: _m.Call.Return(_a0, _a1)} } -func (_m *SchedulableEntityRepoInterface) OnGet(ctx context.Context, ID models.SchedulableEntityKey) *SchedulableEntityRepoInterface_Get { - c_call := _m.On("Get", ctx, ID) +func (_m *SchedulableEntityRepoInterface) OnGet(ctx context.Context, id *core.Identifier) *SchedulableEntityRepoInterface_Get { + c_call := _m.On("Get", ctx, id) return &SchedulableEntityRepoInterface_Get{Call: c_call} } @@ -129,20 +131,20 @@ func (_m *SchedulableEntityRepoInterface) OnGetMatch(matchers ...interface{}) *S return &SchedulableEntityRepoInterface_Get{Call: c_call} } -// Get provides a mock function with given fields: ctx, ID -func (_m *SchedulableEntityRepoInterface) Get(ctx context.Context, ID models.SchedulableEntityKey) (models.SchedulableEntity, error) { - ret := _m.Called(ctx, ID) +// Get provides a mock function with given fields: ctx, id +func (_m *SchedulableEntityRepoInterface) Get(ctx context.Context, id *core.Identifier) (models.SchedulableEntity, error) { + ret := _m.Called(ctx, id) var r0 models.SchedulableEntity - if rf, ok := ret.Get(0).(func(context.Context, models.SchedulableEntityKey) models.SchedulableEntity); ok { - r0 = rf(ctx, ID) + if rf, ok := ret.Get(0).(func(context.Context, *core.Identifier) models.SchedulableEntity); ok { + r0 = rf(ctx, id) } else { r0 = ret.Get(0).(models.SchedulableEntity) } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.SchedulableEntityKey) error); ok { - r1 = rf(ctx, ID) + if rf, ok := ret.Get(1).(func(context.Context, *core.Identifier) error); ok { + r1 = rf(ctx, id) } else { r1 = ret.Error(1) } diff --git a/flyteadmin/scheduler/start.go b/flyteadmin/scheduler/start.go index 1f62e3f11c..e4a76c1b8d 100644 --- a/flyteadmin/scheduler/start.go +++ b/flyteadmin/scheduler/start.go @@ -40,8 +40,7 @@ func StartScheduler(ctx context.Context, pluginRegistry *plugins.Registry) error logger.Fatal(ctx, err) } dbScope := schedulerScope.NewSubScope("database") - var newRepoFunc repoInterfaces.NewRepositoryFunc - newRepoFunc = plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) + newRepoFunc := plugins.Get[repoInterfaces.NewRepositoryFunc](pluginRegistry, plugins.PluginIDNewRepositoryFunction) repo := newRepoFunc( db, errors.NewPostgresErrorTransformer(schedulerScope.NewSubScope("errors")), dbScope) diff --git a/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.cc b/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.cc index b0e25c6638..fc3191538e 100644 --- a/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.cc +++ b/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.cc @@ -1002,6 +1002,7 @@ const ::google::protobuf::uint32 TableStruct_flyteidl_2fdatacatalog_2fdatacatalo PROTOBUF_FIELD_OFFSET(::datacatalog::DatasetID, version_), PROTOBUF_FIELD_OFFSET(::datacatalog::DatasetID, uuid_), PROTOBUF_FIELD_OFFSET(::datacatalog::DatasetID, partition_), + PROTOBUF_FIELD_OFFSET(::datacatalog::DatasetID, org_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::datacatalog::Artifact, _internal_metadata_), ~0u, // no _extensions_ @@ -1136,19 +1137,19 @@ static const ::google::protobuf::internal::MigrationSchema schemas[] PROTOBUF_SE { 149, -1, sizeof(::datacatalog::Dataset)}, { 157, -1, sizeof(::datacatalog::Partition)}, { 164, -1, sizeof(::datacatalog::DatasetID)}, - { 175, -1, sizeof(::datacatalog::Artifact)}, - { 187, -1, sizeof(::datacatalog::ArtifactData)}, - { 194, -1, sizeof(::datacatalog::Tag)}, - { 202, 209, sizeof(::datacatalog::Metadata_KeyMapEntry_DoNotUse)}, - { 211, -1, sizeof(::datacatalog::Metadata)}, - { 217, -1, sizeof(::datacatalog::FilterExpression)}, - { 223, -1, sizeof(::datacatalog::SinglePropertyFilter)}, - { 234, -1, sizeof(::datacatalog::ArtifactPropertyFilter)}, - { 241, -1, sizeof(::datacatalog::TagPropertyFilter)}, - { 248, -1, sizeof(::datacatalog::PartitionPropertyFilter)}, - { 255, -1, sizeof(::datacatalog::KeyValuePair)}, - { 262, -1, sizeof(::datacatalog::DatasetPropertyFilter)}, - { 273, -1, sizeof(::datacatalog::PaginationOptions)}, + { 176, -1, sizeof(::datacatalog::Artifact)}, + { 188, -1, sizeof(::datacatalog::ArtifactData)}, + { 195, -1, sizeof(::datacatalog::Tag)}, + { 203, 210, sizeof(::datacatalog::Metadata_KeyMapEntry_DoNotUse)}, + { 212, -1, sizeof(::datacatalog::Metadata)}, + { 218, -1, sizeof(::datacatalog::FilterExpression)}, + { 224, -1, sizeof(::datacatalog::SinglePropertyFilter)}, + { 235, -1, sizeof(::datacatalog::ArtifactPropertyFilter)}, + { 242, -1, sizeof(::datacatalog::TagPropertyFilter)}, + { 249, -1, sizeof(::datacatalog::PartitionPropertyFilter)}, + { 256, -1, sizeof(::datacatalog::KeyValuePair)}, + { 263, -1, sizeof(::datacatalog::DatasetPropertyFilter)}, + { 274, -1, sizeof(::datacatalog::PaginationOptions)}, }; static ::google::protobuf::Message const * const file_default_instances[] = { @@ -1255,79 +1256,80 @@ const char descriptor_table_protodef_flyteidl_2fdatacatalog_2fdatacatalog_2eprot "datacatalog.DatasetID\022\'\n\010metadata\030\002 \001(\0132" "\025.datacatalog.Metadata\022\025\n\rpartitionKeys\030" "\003 \003(\t\"\'\n\tPartition\022\013\n\003key\030\001 \001(\t\022\r\n\005value" - "\030\002 \001(\t\"l\n\tDatasetID\022\017\n\007project\030\001 \001(\t\022\014\n\004" + "\030\002 \001(\t\"y\n\tDatasetID\022\017\n\007project\030\001 \001(\t\022\014\n\004" "name\030\002 \001(\t\022\016\n\006domain\030\003 \001(\t\022\017\n\007version\030\004 " - "\001(\t\022\014\n\004UUID\030\005 \001(\t\022\021\n\tpartition\030\006 \001(\t\"\215\002\n" - "\010Artifact\022\n\n\002id\030\001 \001(\t\022\'\n\007dataset\030\002 \001(\0132\026" - ".datacatalog.DatasetID\022\'\n\004data\030\003 \003(\0132\031.d" - "atacatalog.ArtifactData\022\'\n\010metadata\030\004 \001(" - "\0132\025.datacatalog.Metadata\022*\n\npartitions\030\005" - " \003(\0132\026.datacatalog.Partition\022\036\n\004tags\030\006 \003" - "(\0132\020.datacatalog.Tag\022.\n\ncreated_at\030\007 \001(\013" - "2\032.google.protobuf.Timestamp\"C\n\014Artifact" - "Data\022\014\n\004name\030\001 \001(\t\022%\n\005value\030\002 \001(\0132\026.flyt" - "eidl.core.Literal\"Q\n\003Tag\022\014\n\004name\030\001 \001(\t\022\023" - "\n\013artifact_id\030\002 \001(\t\022\'\n\007dataset\030\003 \001(\0132\026.d" - "atacatalog.DatasetID\"m\n\010Metadata\0222\n\007key_" - "map\030\001 \003(\0132!.datacatalog.Metadata.KeyMapE" - "ntry\032-\n\013KeyMapEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005valu" - "e\030\002 \001(\t:\0028\001\"F\n\020FilterExpression\0222\n\007filte" - "rs\030\001 \003(\0132!.datacatalog.SinglePropertyFil" - "ter\"\211\003\n\024SinglePropertyFilter\0224\n\ntag_filt" - "er\030\001 \001(\0132\036.datacatalog.TagPropertyFilter" - "H\000\022@\n\020partition_filter\030\002 \001(\0132$.datacatal" - "og.PartitionPropertyFilterH\000\022>\n\017artifact" - "_filter\030\003 \001(\0132#.datacatalog.ArtifactProp" - "ertyFilterH\000\022<\n\016dataset_filter\030\004 \001(\0132\".d" - "atacatalog.DatasetPropertyFilterH\000\022F\n\010op" - "erator\030\n \001(\01624.datacatalog.SinglePropert" - "yFilter.ComparisonOperator\" \n\022Comparison" - "Operator\022\n\n\006EQUALS\020\000B\021\n\017property_filter\"" - ";\n\026ArtifactPropertyFilter\022\025\n\013artifact_id" - "\030\001 \001(\tH\000B\n\n\010property\"3\n\021TagPropertyFilte" - "r\022\022\n\010tag_name\030\001 \001(\tH\000B\n\n\010property\"S\n\027Par" - "titionPropertyFilter\022,\n\007key_val\030\001 \001(\0132\031." - "datacatalog.KeyValuePairH\000B\n\n\010property\"*" - "\n\014KeyValuePair\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001" - "(\t\"\200\001\n\025DatasetPropertyFilter\022\021\n\007project\030" - "\001 \001(\tH\000\022\016\n\004name\030\002 \001(\tH\000\022\020\n\006domain\030\003 \001(\tH" - "\000\022\021\n\007version\030\004 \001(\tH\000\022\023\n\tpartition\030\005 \001(\tH" - "\000B\n\n\010property\"\361\001\n\021PaginationOptions\022\r\n\005l" - "imit\030\001 \001(\r\022\r\n\005token\030\002 \001(\t\0227\n\007sortKey\030\003 \001" - "(\0162&.datacatalog.PaginationOptions.SortK" - "ey\022;\n\tsortOrder\030\004 \001(\0162(.datacatalog.Pagi" - "nationOptions.SortOrder\"*\n\tSortOrder\022\016\n\n" - "DESCENDING\020\000\022\r\n\tASCENDING\020\001\"\034\n\007SortKey\022\021" - "\n\rCREATION_TIME\020\0002\206\007\n\013DataCatalog\022V\n\rCre" - "ateDataset\022!.datacatalog.CreateDatasetRe" - "quest\032\".datacatalog.CreateDatasetRespons" - "e\022M\n\nGetDataset\022\036.datacatalog.GetDataset" - "Request\032\037.datacatalog.GetDatasetResponse" - "\022Y\n\016CreateArtifact\022\".datacatalog.CreateA" - "rtifactRequest\032#.datacatalog.CreateArtif" - "actResponse\022P\n\013GetArtifact\022\037.datacatalog" - ".GetArtifactRequest\032 .datacatalog.GetArt" - "ifactResponse\022A\n\006AddTag\022\032.datacatalog.Ad" - "dTagRequest\032\033.datacatalog.AddTagResponse" - "\022V\n\rListArtifacts\022!.datacatalog.ListArti" - "factsRequest\032\".datacatalog.ListArtifacts" - "Response\022S\n\014ListDatasets\022 .datacatalog.L" - "istDatasetsRequest\032!.datacatalog.ListDat" - "asetsResponse\022Y\n\016UpdateArtifact\022\".dataca" - "talog.UpdateArtifactRequest\032#.datacatalo" - "g.UpdateArtifactResponse\022q\n\026GetOrExtendR" - "eservation\022*.datacatalog.GetOrExtendRese" - "rvationRequest\032+.datacatalog.GetOrExtend" - "ReservationResponse\022e\n\022ReleaseReservatio" - "n\022&.datacatalog.ReleaseReservationReques" - "t\032\'.datacatalog.ReleaseReservationRespon" - "seBCZAgithub.com/flyteorg/flyte/flyteidl" - "/gen/pb-go/flyteidl/datacatalogb\006proto3" + "\001(\t\022\014\n\004UUID\030\005 \001(\t\022\021\n\tpartition\030\006 \001(\t\022\013\n\003" + "org\030\007 \001(\t\"\215\002\n\010Artifact\022\n\n\002id\030\001 \001(\t\022\'\n\007da" + "taset\030\002 \001(\0132\026.datacatalog.DatasetID\022\'\n\004d" + "ata\030\003 \003(\0132\031.datacatalog.ArtifactData\022\'\n\010" + "metadata\030\004 \001(\0132\025.datacatalog.Metadata\022*\n" + "\npartitions\030\005 \003(\0132\026.datacatalog.Partitio" + "n\022\036\n\004tags\030\006 \003(\0132\020.datacatalog.Tag\022.\n\ncre" + "ated_at\030\007 \001(\0132\032.google.protobuf.Timestam" + "p\"C\n\014ArtifactData\022\014\n\004name\030\001 \001(\t\022%\n\005value" + "\030\002 \001(\0132\026.flyteidl.core.Literal\"Q\n\003Tag\022\014\n" + "\004name\030\001 \001(\t\022\023\n\013artifact_id\030\002 \001(\t\022\'\n\007data" + "set\030\003 \001(\0132\026.datacatalog.DatasetID\"m\n\010Met" + "adata\0222\n\007key_map\030\001 \003(\0132!.datacatalog.Met" + "adata.KeyMapEntry\032-\n\013KeyMapEntry\022\013\n\003key\030" + "\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"F\n\020FilterExpres" + "sion\0222\n\007filters\030\001 \003(\0132!.datacatalog.Sing" + "lePropertyFilter\"\211\003\n\024SinglePropertyFilte" + "r\0224\n\ntag_filter\030\001 \001(\0132\036.datacatalog.TagP" + "ropertyFilterH\000\022@\n\020partition_filter\030\002 \001(" + "\0132$.datacatalog.PartitionPropertyFilterH" + "\000\022>\n\017artifact_filter\030\003 \001(\0132#.datacatalog" + ".ArtifactPropertyFilterH\000\022<\n\016dataset_fil" + "ter\030\004 \001(\0132\".datacatalog.DatasetPropertyF" + "ilterH\000\022F\n\010operator\030\n \001(\01624.datacatalog." + "SinglePropertyFilter.ComparisonOperator\"" + " \n\022ComparisonOperator\022\n\n\006EQUALS\020\000B\021\n\017pro" + "perty_filter\";\n\026ArtifactPropertyFilter\022\025" + "\n\013artifact_id\030\001 \001(\tH\000B\n\n\010property\"3\n\021Tag" + "PropertyFilter\022\022\n\010tag_name\030\001 \001(\tH\000B\n\n\010pr" + "operty\"S\n\027PartitionPropertyFilter\022,\n\007key" + "_val\030\001 \001(\0132\031.datacatalog.KeyValuePairH\000B" + "\n\n\010property\"*\n\014KeyValuePair\022\013\n\003key\030\001 \001(\t" + "\022\r\n\005value\030\002 \001(\t\"\200\001\n\025DatasetPropertyFilte" + "r\022\021\n\007project\030\001 \001(\tH\000\022\016\n\004name\030\002 \001(\tH\000\022\020\n\006" + "domain\030\003 \001(\tH\000\022\021\n\007version\030\004 \001(\tH\000\022\023\n\tpar" + "tition\030\005 \001(\tH\000B\n\n\010property\"\361\001\n\021Paginatio" + "nOptions\022\r\n\005limit\030\001 \001(\r\022\r\n\005token\030\002 \001(\t\0227" + "\n\007sortKey\030\003 \001(\0162&.datacatalog.Pagination" + "Options.SortKey\022;\n\tsortOrder\030\004 \001(\0162(.dat" + "acatalog.PaginationOptions.SortOrder\"*\n\t" + "SortOrder\022\016\n\nDESCENDING\020\000\022\r\n\tASCENDING\020\001" + "\"\034\n\007SortKey\022\021\n\rCREATION_TIME\020\0002\206\007\n\013DataC" + "atalog\022V\n\rCreateDataset\022!.datacatalog.Cr" + "eateDatasetRequest\032\".datacatalog.CreateD" + "atasetResponse\022M\n\nGetDataset\022\036.datacatal" + "og.GetDatasetRequest\032\037.datacatalog.GetDa" + "tasetResponse\022Y\n\016CreateArtifact\022\".dataca" + "talog.CreateArtifactRequest\032#.datacatalo" + "g.CreateArtifactResponse\022P\n\013GetArtifact\022" + "\037.datacatalog.GetArtifactRequest\032 .datac" + "atalog.GetArtifactResponse\022A\n\006AddTag\022\032.d" + "atacatalog.AddTagRequest\032\033.datacatalog.A" + "ddTagResponse\022V\n\rListArtifacts\022!.datacat" + "alog.ListArtifactsRequest\032\".datacatalog." + "ListArtifactsResponse\022S\n\014ListDatasets\022 ." + "datacatalog.ListDatasetsRequest\032!.dataca" + "talog.ListDatasetsResponse\022Y\n\016UpdateArti" + "fact\022\".datacatalog.UpdateArtifactRequest" + "\032#.datacatalog.UpdateArtifactResponse\022q\n" + "\026GetOrExtendReservation\022*.datacatalog.Ge" + "tOrExtendReservationRequest\032+.datacatalo" + "g.GetOrExtendReservationResponse\022e\n\022Rele" + "aseReservation\022&.datacatalog.ReleaseRese" + "rvationRequest\032\'.datacatalog.ReleaseRese" + "rvationResponseBCZAgithub.com/flyteorg/f" + "lyte/flyteidl/gen/pb-go/flyteidl/datacat" + "alogb\006proto3" ; ::google::protobuf::internal::DescriptorTable descriptor_table_flyteidl_2fdatacatalog_2fdatacatalog_2eproto = { false, InitDefaults_flyteidl_2fdatacatalog_2fdatacatalog_2eproto, descriptor_table_protodef_flyteidl_2fdatacatalog_2fdatacatalog_2eproto, - "flyteidl/datacatalog/datacatalog.proto", &assign_descriptors_table_flyteidl_2fdatacatalog_2fdatacatalog_2eproto, 4959, + "flyteidl/datacatalog/datacatalog.proto", &assign_descriptors_table_flyteidl_2fdatacatalog_2fdatacatalog_2eproto, 4972, }; void AddDescriptors_flyteidl_2fdatacatalog_2fdatacatalog_2eproto() { @@ -9663,6 +9665,7 @@ const int DatasetID::kDomainFieldNumber; const int DatasetID::kVersionFieldNumber; const int DatasetID::kUUIDFieldNumber; const int DatasetID::kPartitionFieldNumber; +const int DatasetID::kOrgFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 DatasetID::DatasetID() @@ -9698,6 +9701,10 @@ DatasetID::DatasetID(const DatasetID& from) if (from.partition().size() > 0) { partition_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.partition_); } + org_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.org().size() > 0) { + org_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.org_); + } // @@protoc_insertion_point(copy_constructor:datacatalog.DatasetID) } @@ -9710,6 +9717,7 @@ void DatasetID::SharedCtor() { version_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); uuid_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); partition_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + org_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } DatasetID::~DatasetID() { @@ -9724,6 +9732,7 @@ void DatasetID::SharedDtor() { version_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); uuid_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); partition_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + org_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } void DatasetID::SetCachedSize(int size) const { @@ -9747,6 +9756,7 @@ void DatasetID::Clear() { version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); uuid_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); partition_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + org_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); _internal_metadata_.Clear(); } @@ -9859,6 +9869,22 @@ const char* DatasetID::_InternalParse(const char* begin, const char* end, void* ptr += size; break; } + // string org = 7; + case 7: { + if (static_cast<::google::protobuf::uint8>(tag) != 58) goto handle_unusual; + ptr = ::google::protobuf::io::ReadSize(ptr, &size); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + ctx->extra_parse_data().SetFieldName("datacatalog.DatasetID.org"); + object = msg->mutable_org(); + if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) { + parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8; + goto string_till_end; + } + GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx)); + ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx); + ptr += size; + break; + } default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -9983,6 +10009,21 @@ bool DatasetID::MergePartialFromCodedStream( break; } + // string org = 7; + case 7: { + if (static_cast< ::google::protobuf::uint8>(tag) == (58 & 0xFF)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_org())); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->org().data(), static_cast(this->org().length()), + ::google::protobuf::internal::WireFormatLite::PARSE, + "datacatalog.DatasetID.org")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -10070,6 +10111,16 @@ void DatasetID::SerializeWithCachedSizes( 6, this->partition(), output); } + // string org = 7; + if (this->org().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->org().data(), static_cast(this->org().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "datacatalog.DatasetID.org"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 7, this->org(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::google::protobuf::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -10149,6 +10200,17 @@ ::google::protobuf::uint8* DatasetID::InternalSerializeWithCachedSizesToArray( 6, this->partition(), target); } + // string org = 7; + if (this->org().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->org().data(), static_cast(this->org().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "datacatalog.DatasetID.org"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 7, this->org(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -10212,6 +10274,13 @@ size_t DatasetID::ByteSizeLong() const { this->partition()); } + // string org = 7; + if (this->org().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->org()); + } + int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -10263,6 +10332,10 @@ void DatasetID::MergeFrom(const DatasetID& from) { partition_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.partition_); } + if (from.org().size() > 0) { + + org_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.org_); + } } void DatasetID::CopyFrom(const ::google::protobuf::Message& from) { @@ -10302,6 +10375,8 @@ void DatasetID::InternalSwap(DatasetID* other) { GetArenaNoVirtual()); partition_.Swap(&other->partition_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); + org_.Swap(&other->org_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); } ::google::protobuf::Metadata DatasetID::GetMetadata() const { diff --git a/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.h b/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.h index 34a55424d9..c5167c11ba 100644 --- a/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.h +++ b/flyteidl/gen/pb-cpp/flyteidl/datacatalog/datacatalog.pb.h @@ -3532,6 +3532,20 @@ class DatasetID final : ::std::string* release_partition(); void set_allocated_partition(::std::string* partition); + // string org = 7; + void clear_org(); + static const int kOrgFieldNumber = 7; + const ::std::string& org() const; + void set_org(const ::std::string& value); + #if LANG_CXX11 + void set_org(::std::string&& value); + #endif + void set_org(const char* value); + void set_org(const char* value, size_t size); + ::std::string* mutable_org(); + ::std::string* release_org(); + void set_allocated_org(::std::string* org); + // @@protoc_insertion_point(class_scope:datacatalog.DatasetID) private: class HasBitSetters; @@ -3543,6 +3557,7 @@ class DatasetID final : ::google::protobuf::internal::ArenaStringPtr version_; ::google::protobuf::internal::ArenaStringPtr uuid_; ::google::protobuf::internal::ArenaStringPtr partition_; + ::google::protobuf::internal::ArenaStringPtr org_; mutable ::google::protobuf::internal::CachedSize _cached_size_; friend struct ::TableStruct_flyteidl_2fdatacatalog_2fdatacatalog_2eproto; }; @@ -8132,6 +8147,59 @@ inline void DatasetID::set_allocated_partition(::std::string* partition) { // @@protoc_insertion_point(field_set_allocated:datacatalog.DatasetID.partition) } +// string org = 7; +inline void DatasetID::clear_org() { + org_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& DatasetID::org() const { + // @@protoc_insertion_point(field_get:datacatalog.DatasetID.org) + return org_.GetNoArena(); +} +inline void DatasetID::set_org(const ::std::string& value) { + + org_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:datacatalog.DatasetID.org) +} +#if LANG_CXX11 +inline void DatasetID::set_org(::std::string&& value) { + + org_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:datacatalog.DatasetID.org) +} +#endif +inline void DatasetID::set_org(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + org_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:datacatalog.DatasetID.org) +} +inline void DatasetID::set_org(const char* value, size_t size) { + + org_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:datacatalog.DatasetID.org) +} +inline ::std::string* DatasetID::mutable_org() { + + // @@protoc_insertion_point(field_mutable:datacatalog.DatasetID.org) + return org_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* DatasetID::release_org() { + // @@protoc_insertion_point(field_release:datacatalog.DatasetID.org) + + return org_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void DatasetID::set_allocated_org(::std::string* org) { + if (org != nullptr) { + + } else { + + } + org_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), org); + // @@protoc_insertion_point(field_set_allocated:datacatalog.DatasetID.org) +} + // ------------------------------------------------------------------- // Artifact diff --git a/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.go b/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.go index 9c96202d91..e1225ffa0a 100644 --- a/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.go @@ -1325,7 +1325,9 @@ type DatasetID struct { Version string `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` UUID string `protobuf:"bytes,5,opt,name=UUID,proto3" json:"UUID,omitempty"` // Optional, partition key applied to the dataset. - Partition string `protobuf:"bytes,6,opt,name=partition,proto3" json:"partition,omitempty"` + Partition string `protobuf:"bytes,6,opt,name=partition,proto3" json:"partition,omitempty"` + // Optional, org key applied to the resource. + Org string `protobuf:"bytes,7,opt,name=org,proto3" json:"org,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1398,6 +1400,13 @@ func (m *DatasetID) GetPartition() string { return "" } +func (m *DatasetID) GetOrg() string { + if m != nil { + return m.Org + } + return "" +} + // // Artifact message. It is composed of several string fields. type Artifact struct { @@ -2284,114 +2293,114 @@ func init() { } var fileDescriptor_275951237ff4368a = []byte{ - // 1698 bytes of a gzipped FileDescriptorProto + // 1709 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0xdb, 0x46, 0x16, 0x37, 0x25, 0x5b, 0x32, 0x9f, 0x2c, 0x45, 0x9e, 0xd8, 0x8e, 0xac, 0x38, 0xb6, 0xc2, 0x04, 0x5e, 0x23, 0xbb, 0x91, 0xb2, 0x76, 0x12, 0x6c, 0xb2, 0x8b, 0xdd, 0x95, 0x2d, 0xc5, 0x56, 0x1d, 0x7f, 0x84, 0xfe, 0x00, 0xfa, 0x01, 0x08, 0x63, 0x73, 0xcc, 0xb0, 0xa6, 0x44, 0x86, 0x1c, 0xa7, - 0xd6, 0xb1, 0x97, 0x1e, 0x8a, 0xde, 0x0a, 0xf4, 0xd6, 0x53, 0x8f, 0xfd, 0x23, 0x7a, 0xcc, 0xad, - 0xff, 0x50, 0x2f, 0xc5, 0x90, 0x33, 0x14, 0x49, 0x51, 0xb6, 0xe2, 0x43, 0x80, 0x5e, 0x08, 0xce, - 0xcc, 0x7b, 0xbf, 0x79, 0x1f, 0xf3, 0xe6, 0xbd, 0x37, 0xb0, 0x7c, 0x66, 0xf6, 0x28, 0x31, 0x34, - 0xb3, 0xa6, 0x61, 0x8a, 0x4f, 0x31, 0xc5, 0xa6, 0xa5, 0x87, 0xff, 0xab, 0xb6, 0x63, 0x51, 0x0b, - 0xe5, 0x42, 0x53, 0xe5, 0x85, 0x80, 0xe9, 0xd4, 0x72, 0x48, 0xcd, 0x34, 0x28, 0x71, 0xb0, 0xe9, - 0xfa, 0xa4, 0xe5, 0x45, 0xdd, 0xb2, 0x74, 0x93, 0xd4, 0xbc, 0xd1, 0xc9, 0xc5, 0x59, 0x4d, 0xbb, - 0x70, 0x30, 0x35, 0xac, 0x2e, 0x5f, 0x5f, 0x8a, 0xaf, 0x53, 0xa3, 0x43, 0x5c, 0x8a, 0x3b, 0xb6, - 0x4f, 0xa0, 0xbc, 0x82, 0x99, 0x0d, 0x87, 0x60, 0x4a, 0x1a, 0x98, 0x62, 0x97, 0x50, 0x95, 0xbc, - 0xbb, 0x20, 0x2e, 0x45, 0x55, 0xc8, 0x6a, 0xfe, 0x4c, 0x49, 0xaa, 0x48, 0x2b, 0xb9, 0xd5, 0x99, - 0x6a, 0x58, 0x50, 0x41, 0x2d, 0x88, 0x94, 0x3b, 0x30, 0x1b, 0xc3, 0x71, 0x6d, 0xab, 0xeb, 0x12, - 0xa5, 0x09, 0xd3, 0x9b, 0x84, 0xc6, 0xd0, 0x9f, 0xc4, 0xd1, 0xe7, 0x92, 0xd0, 0x5b, 0x8d, 0x3e, - 0x7e, 0x03, 0x50, 0x18, 0xc6, 0x07, 0xff, 0x68, 0x29, 0x7f, 0x92, 0x3c, 0x98, 0xba, 0x43, 0x8d, - 0x33, 0x7c, 0x7a, 0x73, 0x71, 0xd0, 0x7d, 0xc8, 0x61, 0x0e, 0xd2, 0x36, 0xb4, 0x52, 0xaa, 0x22, - 0xad, 0xc8, 0x5b, 0x63, 0x2a, 0x88, 0xc9, 0x96, 0x86, 0xee, 0xc2, 0x24, 0xc5, 0x7a, 0xbb, 0x8b, - 0x3b, 0xa4, 0x94, 0xe6, 0xeb, 0x59, 0x8a, 0xf5, 0x5d, 0xdc, 0x21, 0xeb, 0x05, 0x98, 0x7a, 0x77, - 0x41, 0x9c, 0x5e, 0xfb, 0x2d, 0xee, 0x6a, 0x26, 0x51, 0xb6, 0xe0, 0x76, 0x44, 0x2e, 0xae, 0xdf, - 0x3f, 0x61, 0x52, 0x20, 0x72, 0xc9, 0x66, 0x23, 0x92, 0x05, 0x0c, 0x01, 0x99, 0xf2, 0x99, 0x70, - 0x44, 0x5c, 0xc9, 0x1b, 0x60, 0x95, 0x60, 0x2e, 0x8e, 0xc5, 0xbd, 0xba, 0x06, 0xf9, 0xba, 0xa6, - 0x1d, 0x62, 0x5d, 0xa0, 0x2b, 0x90, 0xa6, 0x58, 0xe7, 0xc0, 0xc5, 0x08, 0x30, 0xa3, 0x62, 0x8b, - 0x4a, 0x11, 0x0a, 0x82, 0x89, 0xc3, 0xfc, 0x26, 0xc1, 0xcc, 0x6b, 0xc3, 0x0d, 0x14, 0x77, 0x6f, - 0xee, 0x91, 0x67, 0x90, 0x39, 0x33, 0x4c, 0x4a, 0x1c, 0xcf, 0x19, 0xb9, 0xd5, 0x7b, 0x11, 0x86, - 0x57, 0xde, 0x52, 0xf3, 0xd2, 0x76, 0x88, 0xeb, 0x1a, 0x56, 0x57, 0xe5, 0xc4, 0xe8, 0xbf, 0x00, - 0x36, 0xd6, 0x8d, 0xae, 0x17, 0x34, 0x9e, 0x9f, 0x72, 0xab, 0x8b, 0x11, 0xd6, 0xfd, 0x60, 0x79, - 0xcf, 0x66, 0x5f, 0x57, 0x0d, 0x71, 0x28, 0xe7, 0x30, 0x1b, 0x53, 0x80, 0xbb, 0x6e, 0x0d, 0x64, - 0x61, 0x47, 0xb7, 0x24, 0x55, 0xd2, 0xc3, 0xed, 0xdd, 0xa7, 0x43, 0xf7, 0x00, 0xba, 0xe4, 0x92, - 0xb6, 0xa9, 0x75, 0x4e, 0xba, 0xfe, 0xa9, 0x52, 0x65, 0x36, 0x73, 0xc8, 0x26, 0x94, 0x1f, 0x24, - 0xb8, 0xcd, 0x76, 0xe3, 0xea, 0x07, 0xd6, 0xea, 0xeb, 0x2e, 0xdd, 0x5c, 0xf7, 0xd4, 0x47, 0xeb, - 0xae, 0xfb, 0xce, 0xeb, 0x4b, 0xc3, 0x55, 0x7f, 0x02, 0x93, 0xdc, 0x2b, 0x42, 0xf3, 0xe4, 0xb0, - 0x0c, 0xa8, 0xae, 0xd3, 0xfb, 0x0f, 0x09, 0x66, 0x8f, 0x6c, 0x2d, 0xe1, 0x50, 0x7f, 0xf2, 0xc8, - 0x45, 0x8f, 0x61, 0x9c, 0x41, 0x95, 0xc6, 0x3d, 0xc5, 0xe6, 0x13, 0x5d, 0xca, 0xb6, 0x55, 0x3d, - 0x32, 0x16, 0x75, 0x1d, 0x42, 0xb1, 0xc7, 0x32, 0x91, 0x10, 0x75, 0x3b, 0x7c, 0x51, 0x0d, 0xc8, - 0x06, 0xee, 0x86, 0x17, 0x30, 0x17, 0x57, 0x9e, 0x1b, 0x7a, 0x29, 0xaa, 0x8b, 0xe4, 0xd9, 0x2d, - 0xa4, 0x89, 0x82, 0x21, 0xaf, 0x12, 0x97, 0x38, 0xef, 0x3d, 0x87, 0xb5, 0x1a, 0xe8, 0x19, 0x00, - 0x37, 0x84, 0x60, 0x18, 0x6e, 0x32, 0x99, 0x53, 0xb6, 0x34, 0x34, 0x1f, 0xb2, 0x88, 0xef, 0x1d, - 0x61, 0x0f, 0xe5, 0x83, 0x04, 0xf7, 0x36, 0x09, 0xdd, 0x73, 0x9a, 0x97, 0x94, 0x74, 0xb5, 0xd0, - 0x76, 0xc2, 0x47, 0x75, 0x28, 0x38, 0xfd, 0xd9, 0xfe, 0xbe, 0xe5, 0xc8, 0xbe, 0x11, 0x39, 0xd5, - 0x7c, 0x88, 0xc3, 0xdf, 0xdf, 0xfa, 0xa6, 0x4b, 0x9c, 0xc0, 0x63, 0x6a, 0xd6, 0x1b, 0xb7, 0x34, - 0xb4, 0x05, 0xe8, 0x2d, 0xc1, 0x0e, 0x3d, 0x21, 0x98, 0xb6, 0x8d, 0x2e, 0x65, 0x5c, 0x26, 0x0f, - 0xe4, 0xf9, 0xaa, 0x9f, 0xfe, 0xaa, 0x22, 0xfd, 0x55, 0x1b, 0x3c, 0x3d, 0xaa, 0xd3, 0x01, 0x53, - 0x8b, 0xf3, 0x28, 0xbf, 0xa4, 0x20, 0x17, 0x92, 0xe2, 0xaf, 0x22, 0x37, 0x7a, 0x01, 0x40, 0x2e, - 0x6d, 0xc3, 0x21, 0x6e, 0x1b, 0xd3, 0xd2, 0x38, 0x97, 0x31, 0x8e, 0x70, 0x28, 0x12, 0xbf, 0x2a, - 0x73, 0xea, 0x3a, 0x8d, 0x9c, 0xce, 0xcc, 0x48, 0xa7, 0x53, 0xf9, 0x0a, 0x16, 0x87, 0xb9, 0x9b, - 0x9f, 0xca, 0x97, 0x90, 0x0b, 0x59, 0x81, 0x1b, 0xad, 0x34, 0xcc, 0x68, 0x6a, 0x98, 0x58, 0xe9, - 0xc1, 0xbc, 0x4a, 0x4c, 0x82, 0x5d, 0xf2, 0xa9, 0x0f, 0x92, 0xb2, 0x00, 0xe5, 0xa4, 0xad, 0x79, - 0xa6, 0xfa, 0x5e, 0x82, 0x2c, 0x0f, 0x0d, 0xb4, 0x0c, 0xa9, 0x6b, 0x83, 0x27, 0x65, 0x68, 0x11, - 0xeb, 0xa6, 0x46, 0xb2, 0x2e, 0x7a, 0x08, 0x79, 0x9b, 0xc5, 0x2f, 0xdb, 0x7b, 0x9b, 0xf4, 0xdc, - 0x52, 0xba, 0x92, 0x5e, 0x91, 0xd5, 0xe8, 0xa4, 0xb2, 0x06, 0xf2, 0xbe, 0x98, 0x40, 0x45, 0x48, - 0x9f, 0x93, 0x1e, 0x0f, 0x7e, 0xf6, 0x8b, 0x66, 0x60, 0xe2, 0x3d, 0x36, 0x2f, 0x44, 0xa8, 0xfa, - 0x03, 0xe5, 0x67, 0x09, 0xe4, 0x40, 0x3e, 0x54, 0x82, 0xac, 0xed, 0x58, 0x5f, 0x13, 0x5e, 0x0c, - 0xc8, 0xaa, 0x18, 0x22, 0x04, 0xe3, 0xa1, 0x38, 0xf7, 0xfe, 0xd1, 0x1c, 0x64, 0x34, 0xab, 0x83, - 0x0d, 0x3f, 0x43, 0xca, 0x2a, 0x1f, 0x31, 0x94, 0xf7, 0xc4, 0x61, 0x49, 0xc5, 0x3b, 0x77, 0xb2, - 0x2a, 0x86, 0x0c, 0xe5, 0xe8, 0xa8, 0xd5, 0xf0, 0xee, 0x3c, 0x59, 0xf5, 0xfe, 0xd1, 0x02, 0xc8, - 0x81, 0x1e, 0xde, 0x71, 0x93, 0xd5, 0xfe, 0x84, 0xf2, 0x21, 0x05, 0x93, 0xe2, 0x86, 0x43, 0x85, - 0xc0, 0xc4, 0xb2, 0x67, 0xca, 0xd0, 0x3d, 0x9f, 0x1a, 0xed, 0x9e, 0x17, 0xf7, 0x74, 0xfa, 0xe3, - 0xef, 0xe9, 0xf1, 0xd1, 0x7c, 0xf5, 0x9c, 0xa5, 0x4f, 0x2e, 0xbd, 0x5b, 0x9a, 0xf0, 0xf6, 0x99, - 0x8b, 0xa5, 0x4f, 0xbe, 0xac, 0x86, 0x28, 0xd1, 0x43, 0x18, 0xa7, 0x58, 0x77, 0x4b, 0x19, 0x8f, - 0x63, 0xb0, 0x56, 0xf2, 0x56, 0x59, 0x54, 0x9f, 0x7a, 0xb5, 0x97, 0xc6, 0xa2, 0x3a, 0x7b, 0x7d, - 0x54, 0x73, 0xea, 0x3a, 0x55, 0xf6, 0x61, 0x2a, 0xac, 0x61, 0xe0, 0x51, 0x29, 0xe4, 0xd1, 0x7f, - 0x84, 0xcf, 0x08, 0x93, 0x5b, 0xb4, 0x19, 0x55, 0xd6, 0x66, 0x54, 0x5f, 0xfb, 0x6d, 0x86, 0x38, - 0x3b, 0x26, 0xa4, 0x0f, 0xb1, 0x9e, 0x08, 0xb4, 0x94, 0x90, 0x4f, 0x23, 0xd9, 0x34, 0xe4, 0xba, - 0xf4, 0x68, 0xb5, 0xfe, 0xb7, 0x12, 0x4c, 0x0a, 0x7b, 0xa3, 0x97, 0x90, 0x3d, 0x27, 0xbd, 0x76, - 0x07, 0xdb, 0xbc, 0x96, 0xb8, 0x9f, 0xe8, 0x97, 0xea, 0x36, 0xe9, 0xed, 0x60, 0xbb, 0xd9, 0xa5, - 0x4e, 0x4f, 0xcd, 0x9c, 0x7b, 0x83, 0xf2, 0x0b, 0xc8, 0x85, 0xa6, 0x47, 0x8d, 0x94, 0x97, 0xa9, - 0x7f, 0x49, 0xca, 0x1e, 0x14, 0xe3, 0x75, 0x13, 0xfa, 0x37, 0x64, 0xfd, 0xca, 0xc9, 0x4d, 0x14, - 0xe5, 0xc0, 0xe8, 0xea, 0x26, 0xd9, 0x77, 0x2c, 0x9b, 0x38, 0xb4, 0xe7, 0x73, 0xab, 0x82, 0x43, - 0xf9, 0x3d, 0x0d, 0x33, 0x49, 0x14, 0xe8, 0x7f, 0x00, 0x2c, 0xb7, 0x46, 0x0a, 0xb8, 0xc5, 0xf8, - 0xa1, 0x88, 0xf2, 0x6c, 0x8d, 0xa9, 0x32, 0xc5, 0x3a, 0x07, 0x78, 0x03, 0xc5, 0xe0, 0x74, 0xb5, - 0x23, 0x35, 0xf0, 0xc3, 0xe4, 0xd3, 0x38, 0x00, 0x76, 0x2b, 0xe0, 0xe7, 0x90, 0xbb, 0x70, 0x2b, - 0x70, 0x2a, 0x47, 0xf4, 0x7d, 0xf7, 0x20, 0x31, 0x8e, 0x06, 0x00, 0x0b, 0x82, 0x9b, 0xe3, 0x6d, - 0x43, 0x41, 0x94, 0x1d, 0x1c, 0xce, 0x8f, 0x31, 0x25, 0xe9, 0x28, 0x0c, 0xa0, 0xe5, 0x39, 0x2f, - 0x07, 0xdb, 0x87, 0x49, 0x46, 0x80, 0xa9, 0xe5, 0x94, 0xa0, 0x22, 0xad, 0x14, 0x56, 0x9f, 0x5e, - 0xeb, 0x87, 0xea, 0x86, 0xd5, 0xb1, 0xb1, 0x63, 0xb8, 0xac, 0x92, 0xf5, 0x79, 0xd5, 0x00, 0x45, - 0xa9, 0x00, 0x1a, 0x5c, 0x47, 0x00, 0x99, 0xe6, 0x9b, 0xa3, 0xfa, 0xeb, 0x83, 0xe2, 0xd8, 0xfa, - 0x34, 0xdc, 0xb2, 0x39, 0x20, 0xd7, 0x40, 0xd9, 0x84, 0xb9, 0x64, 0xfd, 0xe3, 0x25, 0xa6, 0x34, - 0x58, 0x62, 0xae, 0x03, 0x4c, 0x0a, 0x3c, 0xe5, 0x3f, 0x30, 0x3d, 0xe0, 0xe1, 0x48, 0x0d, 0x2a, - 0xc5, 0xbb, 0xc7, 0x30, 0xf7, 0x97, 0x70, 0x67, 0x88, 0x63, 0xd1, 0x53, 0x3f, 0x74, 0x58, 0x5d, - 0x21, 0xf1, 0xba, 0x22, 0x6c, 0xa7, 0x6d, 0xd2, 0x3b, 0x66, 0xe7, 0x7d, 0x1f, 0x1b, 0xcc, 0xca, - 0x2c, 0x68, 0x8e, 0xb1, 0x19, 0x01, 0x7f, 0x0e, 0x53, 0x61, 0xaa, 0x91, 0x73, 0xcd, 0xaf, 0x12, - 0xcc, 0x26, 0x7a, 0x13, 0x95, 0x63, 0x79, 0x87, 0xa9, 0x25, 0x32, 0xcf, 0x4c, 0x38, 0xf3, 0x6c, - 0x8d, 0xf1, 0x0b, 0xa6, 0x14, 0xcd, 0x3d, 0x4c, 0x52, 0x9e, 0x7d, 0xca, 0xb1, 0xec, 0xc3, 0xb0, - 0x44, 0xfe, 0x59, 0x0c, 0xe7, 0x9a, 0x09, 0xbe, 0xda, 0x9f, 0x8a, 0x68, 0xf9, 0x63, 0x0a, 0xa6, - 0x07, 0x3a, 0x1d, 0xa6, 0x99, 0x69, 0x74, 0x0c, 0x5f, 0xce, 0xbc, 0xea, 0x0f, 0xd8, 0x6c, 0xb8, - 0x49, 0xf1, 0x07, 0xe8, 0xff, 0x90, 0x75, 0x2d, 0x87, 0x6e, 0x93, 0x9e, 0x27, 0x64, 0x61, 0x75, - 0xf9, 0xea, 0x36, 0xaa, 0x7a, 0xe0, 0x53, 0xab, 0x82, 0x0d, 0xbd, 0x02, 0x99, 0xfd, 0xee, 0x39, - 0x1a, 0x0f, 0x8e, 0xc2, 0xea, 0xca, 0x08, 0x18, 0x1e, 0xbd, 0xda, 0x67, 0x55, 0x1e, 0x81, 0x1c, - 0xcc, 0xa3, 0x02, 0x40, 0xa3, 0x79, 0xb0, 0xd1, 0xdc, 0x6d, 0xb4, 0x76, 0x37, 0x8b, 0x63, 0x28, - 0x0f, 0x72, 0x3d, 0x18, 0x4a, 0xca, 0x02, 0x64, 0xb9, 0x1c, 0x68, 0x1a, 0xf2, 0x1b, 0x6a, 0xb3, - 0x7e, 0xd8, 0xda, 0xdb, 0x6d, 0x1f, 0xb6, 0x76, 0x9a, 0xc5, 0xb1, 0xd5, 0xef, 0xb2, 0x90, 0x63, - 0x3e, 0xdc, 0xf0, 0x05, 0x40, 0xc7, 0x90, 0x8f, 0xbc, 0xf0, 0xa0, 0xe8, 0xed, 0x97, 0xf4, 0x8a, - 0x54, 0x56, 0xae, 0x22, 0xe1, 0xe5, 0xe2, 0x0e, 0x40, 0xff, 0x65, 0x07, 0x45, 0x6f, 0xbe, 0x81, - 0x97, 0xa3, 0xf2, 0xd2, 0xd0, 0x75, 0x0e, 0xf7, 0x39, 0x14, 0xa2, 0x6f, 0x16, 0x28, 0x49, 0x88, - 0x58, 0x1f, 0x59, 0x7e, 0x70, 0x25, 0x0d, 0x87, 0xde, 0x87, 0x5c, 0xe8, 0x91, 0x06, 0x0d, 0x88, - 0x12, 0x07, 0xad, 0x0c, 0x27, 0xe0, 0x88, 0x75, 0xc8, 0xf8, 0x2f, 0x22, 0x28, 0x5a, 0xc3, 0x46, - 0xde, 0x56, 0xca, 0x77, 0x13, 0xd7, 0x38, 0xc4, 0x31, 0xe4, 0x23, 0x0f, 0x10, 0x31, 0xb7, 0x24, - 0xbd, 0xae, 0xc4, 0xdc, 0x92, 0xfc, 0x7e, 0x71, 0x00, 0x53, 0xe1, 0xe6, 0x1e, 0x55, 0x06, 0x78, - 0x62, 0xaf, 0x10, 0xe5, 0xfb, 0x57, 0x50, 0xf4, 0x9d, 0x13, 0x6d, 0x65, 0x63, 0xce, 0x49, 0x6c, - 0xf2, 0x63, 0xce, 0x19, 0xd2, 0x0b, 0xbf, 0x83, 0xb9, 0xe4, 0xbe, 0x04, 0x3d, 0x8a, 0xbb, 0x61, - 0x78, 0xaf, 0x5a, 0xfe, 0xfb, 0x48, 0xb4, 0x7c, 0x4b, 0x02, 0x68, 0xb0, 0x63, 0x40, 0xcb, 0xb1, - 0x6e, 0x64, 0x48, 0x37, 0x53, 0xfe, 0xdb, 0xb5, 0x74, 0xfe, 0x36, 0xeb, 0x1b, 0x5f, 0xd4, 0x75, - 0x83, 0xbe, 0xbd, 0x38, 0xa9, 0x9e, 0x5a, 0x9d, 0x9a, 0x57, 0xa7, 0x59, 0x8e, 0xee, 0xff, 0xd4, - 0x82, 0xd7, 0x61, 0x9d, 0x74, 0x6b, 0xf6, 0xc9, 0x63, 0xdd, 0xaa, 0x25, 0xbd, 0x32, 0x9f, 0x64, - 0xbc, 0x92, 0x71, 0xed, 0xcf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd1, 0x10, 0x8c, 0x86, 0x84, 0x16, - 0x00, 0x00, + 0xd6, 0xb1, 0x97, 0x1e, 0x8a, 0xde, 0x0a, 0xf4, 0x1f, 0xe8, 0xb1, 0x87, 0xfe, 0x09, 0x3d, 0xe6, + 0xd6, 0x7f, 0xa8, 0x97, 0x62, 0xc8, 0x19, 0x8a, 0xa4, 0x28, 0x5b, 0xf1, 0x21, 0x40, 0x2f, 0x04, + 0x67, 0xe6, 0xbd, 0xdf, 0xbc, 0x8f, 0x79, 0xf3, 0xde, 0x1b, 0x58, 0x3e, 0x33, 0x7b, 0x94, 0x18, + 0x9a, 0x59, 0xd3, 0x30, 0xc5, 0xa7, 0x98, 0x62, 0xd3, 0xd2, 0xc3, 0xff, 0x55, 0xdb, 0xb1, 0xa8, + 0x85, 0x72, 0xa1, 0xa9, 0xf2, 0x42, 0xc0, 0x74, 0x6a, 0x39, 0xa4, 0x66, 0x1a, 0x94, 0x38, 0xd8, + 0x74, 0x7d, 0xd2, 0xf2, 0xa2, 0x6e, 0x59, 0xba, 0x49, 0x6a, 0xde, 0xe8, 0xe4, 0xe2, 0xac, 0xa6, + 0x5d, 0x38, 0x98, 0x1a, 0x56, 0x97, 0xaf, 0x2f, 0xc5, 0xd7, 0xa9, 0xd1, 0x21, 0x2e, 0xc5, 0x1d, + 0xdb, 0x27, 0x50, 0x5e, 0xc1, 0xcc, 0x86, 0x43, 0x30, 0x25, 0x0d, 0x4c, 0xb1, 0x4b, 0xa8, 0x4a, + 0xde, 0x5d, 0x10, 0x97, 0xa2, 0x2a, 0x64, 0x35, 0x7f, 0xa6, 0x24, 0x55, 0xa4, 0x95, 0xdc, 0xea, + 0x4c, 0x35, 0x2c, 0xa8, 0xa0, 0x16, 0x44, 0xca, 0x1d, 0x98, 0x8d, 0xe1, 0xb8, 0xb6, 0xd5, 0x75, + 0x89, 0xd2, 0x84, 0xe9, 0x4d, 0x42, 0x63, 0xe8, 0x4f, 0xe2, 0xe8, 0x73, 0x49, 0xe8, 0xad, 0x46, + 0x1f, 0xbf, 0x01, 0x28, 0x0c, 0xe3, 0x83, 0x7f, 0xb4, 0x94, 0x3f, 0x49, 0x1e, 0x4c, 0xdd, 0xa1, + 0xc6, 0x19, 0x3e, 0xbd, 0xb9, 0x38, 0xe8, 0x3e, 0xe4, 0x30, 0x07, 0x69, 0x1b, 0x5a, 0x29, 0x55, + 0x91, 0x56, 0xe4, 0xad, 0x31, 0x15, 0xc4, 0x64, 0x4b, 0x43, 0x77, 0x61, 0x92, 0x62, 0xbd, 0xdd, + 0xc5, 0x1d, 0x52, 0x4a, 0xf3, 0xf5, 0x2c, 0xc5, 0xfa, 0x2e, 0xee, 0x90, 0xf5, 0x02, 0x4c, 0xbd, + 0xbb, 0x20, 0x4e, 0xaf, 0xfd, 0x16, 0x77, 0x35, 0x93, 0x28, 0x5b, 0x70, 0x3b, 0x22, 0x17, 0xd7, + 0xef, 0x9f, 0x30, 0x29, 0x10, 0xb9, 0x64, 0xb3, 0x11, 0xc9, 0x02, 0x86, 0x80, 0x4c, 0xf9, 0x4c, + 0x38, 0x22, 0xae, 0xe4, 0x0d, 0xb0, 0x4a, 0x30, 0x17, 0xc7, 0xe2, 0x5e, 0x5d, 0x83, 0x7c, 0x5d, + 0xd3, 0x0e, 0xb1, 0x2e, 0xd0, 0x15, 0x48, 0x53, 0xac, 0x73, 0xe0, 0x62, 0x04, 0x98, 0x51, 0xb1, + 0x45, 0xa5, 0x08, 0x05, 0xc1, 0xc4, 0x61, 0x7e, 0x93, 0x60, 0xe6, 0xb5, 0xe1, 0x06, 0x8a, 0xbb, + 0x37, 0xf7, 0xc8, 0x33, 0xc8, 0x9c, 0x19, 0x26, 0x25, 0x8e, 0xe7, 0x8c, 0xdc, 0xea, 0xbd, 0x08, + 0xc3, 0x2b, 0x6f, 0xa9, 0x79, 0x69, 0x3b, 0xc4, 0x75, 0x0d, 0xab, 0xab, 0x72, 0x62, 0xf4, 0x5f, + 0x00, 0x1b, 0xeb, 0x46, 0xd7, 0x0b, 0x1a, 0xcf, 0x4f, 0xb9, 0xd5, 0xc5, 0x08, 0xeb, 0x7e, 0xb0, + 0xbc, 0x67, 0xb3, 0xaf, 0xab, 0x86, 0x38, 0x94, 0x73, 0x98, 0x8d, 0x29, 0xc0, 0x5d, 0xb7, 0x06, + 0xb2, 0xb0, 0xa3, 0x5b, 0x92, 0x2a, 0xe9, 0xe1, 0xf6, 0xee, 0xd3, 0xa1, 0x7b, 0x00, 0x5d, 0x72, + 0x49, 0xdb, 0xd4, 0x3a, 0x27, 0x5d, 0xff, 0x54, 0xa9, 0x32, 0x9b, 0x39, 0x64, 0x13, 0xca, 0x0f, + 0x12, 0xdc, 0x66, 0xbb, 0x71, 0xf5, 0x03, 0x6b, 0xf5, 0x75, 0x97, 0x6e, 0xae, 0x7b, 0xea, 0xa3, + 0x75, 0xd7, 0x7d, 0xe7, 0xf5, 0xa5, 0xe1, 0xaa, 0x3f, 0x81, 0x49, 0xee, 0x15, 0xa1, 0x79, 0x72, + 0x58, 0x06, 0x54, 0xd7, 0xe9, 0xfd, 0x87, 0x04, 0xb3, 0x47, 0xb6, 0x96, 0x70, 0xa8, 0x3f, 0x79, + 0xe4, 0xa2, 0xc7, 0x30, 0xce, 0xa0, 0x4a, 0xe3, 0x9e, 0x62, 0xf3, 0x89, 0x2e, 0x65, 0xdb, 0xaa, + 0x1e, 0x19, 0x8b, 0xba, 0x0e, 0xa1, 0xd8, 0x63, 0x99, 0x48, 0x88, 0xba, 0x1d, 0xbe, 0xa8, 0x06, + 0x64, 0x03, 0x77, 0xc3, 0x0b, 0x98, 0x8b, 0x2b, 0xcf, 0x0d, 0xbd, 0x14, 0xd5, 0x45, 0xf2, 0xec, + 0x16, 0xd2, 0x44, 0xc1, 0x90, 0x57, 0x89, 0x4b, 0x9c, 0xf7, 0x9e, 0xc3, 0x5a, 0x0d, 0xf4, 0x0c, + 0x80, 0x1b, 0x42, 0x30, 0x0c, 0x37, 0x99, 0xcc, 0x29, 0x5b, 0x1a, 0x9a, 0x0f, 0x59, 0xc4, 0xf7, + 0x8e, 0xb0, 0x87, 0xf2, 0x41, 0x82, 0x7b, 0x9b, 0x84, 0xee, 0x39, 0xcd, 0x4b, 0x4a, 0xba, 0x5a, + 0x68, 0x3b, 0xe1, 0xa3, 0x3a, 0x14, 0x9c, 0xfe, 0x6c, 0x7f, 0xdf, 0x72, 0x64, 0xdf, 0x88, 0x9c, + 0x6a, 0x3e, 0xc4, 0xe1, 0xef, 0x6f, 0x7d, 0xd3, 0x25, 0x4e, 0xe0, 0x31, 0x35, 0xeb, 0x8d, 0x5b, + 0x1a, 0xda, 0x02, 0xf4, 0x96, 0x60, 0x87, 0x9e, 0x10, 0x4c, 0xdb, 0x46, 0x97, 0x32, 0x2e, 0x93, + 0x07, 0xf2, 0x7c, 0xd5, 0x4f, 0x7f, 0x55, 0x91, 0xfe, 0xaa, 0x0d, 0x9e, 0x1e, 0xd5, 0xe9, 0x80, + 0xa9, 0xc5, 0x79, 0x94, 0x9f, 0x53, 0x90, 0x0b, 0x49, 0xf1, 0x57, 0x91, 0x1b, 0xbd, 0x00, 0x20, + 0x97, 0xb6, 0xe1, 0x10, 0xb7, 0x8d, 0x69, 0x69, 0x9c, 0xcb, 0x18, 0x47, 0x38, 0x14, 0x89, 0x5f, + 0x95, 0x39, 0x75, 0x9d, 0x46, 0x4e, 0x67, 0x66, 0xa4, 0xd3, 0xa9, 0x7c, 0x05, 0x8b, 0xc3, 0xdc, + 0xcd, 0x4f, 0xe5, 0x4b, 0xc8, 0x85, 0xac, 0xc0, 0x8d, 0x56, 0x1a, 0x66, 0x34, 0x35, 0x4c, 0xac, + 0xf4, 0x60, 0x5e, 0x25, 0x26, 0xc1, 0x2e, 0xf9, 0xd4, 0x07, 0x49, 0x59, 0x80, 0x72, 0xd2, 0xd6, + 0x3c, 0x53, 0x7d, 0x2f, 0x41, 0x96, 0x87, 0x06, 0x5a, 0x86, 0xd4, 0xb5, 0xc1, 0x93, 0x32, 0xb4, + 0x88, 0x75, 0x53, 0x23, 0x59, 0x17, 0x3d, 0x84, 0xbc, 0xcd, 0xe2, 0x97, 0xed, 0xbd, 0x4d, 0x7a, + 0x6e, 0x29, 0x5d, 0x49, 0xaf, 0xc8, 0x6a, 0x74, 0x52, 0x59, 0x03, 0x79, 0x5f, 0x4c, 0xa0, 0x22, + 0xa4, 0xcf, 0x49, 0x8f, 0x07, 0x3f, 0xfb, 0x45, 0x33, 0x30, 0xf1, 0x1e, 0x9b, 0x17, 0x22, 0x54, + 0xfd, 0x81, 0xf2, 0xab, 0x04, 0x72, 0x20, 0x1f, 0x2a, 0x41, 0xd6, 0x76, 0xac, 0xaf, 0x09, 0x2f, + 0x06, 0x64, 0x55, 0x0c, 0x11, 0x82, 0xf1, 0x50, 0x9c, 0x7b, 0xff, 0x68, 0x0e, 0x32, 0x9a, 0xd5, + 0xc1, 0x86, 0x9f, 0x21, 0x65, 0x95, 0x8f, 0x18, 0xca, 0x7b, 0xe2, 0xb0, 0xa4, 0xe2, 0x9d, 0x3b, + 0x59, 0x15, 0x43, 0x86, 0x72, 0x74, 0xd4, 0x6a, 0x78, 0x77, 0x9e, 0xac, 0x7a, 0xff, 0x68, 0x01, + 0xe4, 0x40, 0x0f, 0xef, 0xb8, 0xc9, 0x6a, 0x7f, 0x82, 0xe9, 0x61, 0x39, 0x7a, 0x29, 0xeb, 0xeb, + 0x61, 0x39, 0xba, 0xf2, 0x21, 0x05, 0x93, 0xe2, 0xce, 0x43, 0x85, 0xc0, 0xe8, 0xb2, 0x67, 0xdc, + 0xd0, 0xcd, 0x9f, 0x1a, 0xed, 0xe6, 0x17, 0x37, 0x77, 0xfa, 0xe3, 0x6f, 0xee, 0xf1, 0xd1, 0xbc, + 0xf7, 0x9c, 0x25, 0x54, 0xae, 0x8f, 0x5b, 0x9a, 0xf0, 0xf6, 0x99, 0x8b, 0x25, 0x54, 0xbe, 0xac, + 0x86, 0x28, 0xd1, 0x43, 0x18, 0xa7, 0x58, 0x77, 0x4b, 0x19, 0x8f, 0x63, 0xb0, 0x7a, 0xf2, 0x56, + 0x59, 0x9c, 0x9f, 0x7a, 0xd5, 0x98, 0xc6, 0xe2, 0x3c, 0x7b, 0x7d, 0x9c, 0x73, 0xea, 0x3a, 0x55, + 0xf6, 0x61, 0x2a, 0xac, 0x61, 0xe0, 0x63, 0x29, 0xe4, 0xe3, 0x7f, 0x84, 0x4f, 0x0d, 0x93, 0x5b, + 0x34, 0x1e, 0x55, 0xd6, 0x78, 0x54, 0x5f, 0xfb, 0x8d, 0x87, 0x38, 0x4d, 0x26, 0xa4, 0x0f, 0xb1, + 0x9e, 0x08, 0xb4, 0x94, 0x90, 0x61, 0x23, 0xf9, 0x35, 0xe4, 0xba, 0xf4, 0x68, 0xd5, 0xff, 0xb7, + 0x12, 0x4c, 0x0a, 0x7b, 0xa3, 0x97, 0x90, 0x3d, 0x27, 0xbd, 0x76, 0x07, 0xdb, 0xbc, 0xba, 0xb8, + 0x9f, 0xe8, 0x97, 0xea, 0x36, 0xe9, 0xed, 0x60, 0xbb, 0xd9, 0xa5, 0x4e, 0x4f, 0xcd, 0x9c, 0x7b, + 0x83, 0xf2, 0x0b, 0xc8, 0x85, 0xa6, 0x47, 0x8d, 0x9d, 0x97, 0xa9, 0x7f, 0x49, 0xca, 0x1e, 0x14, + 0xe3, 0x95, 0x14, 0xfa, 0x37, 0x64, 0xfd, 0x5a, 0xca, 0x4d, 0x14, 0xe5, 0xc0, 0xe8, 0xea, 0x26, + 0xd9, 0x77, 0x2c, 0x9b, 0x38, 0xb4, 0xe7, 0x73, 0xab, 0x82, 0x43, 0xf9, 0x3d, 0x0d, 0x33, 0x49, + 0x14, 0xe8, 0x7f, 0x00, 0x2c, 0xdb, 0x46, 0x4a, 0xba, 0xc5, 0xf8, 0xa1, 0x88, 0xf2, 0x6c, 0x8d, + 0xa9, 0x32, 0xc5, 0x3a, 0x07, 0x78, 0x03, 0xc5, 0xe0, 0x74, 0xb5, 0x23, 0x55, 0xf1, 0xc3, 0xe4, + 0xd3, 0x38, 0x00, 0x76, 0x2b, 0xe0, 0xe7, 0x90, 0xbb, 0x70, 0x2b, 0x70, 0x2a, 0x47, 0xf4, 0x7d, + 0xf7, 0x20, 0x31, 0x8e, 0x06, 0x00, 0x0b, 0x82, 0x9b, 0xe3, 0x6d, 0x43, 0x41, 0x14, 0x22, 0x1c, + 0xce, 0x8f, 0x31, 0x25, 0xe9, 0x28, 0x0c, 0xa0, 0xe5, 0x39, 0x2f, 0x07, 0xdb, 0x87, 0x49, 0x46, + 0x80, 0xa9, 0xe5, 0x94, 0xa0, 0x22, 0xad, 0x14, 0x56, 0x9f, 0x5e, 0xeb, 0x87, 0xea, 0x86, 0xd5, + 0xb1, 0xb1, 0x63, 0xb8, 0xac, 0xb6, 0xf5, 0x79, 0xd5, 0x00, 0x45, 0xa9, 0x00, 0x1a, 0x5c, 0x47, + 0x00, 0x99, 0xe6, 0x9b, 0xa3, 0xfa, 0xeb, 0x83, 0xe2, 0xd8, 0xfa, 0x34, 0xdc, 0xb2, 0x39, 0x20, + 0xd7, 0x40, 0xd9, 0x84, 0xb9, 0x64, 0xfd, 0xe3, 0x45, 0xa7, 0x34, 0x58, 0x74, 0xae, 0x03, 0x4c, + 0x0a, 0x3c, 0xe5, 0x3f, 0x30, 0x3d, 0xe0, 0xe1, 0x48, 0x55, 0x2a, 0xc5, 0xfb, 0xc9, 0x30, 0xf7, + 0x97, 0x70, 0x67, 0x88, 0x63, 0xd1, 0x53, 0x3f, 0x74, 0x58, 0xa5, 0x21, 0xf1, 0x4a, 0x23, 0x6c, + 0xa7, 0x6d, 0xd2, 0x3b, 0x66, 0xe7, 0x7d, 0x1f, 0x1b, 0xcc, 0xca, 0x2c, 0x68, 0x8e, 0xb1, 0x19, + 0x01, 0x7f, 0x0e, 0x53, 0x61, 0xaa, 0x91, 0xb3, 0xcf, 0x2f, 0x12, 0xcc, 0x26, 0x7a, 0x13, 0x95, + 0x63, 0x99, 0x88, 0xa9, 0x25, 0x72, 0xd1, 0x4c, 0x38, 0x17, 0x6d, 0x8d, 0xf1, 0x0b, 0xa6, 0x14, + 0xcd, 0x46, 0x4c, 0x52, 0x9e, 0x8f, 0xca, 0xb1, 0x7c, 0xc4, 0xb0, 0x44, 0x46, 0x5a, 0x0c, 0x67, + 0x9f, 0x09, 0xbe, 0xda, 0x9f, 0x8a, 0x68, 0xf9, 0x63, 0x0a, 0xa6, 0x07, 0x7a, 0x1f, 0xa6, 0x99, + 0x69, 0x74, 0x0c, 0x5f, 0xce, 0xbc, 0xea, 0x0f, 0xd8, 0x6c, 0xb8, 0x6d, 0xf1, 0x07, 0xe8, 0xff, + 0x90, 0x75, 0x2d, 0x87, 0x6e, 0x93, 0x9e, 0x27, 0x64, 0x61, 0x75, 0xf9, 0xea, 0xc6, 0xaa, 0x7a, + 0xe0, 0x53, 0xab, 0x82, 0x0d, 0xbd, 0x02, 0x99, 0xfd, 0xee, 0x39, 0x1a, 0x0f, 0x8e, 0xc2, 0xea, + 0xca, 0x08, 0x18, 0x1e, 0xbd, 0xda, 0x67, 0x55, 0x1e, 0x81, 0x1c, 0xcc, 0xa3, 0x02, 0x40, 0xa3, + 0x79, 0xb0, 0xd1, 0xdc, 0x6d, 0xb4, 0x76, 0x37, 0x8b, 0x63, 0x28, 0x0f, 0x72, 0x3d, 0x18, 0x4a, + 0xca, 0x02, 0x64, 0xb9, 0x1c, 0x68, 0x1a, 0xf2, 0x1b, 0x6a, 0xb3, 0x7e, 0xd8, 0xda, 0xdb, 0x6d, + 0x1f, 0xb6, 0x76, 0x9a, 0xc5, 0xb1, 0xd5, 0xef, 0xb2, 0x90, 0x63, 0x3e, 0xdc, 0xf0, 0x05, 0x40, + 0xc7, 0x90, 0x8f, 0xbc, 0xf9, 0xa0, 0xe8, 0xed, 0x97, 0xf4, 0xae, 0x54, 0x56, 0xae, 0x22, 0xe1, + 0x05, 0xe4, 0x0e, 0x40, 0xff, 0xad, 0x07, 0x45, 0x6f, 0xbe, 0x81, 0xb7, 0xa4, 0xf2, 0xd2, 0xd0, + 0x75, 0x0e, 0xf7, 0x39, 0x14, 0xa2, 0xaf, 0x18, 0x28, 0x49, 0x88, 0x58, 0x67, 0x59, 0x7e, 0x70, + 0x25, 0x0d, 0x87, 0xde, 0x87, 0x5c, 0xe8, 0xd9, 0x06, 0x0d, 0x88, 0x12, 0x07, 0xad, 0x0c, 0x27, + 0xe0, 0x88, 0x75, 0xc8, 0xf8, 0x6f, 0x24, 0x28, 0x5a, 0xd5, 0x46, 0x5e, 0x5b, 0xca, 0x77, 0x13, + 0xd7, 0x38, 0xc4, 0x31, 0xe4, 0x23, 0x4f, 0x12, 0x31, 0xb7, 0x24, 0xbd, 0xb7, 0xc4, 0xdc, 0x92, + 0xfc, 0xa2, 0x71, 0x00, 0x53, 0xe1, 0x76, 0x1f, 0x55, 0x06, 0x78, 0x62, 0xef, 0x12, 0xe5, 0xfb, + 0x57, 0x50, 0xf4, 0x9d, 0x13, 0x6d, 0x6e, 0x63, 0xce, 0x49, 0x6c, 0xfb, 0x63, 0xce, 0x19, 0xd2, + 0x1d, 0xbf, 0x83, 0xb9, 0xe4, 0x4e, 0x05, 0x3d, 0x8a, 0xbb, 0x61, 0x78, 0xf7, 0x5a, 0xfe, 0xfb, + 0x48, 0xb4, 0x7c, 0x4b, 0x02, 0x68, 0xb0, 0x87, 0x40, 0xcb, 0xb1, 0xfe, 0x64, 0x48, 0x7f, 0x53, + 0xfe, 0xdb, 0xb5, 0x74, 0xfe, 0x36, 0xeb, 0x1b, 0x5f, 0xd4, 0x75, 0x83, 0xbe, 0xbd, 0x38, 0xa9, + 0x9e, 0x5a, 0x9d, 0x9a, 0x57, 0xa7, 0x59, 0x8e, 0xee, 0xff, 0xd4, 0x82, 0xf7, 0x62, 0x9d, 0x74, + 0x6b, 0xf6, 0xc9, 0x63, 0xdd, 0xaa, 0x25, 0xbd, 0x3b, 0x9f, 0x64, 0xbc, 0x92, 0x71, 0xed, 0xcf, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x23, 0xbf, 0xe7, 0xc8, 0x96, 0x16, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.validate.go b/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.validate.go index bbf46471c0..356264611d 100644 --- a/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.validate.go +++ b/flyteidl/gen/pb-go/flyteidl/datacatalog/datacatalog.pb.validate.go @@ -1984,6 +1984,8 @@ func (m *DatasetID) Validate() error { // no validation rules for Partition + // no validation rules for Org + return nil } diff --git a/flyteidl/gen/pb-java/datacatalog/Datacatalog.java b/flyteidl/gen/pb-java/datacatalog/Datacatalog.java index 88d3ceafd1..12c4942382 100644 --- a/flyteidl/gen/pb-java/datacatalog/Datacatalog.java +++ b/flyteidl/gen/pb-java/datacatalog/Datacatalog.java @@ -19972,6 +19972,24 @@ public interface DatasetIDOrBuilder extends */ com.google.protobuf.ByteString getPartitionBytes(); + + /** + * 
+     * Optional, org key applied to the resource.
+     *
+ * + * string org = 7; + */ + java.lang.String getOrg(); + /** + * 
+     * Optional, org key applied to the resource.
+     *
+ * + * string org = 7; + */ + com.google.protobuf.ByteString + getOrgBytes(); } /** * 
@@ -19996,6 +20014,7 @@ private DatasetID() {
       version_ = "";
       uUID_ = "";
       partition_ = "";
+      org_ = "";
     }
 
     @java.lang.Override
@@ -20058,6 +20077,12 @@ private DatasetID(
               partition_ = s;
               break;
             }
+            case 58: {
+              java.lang.String s = input.readStringRequireUtf8();
+
+              org_ = s;
+              break;
+            }
             default: {
               if (!parseUnknownField(
                   input, unknownFields, extensionRegistry, tag)) {
@@ -20342,6 +20367,48 @@ public java.lang.String getPartition() {
       }
     }
 
+    public static final int ORG_FIELD_NUMBER = 7;
+    private volatile java.lang.Object org_;
+    /**
+     * 
+     * Optional, org key applied to the resource.
+     * 

+     *
+     * string org = 7;
+     */
+    public java.lang.String getOrg() {
+      java.lang.Object ref = org_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        org_ = s;
+        return s;
+      }
+    }
+    /**
+     * 
+     * Optional, org key applied to the resource.
+     * 

+     *
+     * string org = 7;
+     */
+    public com.google.protobuf.ByteString
+        getOrgBytes() {
+      java.lang.Object ref = org_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        org_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
     private byte memoizedIsInitialized = -1;
     @java.lang.Override
     public final boolean isInitialized() {
@@ -20374,6 +20441,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
       if (!getPartitionBytes().isEmpty()) {
         com.google.protobuf.GeneratedMessageV3.writeString(output, 6, partition_);
       }
+      if (!getOrgBytes().isEmpty()) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 7, org_);
+      }
       unknownFields.writeTo(output);
     }
 
@@ -20401,6 +20471,9 @@ public int getSerializedSize() {
       if (!getPartitionBytes().isEmpty()) {
         size += com.google.protobuf.GeneratedMessageV3.computeStringSize(6, partition_);
       }
+      if (!getOrgBytes().isEmpty()) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(7, org_);
+      }
       size += unknownFields.getSerializedSize();
       memoizedSize = size;
       return size;
@@ -20428,6 +20501,8 @@ public boolean equals(final java.lang.Object obj) {
           .equals(other.getUUID())) return false;
       if (!getPartition()
           .equals(other.getPartition())) return false;
+      if (!getOrg()
+          .equals(other.getOrg())) return false;
       if (!unknownFields.equals(other.unknownFields)) return false;
       return true;
     }
@@ -20451,6 +20526,8 @@ public int hashCode() {
       hash = (53 * hash) + getUUID().hashCode();
       hash = (37 * hash) + PARTITION_FIELD_NUMBER;
       hash = (53 * hash) + getPartition().hashCode();
+      hash = (37 * hash) + ORG_FIELD_NUMBER;
+      hash = (53 * hash) + getOrg().hashCode();
       hash = (29 * hash) + unknownFields.hashCode();
       memoizedHashCode = hash;
       return hash;
@@ -20600,6 +20677,8 @@ public Builder clear() {
 
         partition_ = "";
 
+        org_ = "";
+
         return this;
       }
 
@@ -20632,6 +20711,7 @@ public datacatalog.Datacatalog.DatasetID buildPartial() {
         result.version_ = version_;
         result.uUID_ = uUID_;
         result.partition_ = partition_;
+        result.org_ = org_;
         onBuilt();
         return result;
       }
@@ -20704,6 +20784,10 @@ public Builder mergeFrom(datacatalog.Datacatalog.DatasetID other) {
           partition_ = other.partition_;
           onChanged();
         }
+        if (!other.getOrg().isEmpty()) {
+          org_ = other.org_;
+          onChanged();
+        }
         this.mergeUnknownFields(other.unknownFields);
         onChanged();
         return this;
@@ -21266,6 +21350,95 @@ public Builder setPartitionBytes(
         onChanged();
         return this;
       }
+
+      private java.lang.Object org_ = "";
+      /**
+       * 
+       * Optional, org key applied to the resource.
+       * 

+       *
+       * string org = 7;
+       */
+      public java.lang.String getOrg() {
+        java.lang.Object ref = org_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs =
+              (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          org_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * 
+       * Optional, org key applied to the resource.
+       * 

+       *
+       * string org = 7;
+       */
+      public com.google.protobuf.ByteString
+          getOrgBytes() {
+        java.lang.Object ref = org_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b = 
+              com.google.protobuf.ByteString.copyFromUtf8(
+                  (java.lang.String) ref);
+          org_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * 
+       * Optional, org key applied to the resource.
+       * 

+       *
+       * string org = 7;
+       */
+      public Builder setOrg(
+          java.lang.String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  
+        org_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * Optional, org key applied to the resource.
+       * 

+       *
+       * string org = 7;
+       */
+      public Builder clearOrg() {
+        
+        org_ = getDefaultInstance().getOrg();
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * Optional, org key applied to the resource.
+       * 

+       *
+       * string org = 7;
+       */
+      public Builder setOrgBytes(
+          com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+        
+        org_ = value;
+        onChanged();
+        return this;
+      }
       @java.lang.Override
       public final Builder setUnknownFields(
           final com.google.protobuf.UnknownFieldSet unknownFields) {
@@ -34242,74 +34415,75 @@ public datacatalog.Datacatalog.PaginationOptions getDefaultInstanceForType() {
       "datacatalog.DatasetID\022\'\n\010metadata\030\002 \001(\0132" +
       "\025.datacatalog.Metadata\022\025\n\rpartitionKeys\030" +
       "\003 \003(\t\"\'\n\tPartition\022\013\n\003key\030\001 \001(\t\022\r\n\005value" +
-      "\030\002 \001(\t\"l\n\tDatasetID\022\017\n\007project\030\001 \001(\t\022\014\n\004" +
+      "\030\002 \001(\t\"y\n\tDatasetID\022\017\n\007project\030\001 \001(\t\022\014\n\004" +
       "name\030\002 \001(\t\022\016\n\006domain\030\003 \001(\t\022\017\n\007version\030\004 " +
-      "\001(\t\022\014\n\004UUID\030\005 \001(\t\022\021\n\tpartition\030\006 \001(\t\"\215\002\n" +
-      "\010Artifact\022\n\n\002id\030\001 \001(\t\022\'\n\007dataset\030\002 \001(\0132\026" +
-      ".datacatalog.DatasetID\022\'\n\004data\030\003 \003(\0132\031.d" +
-      "atacatalog.ArtifactData\022\'\n\010metadata\030\004 \001(" +
-      "\0132\025.datacatalog.Metadata\022*\n\npartitions\030\005" +
-      " \003(\0132\026.datacatalog.Partition\022\036\n\004tags\030\006 \003" +
-      "(\0132\020.datacatalog.Tag\022.\n\ncreated_at\030\007 \001(\013" +
-      "2\032.google.protobuf.Timestamp\"C\n\014Artifact" +
-      "Data\022\014\n\004name\030\001 \001(\t\022%\n\005value\030\002 \001(\0132\026.flyt" +
-      "eidl.core.Literal\"Q\n\003Tag\022\014\n\004name\030\001 \001(\t\022\023" +
-      "\n\013artifact_id\030\002 \001(\t\022\'\n\007dataset\030\003 \001(\0132\026.d" +
-      "atacatalog.DatasetID\"m\n\010Metadata\0222\n\007key_" +
-      "map\030\001 \003(\0132!.datacatalog.Metadata.KeyMapE" +
-      "ntry\032-\n\013KeyMapEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005valu" +
-      "e\030\002 \001(\t:\0028\001\"F\n\020FilterExpression\0222\n\007filte" +
-      "rs\030\001 \003(\0132!.datacatalog.SinglePropertyFil" +
-      "ter\"\211\003\n\024SinglePropertyFilter\0224\n\ntag_filt" +
-      "er\030\001 \001(\0132\036.datacatalog.TagPropertyFilter" +
-      "H\000\022@\n\020partition_filter\030\002 \001(\0132$.datacatal" +
-      "og.PartitionPropertyFilterH\000\022>\n\017artifact" +
-      "_filter\030\003 \001(\0132#.datacatalog.ArtifactProp" +
-      "ertyFilterH\000\022<\n\016dataset_filter\030\004 \001(\0132\".d" +
-      "atacatalog.DatasetPropertyFilterH\000\022F\n\010op" +
-      "erator\030\n \001(\01624.datacatalog.SinglePropert" +
-      "yFilter.ComparisonOperator\" \n\022Comparison" +
-      "Operator\022\n\n\006EQUALS\020\000B\021\n\017property_filter\"" +
-      ";\n\026ArtifactPropertyFilter\022\025\n\013artifact_id" +
-      "\030\001 \001(\tH\000B\n\n\010property\"3\n\021TagPropertyFilte" +
-      "r\022\022\n\010tag_name\030\001 \001(\tH\000B\n\n\010property\"S\n\027Par" +
-      "titionPropertyFilter\022,\n\007key_val\030\001 \001(\0132\031." +
-      "datacatalog.KeyValuePairH\000B\n\n\010property\"*" +
-      "\n\014KeyValuePair\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001" +
-      "(\t\"\200\001\n\025DatasetPropertyFilter\022\021\n\007project\030" +
-      "\001 \001(\tH\000\022\016\n\004name\030\002 \001(\tH\000\022\020\n\006domain\030\003 \001(\tH" +
-      "\000\022\021\n\007version\030\004 \001(\tH\000\022\023\n\tpartition\030\005 \001(\tH" +
-      "\000B\n\n\010property\"\361\001\n\021PaginationOptions\022\r\n\005l" +
-      "imit\030\001 \001(\r\022\r\n\005token\030\002 \001(\t\0227\n\007sortKey\030\003 \001" +
-      "(\0162&.datacatalog.PaginationOptions.SortK" +
-      "ey\022;\n\tsortOrder\030\004 \001(\0162(.datacatalog.Pagi" +
-      "nationOptions.SortOrder\"*\n\tSortOrder\022\016\n\n" +
-      "DESCENDING\020\000\022\r\n\tASCENDING\020\001\"\034\n\007SortKey\022\021" +
-      "\n\rCREATION_TIME\020\0002\206\007\n\013DataCatalog\022V\n\rCre" +
-      "ateDataset\022!.datacatalog.CreateDatasetRe" +
-      "quest\032\".datacatalog.CreateDatasetRespons" +
-      "e\022M\n\nGetDataset\022\036.datacatalog.GetDataset" +
-      "Request\032\037.datacatalog.GetDatasetResponse" +
-      "\022Y\n\016CreateArtifact\022\".datacatalog.CreateA" +
-      "rtifactRequest\032#.datacatalog.CreateArtif" +
-      "actResponse\022P\n\013GetArtifact\022\037.datacatalog" +
-      ".GetArtifactRequest\032 .datacatalog.GetArt" +
-      "ifactResponse\022A\n\006AddTag\022\032.datacatalog.Ad" +
-      "dTagRequest\032\033.datacatalog.AddTagResponse" +
-      "\022V\n\rListArtifacts\022!.datacatalog.ListArti" +
-      "factsRequest\032\".datacatalog.ListArtifacts" +
-      "Response\022S\n\014ListDatasets\022 .datacatalog.L" +
-      "istDatasetsRequest\032!.datacatalog.ListDat" +
-      "asetsResponse\022Y\n\016UpdateArtifact\022\".dataca" +
-      "talog.UpdateArtifactRequest\032#.datacatalo" +
-      "g.UpdateArtifactResponse\022q\n\026GetOrExtendR" +
-      "eservation\022*.datacatalog.GetOrExtendRese" +
-      "rvationRequest\032+.datacatalog.GetOrExtend" +
-      "ReservationResponse\022e\n\022ReleaseReservatio" +
-      "n\022&.datacatalog.ReleaseReservationReques" +
-      "t\032\'.datacatalog.ReleaseReservationRespon" +
-      "seBCZAgithub.com/flyteorg/flyte/flyteidl" +
-      "/gen/pb-go/flyteidl/datacatalogb\006proto3"
+      "\001(\t\022\014\n\004UUID\030\005 \001(\t\022\021\n\tpartition\030\006 \001(\t\022\013\n\003" +
+      "org\030\007 \001(\t\"\215\002\n\010Artifact\022\n\n\002id\030\001 \001(\t\022\'\n\007da" +
+      "taset\030\002 \001(\0132\026.datacatalog.DatasetID\022\'\n\004d" +
+      "ata\030\003 \003(\0132\031.datacatalog.ArtifactData\022\'\n\010" +
+      "metadata\030\004 \001(\0132\025.datacatalog.Metadata\022*\n" +
+      "\npartitions\030\005 \003(\0132\026.datacatalog.Partitio" +
+      "n\022\036\n\004tags\030\006 \003(\0132\020.datacatalog.Tag\022.\n\ncre" +
+      "ated_at\030\007 \001(\0132\032.google.protobuf.Timestam" +
+      "p\"C\n\014ArtifactData\022\014\n\004name\030\001 \001(\t\022%\n\005value" +
+      "\030\002 \001(\0132\026.flyteidl.core.Literal\"Q\n\003Tag\022\014\n" +
+      "\004name\030\001 \001(\t\022\023\n\013artifact_id\030\002 \001(\t\022\'\n\007data" +
+      "set\030\003 \001(\0132\026.datacatalog.DatasetID\"m\n\010Met" +
+      "adata\0222\n\007key_map\030\001 \003(\0132!.datacatalog.Met" +
+      "adata.KeyMapEntry\032-\n\013KeyMapEntry\022\013\n\003key\030" +
+      "\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"F\n\020FilterExpres" +
+      "sion\0222\n\007filters\030\001 \003(\0132!.datacatalog.Sing" +
+      "lePropertyFilter\"\211\003\n\024SinglePropertyFilte" +
+      "r\0224\n\ntag_filter\030\001 \001(\0132\036.datacatalog.TagP" +
+      "ropertyFilterH\000\022@\n\020partition_filter\030\002 \001(" +
+      "\0132$.datacatalog.PartitionPropertyFilterH" +
+      "\000\022>\n\017artifact_filter\030\003 \001(\0132#.datacatalog" +
+      ".ArtifactPropertyFilterH\000\022<\n\016dataset_fil" +
+      "ter\030\004 \001(\0132\".datacatalog.DatasetPropertyF" +
+      "ilterH\000\022F\n\010operator\030\n \001(\01624.datacatalog." +
+      "SinglePropertyFilter.ComparisonOperator\"" +
+      " \n\022ComparisonOperator\022\n\n\006EQUALS\020\000B\021\n\017pro" +
+      "perty_filter\";\n\026ArtifactPropertyFilter\022\025" +
+      "\n\013artifact_id\030\001 \001(\tH\000B\n\n\010property\"3\n\021Tag" +
+      "PropertyFilter\022\022\n\010tag_name\030\001 \001(\tH\000B\n\n\010pr" +
+      "operty\"S\n\027PartitionPropertyFilter\022,\n\007key" +
+      "_val\030\001 \001(\0132\031.datacatalog.KeyValuePairH\000B" +
+      "\n\n\010property\"*\n\014KeyValuePair\022\013\n\003key\030\001 \001(\t" +
+      "\022\r\n\005value\030\002 \001(\t\"\200\001\n\025DatasetPropertyFilte" +
+      "r\022\021\n\007project\030\001 \001(\tH\000\022\016\n\004name\030\002 \001(\tH\000\022\020\n\006" +
+      "domain\030\003 \001(\tH\000\022\021\n\007version\030\004 \001(\tH\000\022\023\n\tpar" +
+      "tition\030\005 \001(\tH\000B\n\n\010property\"\361\001\n\021Paginatio" +
+      "nOptions\022\r\n\005limit\030\001 \001(\r\022\r\n\005token\030\002 \001(\t\0227" +
+      "\n\007sortKey\030\003 \001(\0162&.datacatalog.Pagination" +
+      "Options.SortKey\022;\n\tsortOrder\030\004 \001(\0162(.dat" +
+      "acatalog.PaginationOptions.SortOrder\"*\n\t" +
+      "SortOrder\022\016\n\nDESCENDING\020\000\022\r\n\tASCENDING\020\001" +
+      "\"\034\n\007SortKey\022\021\n\rCREATION_TIME\020\0002\206\007\n\013DataC" +
+      "atalog\022V\n\rCreateDataset\022!.datacatalog.Cr" +
+      "eateDatasetRequest\032\".datacatalog.CreateD" +
+      "atasetResponse\022M\n\nGetDataset\022\036.datacatal" +
+      "og.GetDatasetRequest\032\037.datacatalog.GetDa" +
+      "tasetResponse\022Y\n\016CreateArtifact\022\".dataca" +
+      "talog.CreateArtifactRequest\032#.datacatalo" +
+      "g.CreateArtifactResponse\022P\n\013GetArtifact\022" +
+      "\037.datacatalog.GetArtifactRequest\032 .datac" +
+      "atalog.GetArtifactResponse\022A\n\006AddTag\022\032.d" +
+      "atacatalog.AddTagRequest\032\033.datacatalog.A" +
+      "ddTagResponse\022V\n\rListArtifacts\022!.datacat" +
+      "alog.ListArtifactsRequest\032\".datacatalog." +
+      "ListArtifactsResponse\022S\n\014ListDatasets\022 ." +
+      "datacatalog.ListDatasetsRequest\032!.dataca" +
+      "talog.ListDatasetsResponse\022Y\n\016UpdateArti" +
+      "fact\022\".datacatalog.UpdateArtifactRequest" +
+      "\032#.datacatalog.UpdateArtifactResponse\022q\n" +
+      "\026GetOrExtendReservation\022*.datacatalog.Ge" +
+      "tOrExtendReservationRequest\032+.datacatalo" +
+      "g.GetOrExtendReservationResponse\022e\n\022Rele" +
+      "aseReservation\022&.datacatalog.ReleaseRese" +
+      "rvationRequest\032\'.datacatalog.ReleaseRese" +
+      "rvationResponseBCZAgithub.com/flyteorg/f" +
+      "lyte/flyteidl/gen/pb-go/flyteidl/datacat" +
+      "alogb\006proto3"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
         new com.google.protobuf.Descriptors.FileDescriptor.    InternalDescriptorAssigner() {
@@ -34475,7 +34649,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors(
     internal_static_datacatalog_DatasetID_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_datacatalog_DatasetID_descriptor,
-        new java.lang.String[] { "Project", "Name", "Domain", "Version", "UUID", "Partition", });
+        new java.lang.String[] { "Project", "Name", "Domain", "Version", "UUID", "Partition", "Org", });
     internal_static_datacatalog_Artifact_descriptor =
       getDescriptor().getMessageTypes().get(25);
     internal_static_datacatalog_Artifact_fieldAccessorTable = new
diff --git a/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.py b/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.py
index 26d3c873dc..7a86aa9316 100644
--- a/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.py
+++ b/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.py
@@ -16,7 +16,7 @@
 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&flyteidl/datacatalog/datacatalog.proto\x12\x0b\x64\x61tacatalog\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"F\n\x14\x43reateDatasetRequest\x12.\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x14.datacatalog.DatasetR\x07\x64\x61taset\"\x17\n\x15\x43reateDatasetResponse\"E\n\x11GetDatasetRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\"D\n\x12GetDatasetResponse\x12.\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x14.datacatalog.DatasetR\x07\x64\x61taset\"\x96\x01\n\x12GetArtifactRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12!\n\x0b\x61rtifact_id\x18\x02 \x01(\tH\x00R\nartifactId\x12\x1b\n\x08tag_name\x18\x03 \x01(\tH\x00R\x07tagNameB\x0e\n\x0cquery_handle\"H\n\x13GetArtifactResponse\x12\x31\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x15.datacatalog.ArtifactR\x08\x61rtifact\"J\n\x15\x43reateArtifactRequest\x12\x31\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x15.datacatalog.ArtifactR\x08\x61rtifact\"\x18\n\x16\x43reateArtifactResponse\"3\n\rAddTagRequest\x12\"\n\x03tag\x18\x01 \x01(\x0b\x32\x10.datacatalog.TagR\x03tag\"\x10\n\x0e\x41\x64\x64TagResponse\"\xbf\x01\n\x14ListArtifactsRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12\x35\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x1d.datacatalog.FilterExpressionR\x06\x66ilter\x12>\n\npagination\x18\x03 \x01(\x0b\x32\x1e.datacatalog.PaginationOptionsR\npagination\"k\n\x15ListArtifactsResponse\x12\x33\n\tartifacts\x18\x01 \x03(\x0b\x32\x15.datacatalog.ArtifactR\tartifacts\x12\x1d\n\nnext_token\x18\x02 \x01(\tR\tnextToken\"\x8c\x01\n\x13ListDatasetsRequest\x12\x35\n\x06\x66ilter\x18\x01 \x01(\x0b\x32\x1d.datacatalog.FilterExpressionR\x06\x66ilter\x12>\n\npagination\x18\x02 \x01(\x0b\x32\x1e.datacatalog.PaginationOptionsR\npagination\"g\n\x14ListDatasetsResponse\x12\x30\n\x08\x64\x61tasets\x18\x01 \x03(\x0b\x32\x14.datacatalog.DatasetR\x08\x64\x61tasets\x12\x1d\n\nnext_token\x18\x02 \x01(\tR\tnextToken\"\xfb\x01\n\x15UpdateArtifactRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12!\n\x0b\x61rtifact_id\x18\x02 \x01(\tH\x00R\nartifactId\x12\x1b\n\x08tag_name\x18\x03 \x01(\tH\x00R\x07tagName\x12-\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32\x19.datacatalog.ArtifactDataR\x04\x64\x61ta\x12\x31\n\x08metadata\x18\x05 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadataB\x0e\n\x0cquery_handle\"9\n\x16UpdateArtifactResponse\x12\x1f\n\x0b\x61rtifact_id\x18\x01 \x01(\tR\nartifactId\"a\n\rReservationID\x12\x35\n\ndataset_id\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\tdatasetId\x12\x19\n\x08tag_name\x18\x02 \x01(\tR\x07tagName\"\xc7\x01\n\x1dGetOrExtendReservationRequest\x12\x41\n\x0ereservation_id\x18\x01 \x01(\x0b\x32\x1a.datacatalog.ReservationIDR\rreservationId\x12\x19\n\x08owner_id\x18\x02 \x01(\tR\x07ownerId\x12H\n\x12heartbeat_interval\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x11heartbeatInterval\"\xa3\x02\n\x0bReservation\x12\x41\n\x0ereservation_id\x18\x01 \x01(\x0b\x32\x1a.datacatalog.ReservationIDR\rreservationId\x12\x19\n\x08owner_id\x18\x02 \x01(\tR\x07ownerId\x12H\n\x12heartbeat_interval\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x11heartbeatInterval\x12\x39\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\x12\x31\n\x08metadata\x18\x06 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadata\"\\\n\x1eGetOrExtendReservationResponse\x12:\n\x0breservation\x18\x01 \x01(\x0b\x32\x18.datacatalog.ReservationR\x0breservation\"y\n\x19ReleaseReservationRequest\x12\x41\n\x0ereservation_id\x18\x01 \x01(\x0b\x32\x1a.datacatalog.ReservationIDR\rreservationId\x12\x19\n\x08owner_id\x18\x02 \x01(\tR\x07ownerId\"\x1c\n\x1aReleaseReservationResponse\"\x8a\x01\n\x07\x44\x61taset\x12&\n\x02id\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x02id\x12\x31\n\x08metadata\x18\x02 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadata\x12$\n\rpartitionKeys\x18\x03 \x03(\tR\rpartitionKeys\"3\n\tPartition\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"\x9d\x01\n\tDatasetID\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x16\n\x06\x64omain\x18\x03 \x01(\tR\x06\x64omain\x12\x18\n\x07version\x18\x04 \x01(\tR\x07version\x12\x12\n\x04UUID\x18\x05 \x01(\tR\x04UUID\x12\x1c\n\tpartition\x18\x06 \x01(\tR\tpartition\"\xc7\x02\n\x08\x41rtifact\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x30\n\x07\x64\x61taset\x18\x02 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12-\n\x04\x64\x61ta\x18\x03 \x03(\x0b\x32\x19.datacatalog.ArtifactDataR\x04\x64\x61ta\x12\x31\n\x08metadata\x18\x04 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadata\x12\x36\n\npartitions\x18\x05 \x03(\x0b\x32\x16.datacatalog.PartitionR\npartitions\x12$\n\x04tags\x18\x06 \x03(\x0b\x32\x10.datacatalog.TagR\x04tags\x12\x39\n\ncreated_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\"P\n\x0c\x41rtifactData\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value\"l\n\x03Tag\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0b\x61rtifact_id\x18\x02 \x01(\tR\nartifactId\x12\x30\n\x07\x64\x61taset\x18\x03 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\"\x81\x01\n\x08Metadata\x12:\n\x07key_map\x18\x01 \x03(\x0b\x32!.datacatalog.Metadata.KeyMapEntryR\x06keyMap\x1a\x39\n\x0bKeyMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"O\n\x10\x46ilterExpression\x12;\n\x07\x66ilters\x18\x01 \x03(\x0b\x32!.datacatalog.SinglePropertyFilterR\x07\x66ilters\"\xce\x03\n\x14SinglePropertyFilter\x12?\n\ntag_filter\x18\x01 \x01(\x0b\x32\x1e.datacatalog.TagPropertyFilterH\x00R\ttagFilter\x12Q\n\x10partition_filter\x18\x02 \x01(\x0b\x32$.datacatalog.PartitionPropertyFilterH\x00R\x0fpartitionFilter\x12N\n\x0f\x61rtifact_filter\x18\x03 \x01(\x0b\x32#.datacatalog.ArtifactPropertyFilterH\x00R\x0e\x61rtifactFilter\x12K\n\x0e\x64\x61taset_filter\x18\x04 \x01(\x0b\x32\".datacatalog.DatasetPropertyFilterH\x00R\rdatasetFilter\x12P\n\x08operator\x18\n \x01(\x0e\x32\x34.datacatalog.SinglePropertyFilter.ComparisonOperatorR\x08operator\" \n\x12\x43omparisonOperator\x12\n\n\x06\x45QUALS\x10\x00\x42\x11\n\x0fproperty_filter\"G\n\x16\x41rtifactPropertyFilter\x12!\n\x0b\x61rtifact_id\x18\x01 \x01(\tH\x00R\nartifactIdB\n\n\x08property\"<\n\x11TagPropertyFilter\x12\x1b\n\x08tag_name\x18\x01 \x01(\tH\x00R\x07tagNameB\n\n\x08property\"[\n\x17PartitionPropertyFilter\x12\x34\n\x07key_val\x18\x01 \x01(\x0b\x32\x19.datacatalog.KeyValuePairH\x00R\x06keyValB\n\n\x08property\"6\n\x0cKeyValuePair\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"\xab\x01\n\x15\x44\x61tasetPropertyFilter\x12\x1a\n\x07project\x18\x01 \x01(\tH\x00R\x07project\x12\x14\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x12\x18\n\x06\x64omain\x18\x03 \x01(\tH\x00R\x06\x64omain\x12\x1a\n\x07version\x18\x04 \x01(\tH\x00R\x07version\x12\x1e\n\tpartition\x18\x05 \x01(\tH\x00R\tpartitionB\n\n\x08property\"\x93\x02\n\x11PaginationOptions\x12\x14\n\x05limit\x18\x01 \x01(\rR\x05limit\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\x12@\n\x07sortKey\x18\x03 \x01(\x0e\x32&.datacatalog.PaginationOptions.SortKeyR\x07sortKey\x12\x46\n\tsortOrder\x18\x04 \x01(\x0e\x32(.datacatalog.PaginationOptions.SortOrderR\tsortOrder\"*\n\tSortOrder\x12\x0e\n\nDESCENDING\x10\x00\x12\r\n\tASCENDING\x10\x01\"\x1c\n\x07SortKey\x12\x11\n\rCREATION_TIME\x10\x00\x32\x86\x07\n\x0b\x44\x61taCatalog\x12V\n\rCreateDataset\x12!.datacatalog.CreateDatasetRequest\x1a\".datacatalog.CreateDatasetResponse\x12M\n\nGetDataset\x12\x1e.datacatalog.GetDatasetRequest\x1a\x1f.datacatalog.GetDatasetResponse\x12Y\n\x0e\x43reateArtifact\x12\".datacatalog.CreateArtifactRequest\x1a#.datacatalog.CreateArtifactResponse\x12P\n\x0bGetArtifact\x12\x1f.datacatalog.GetArtifactRequest\x1a .datacatalog.GetArtifactResponse\x12\x41\n\x06\x41\x64\x64Tag\x12\x1a.datacatalog.AddTagRequest\x1a\x1b.datacatalog.AddTagResponse\x12V\n\rListArtifacts\x12!.datacatalog.ListArtifactsRequest\x1a\".datacatalog.ListArtifactsResponse\x12S\n\x0cListDatasets\x12 .datacatalog.ListDatasetsRequest\x1a!.datacatalog.ListDatasetsResponse\x12Y\n\x0eUpdateArtifact\x12\".datacatalog.UpdateArtifactRequest\x1a#.datacatalog.UpdateArtifactResponse\x12q\n\x16GetOrExtendReservation\x12*.datacatalog.GetOrExtendReservationRequest\x1a+.datacatalog.GetOrExtendReservationResponse\x12\x65\n\x12ReleaseReservation\x12&.datacatalog.ReleaseReservationRequest\x1a\'.datacatalog.ReleaseReservationResponseB\xb2\x01\n\x0f\x63om.datacatalogB\x10\x44\x61tacatalogProtoP\x01ZAgithub.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tacatalog\xca\x02\x0b\x44\x61tacatalog\xe2\x02\x17\x44\x61tacatalog\\GPBMetadata\xea\x02\x0b\x44\x61tacatalogb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&flyteidl/datacatalog/datacatalog.proto\x12\x0b\x64\x61tacatalog\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"F\n\x14\x43reateDatasetRequest\x12.\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x14.datacatalog.DatasetR\x07\x64\x61taset\"\x17\n\x15\x43reateDatasetResponse\"E\n\x11GetDatasetRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\"D\n\x12GetDatasetResponse\x12.\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x14.datacatalog.DatasetR\x07\x64\x61taset\"\x96\x01\n\x12GetArtifactRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12!\n\x0b\x61rtifact_id\x18\x02 \x01(\tH\x00R\nartifactId\x12\x1b\n\x08tag_name\x18\x03 \x01(\tH\x00R\x07tagNameB\x0e\n\x0cquery_handle\"H\n\x13GetArtifactResponse\x12\x31\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x15.datacatalog.ArtifactR\x08\x61rtifact\"J\n\x15\x43reateArtifactRequest\x12\x31\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x15.datacatalog.ArtifactR\x08\x61rtifact\"\x18\n\x16\x43reateArtifactResponse\"3\n\rAddTagRequest\x12\"\n\x03tag\x18\x01 \x01(\x0b\x32\x10.datacatalog.TagR\x03tag\"\x10\n\x0e\x41\x64\x64TagResponse\"\xbf\x01\n\x14ListArtifactsRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12\x35\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x1d.datacatalog.FilterExpressionR\x06\x66ilter\x12>\n\npagination\x18\x03 \x01(\x0b\x32\x1e.datacatalog.PaginationOptionsR\npagination\"k\n\x15ListArtifactsResponse\x12\x33\n\tartifacts\x18\x01 \x03(\x0b\x32\x15.datacatalog.ArtifactR\tartifacts\x12\x1d\n\nnext_token\x18\x02 \x01(\tR\tnextToken\"\x8c\x01\n\x13ListDatasetsRequest\x12\x35\n\x06\x66ilter\x18\x01 \x01(\x0b\x32\x1d.datacatalog.FilterExpressionR\x06\x66ilter\x12>\n\npagination\x18\x02 \x01(\x0b\x32\x1e.datacatalog.PaginationOptionsR\npagination\"g\n\x14ListDatasetsResponse\x12\x30\n\x08\x64\x61tasets\x18\x01 \x03(\x0b\x32\x14.datacatalog.DatasetR\x08\x64\x61tasets\x12\x1d\n\nnext_token\x18\x02 \x01(\tR\tnextToken\"\xfb\x01\n\x15UpdateArtifactRequest\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12!\n\x0b\x61rtifact_id\x18\x02 \x01(\tH\x00R\nartifactId\x12\x1b\n\x08tag_name\x18\x03 \x01(\tH\x00R\x07tagName\x12-\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32\x19.datacatalog.ArtifactDataR\x04\x64\x61ta\x12\x31\n\x08metadata\x18\x05 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadataB\x0e\n\x0cquery_handle\"9\n\x16UpdateArtifactResponse\x12\x1f\n\x0b\x61rtifact_id\x18\x01 \x01(\tR\nartifactId\"a\n\rReservationID\x12\x35\n\ndataset_id\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\tdatasetId\x12\x19\n\x08tag_name\x18\x02 \x01(\tR\x07tagName\"\xc7\x01\n\x1dGetOrExtendReservationRequest\x12\x41\n\x0ereservation_id\x18\x01 \x01(\x0b\x32\x1a.datacatalog.ReservationIDR\rreservationId\x12\x19\n\x08owner_id\x18\x02 \x01(\tR\x07ownerId\x12H\n\x12heartbeat_interval\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x11heartbeatInterval\"\xa3\x02\n\x0bReservation\x12\x41\n\x0ereservation_id\x18\x01 \x01(\x0b\x32\x1a.datacatalog.ReservationIDR\rreservationId\x12\x19\n\x08owner_id\x18\x02 \x01(\tR\x07ownerId\x12H\n\x12heartbeat_interval\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x11heartbeatInterval\x12\x39\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\x12\x31\n\x08metadata\x18\x06 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadata\"\\\n\x1eGetOrExtendReservationResponse\x12:\n\x0breservation\x18\x01 \x01(\x0b\x32\x18.datacatalog.ReservationR\x0breservation\"y\n\x19ReleaseReservationRequest\x12\x41\n\x0ereservation_id\x18\x01 \x01(\x0b\x32\x1a.datacatalog.ReservationIDR\rreservationId\x12\x19\n\x08owner_id\x18\x02 \x01(\tR\x07ownerId\"\x1c\n\x1aReleaseReservationResponse\"\x8a\x01\n\x07\x44\x61taset\x12&\n\x02id\x18\x01 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x02id\x12\x31\n\x08metadata\x18\x02 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadata\x12$\n\rpartitionKeys\x18\x03 \x03(\tR\rpartitionKeys\"3\n\tPartition\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"\xaf\x01\n\tDatasetID\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x16\n\x06\x64omain\x18\x03 \x01(\tR\x06\x64omain\x12\x18\n\x07version\x18\x04 \x01(\tR\x07version\x12\x12\n\x04UUID\x18\x05 \x01(\tR\x04UUID\x12\x1c\n\tpartition\x18\x06 \x01(\tR\tpartition\x12\x10\n\x03org\x18\x07 \x01(\tR\x03org\"\xc7\x02\n\x08\x41rtifact\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x30\n\x07\x64\x61taset\x18\x02 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\x12-\n\x04\x64\x61ta\x18\x03 \x03(\x0b\x32\x19.datacatalog.ArtifactDataR\x04\x64\x61ta\x12\x31\n\x08metadata\x18\x04 \x01(\x0b\x32\x15.datacatalog.MetadataR\x08metadata\x12\x36\n\npartitions\x18\x05 \x03(\x0b\x32\x16.datacatalog.PartitionR\npartitions\x12$\n\x04tags\x18\x06 \x03(\x0b\x32\x10.datacatalog.TagR\x04tags\x12\x39\n\ncreated_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\"P\n\x0c\x41rtifactData\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value\"l\n\x03Tag\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0b\x61rtifact_id\x18\x02 \x01(\tR\nartifactId\x12\x30\n\x07\x64\x61taset\x18\x03 \x01(\x0b\x32\x16.datacatalog.DatasetIDR\x07\x64\x61taset\"\x81\x01\n\x08Metadata\x12:\n\x07key_map\x18\x01 \x03(\x0b\x32!.datacatalog.Metadata.KeyMapEntryR\x06keyMap\x1a\x39\n\x0bKeyMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"O\n\x10\x46ilterExpression\x12;\n\x07\x66ilters\x18\x01 \x03(\x0b\x32!.datacatalog.SinglePropertyFilterR\x07\x66ilters\"\xce\x03\n\x14SinglePropertyFilter\x12?\n\ntag_filter\x18\x01 \x01(\x0b\x32\x1e.datacatalog.TagPropertyFilterH\x00R\ttagFilter\x12Q\n\x10partition_filter\x18\x02 \x01(\x0b\x32$.datacatalog.PartitionPropertyFilterH\x00R\x0fpartitionFilter\x12N\n\x0f\x61rtifact_filter\x18\x03 \x01(\x0b\x32#.datacatalog.ArtifactPropertyFilterH\x00R\x0e\x61rtifactFilter\x12K\n\x0e\x64\x61taset_filter\x18\x04 \x01(\x0b\x32\".datacatalog.DatasetPropertyFilterH\x00R\rdatasetFilter\x12P\n\x08operator\x18\n \x01(\x0e\x32\x34.datacatalog.SinglePropertyFilter.ComparisonOperatorR\x08operator\" \n\x12\x43omparisonOperator\x12\n\n\x06\x45QUALS\x10\x00\x42\x11\n\x0fproperty_filter\"G\n\x16\x41rtifactPropertyFilter\x12!\n\x0b\x61rtifact_id\x18\x01 \x01(\tH\x00R\nartifactIdB\n\n\x08property\"<\n\x11TagPropertyFilter\x12\x1b\n\x08tag_name\x18\x01 \x01(\tH\x00R\x07tagNameB\n\n\x08property\"[\n\x17PartitionPropertyFilter\x12\x34\n\x07key_val\x18\x01 \x01(\x0b\x32\x19.datacatalog.KeyValuePairH\x00R\x06keyValB\n\n\x08property\"6\n\x0cKeyValuePair\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"\xab\x01\n\x15\x44\x61tasetPropertyFilter\x12\x1a\n\x07project\x18\x01 \x01(\tH\x00R\x07project\x12\x14\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x12\x18\n\x06\x64omain\x18\x03 \x01(\tH\x00R\x06\x64omain\x12\x1a\n\x07version\x18\x04 \x01(\tH\x00R\x07version\x12\x1e\n\tpartition\x18\x05 \x01(\tH\x00R\tpartitionB\n\n\x08property\"\x93\x02\n\x11PaginationOptions\x12\x14\n\x05limit\x18\x01 \x01(\rR\x05limit\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\x12@\n\x07sortKey\x18\x03 \x01(\x0e\x32&.datacatalog.PaginationOptions.SortKeyR\x07sortKey\x12\x46\n\tsortOrder\x18\x04 \x01(\x0e\x32(.datacatalog.PaginationOptions.SortOrderR\tsortOrder\"*\n\tSortOrder\x12\x0e\n\nDESCENDING\x10\x00\x12\r\n\tASCENDING\x10\x01\"\x1c\n\x07SortKey\x12\x11\n\rCREATION_TIME\x10\x00\x32\x86\x07\n\x0b\x44\x61taCatalog\x12V\n\rCreateDataset\x12!.datacatalog.CreateDatasetRequest\x1a\".datacatalog.CreateDatasetResponse\x12M\n\nGetDataset\x12\x1e.datacatalog.GetDatasetRequest\x1a\x1f.datacatalog.GetDatasetResponse\x12Y\n\x0e\x43reateArtifact\x12\".datacatalog.CreateArtifactRequest\x1a#.datacatalog.CreateArtifactResponse\x12P\n\x0bGetArtifact\x12\x1f.datacatalog.GetArtifactRequest\x1a .datacatalog.GetArtifactResponse\x12\x41\n\x06\x41\x64\x64Tag\x12\x1a.datacatalog.AddTagRequest\x1a\x1b.datacatalog.AddTagResponse\x12V\n\rListArtifacts\x12!.datacatalog.ListArtifactsRequest\x1a\".datacatalog.ListArtifactsResponse\x12S\n\x0cListDatasets\x12 .datacatalog.ListDatasetsRequest\x1a!.datacatalog.ListDatasetsResponse\x12Y\n\x0eUpdateArtifact\x12\".datacatalog.UpdateArtifactRequest\x1a#.datacatalog.UpdateArtifactResponse\x12q\n\x16GetOrExtendReservation\x12*.datacatalog.GetOrExtendReservationRequest\x1a+.datacatalog.GetOrExtendReservationResponse\x12\x65\n\x12ReleaseReservation\x12&.datacatalog.ReleaseReservationRequest\x1a\'.datacatalog.ReleaseReservationResponseB\xb2\x01\n\x0f\x63om.datacatalogB\x10\x44\x61tacatalogProtoP\x01ZAgithub.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tacatalog\xca\x02\x0b\x44\x61tacatalog\xe2\x02\x17\x44\x61tacatalog\\GPBMetadata\xea\x02\x0b\x44\x61tacatalogb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -76,39 +76,39 @@
   _globals['_PARTITION']._serialized_start=2635
   _globals['_PARTITION']._serialized_end=2686
   _globals['_DATASETID']._serialized_start=2689
-  _globals['_DATASETID']._serialized_end=2846
-  _globals['_ARTIFACT']._serialized_start=2849
-  _globals['_ARTIFACT']._serialized_end=3176
-  _globals['_ARTIFACTDATA']._serialized_start=3178
-  _globals['_ARTIFACTDATA']._serialized_end=3258
-  _globals['_TAG']._serialized_start=3260
-  _globals['_TAG']._serialized_end=3368
-  _globals['_METADATA']._serialized_start=3371
-  _globals['_METADATA']._serialized_end=3500
-  _globals['_METADATA_KEYMAPENTRY']._serialized_start=3443
-  _globals['_METADATA_KEYMAPENTRY']._serialized_end=3500
-  _globals['_FILTEREXPRESSION']._serialized_start=3502
-  _globals['_FILTEREXPRESSION']._serialized_end=3581
-  _globals['_SINGLEPROPERTYFILTER']._serialized_start=3584
-  _globals['_SINGLEPROPERTYFILTER']._serialized_end=4046
-  _globals['_SINGLEPROPERTYFILTER_COMPARISONOPERATOR']._serialized_start=3995
-  _globals['_SINGLEPROPERTYFILTER_COMPARISONOPERATOR']._serialized_end=4027
-  _globals['_ARTIFACTPROPERTYFILTER']._serialized_start=4048
-  _globals['_ARTIFACTPROPERTYFILTER']._serialized_end=4119
-  _globals['_TAGPROPERTYFILTER']._serialized_start=4121
-  _globals['_TAGPROPERTYFILTER']._serialized_end=4181
-  _globals['_PARTITIONPROPERTYFILTER']._serialized_start=4183
-  _globals['_PARTITIONPROPERTYFILTER']._serialized_end=4274
-  _globals['_KEYVALUEPAIR']._serialized_start=4276
-  _globals['_KEYVALUEPAIR']._serialized_end=4330
-  _globals['_DATASETPROPERTYFILTER']._serialized_start=4333
-  _globals['_DATASETPROPERTYFILTER']._serialized_end=4504
-  _globals['_PAGINATIONOPTIONS']._serialized_start=4507
-  _globals['_PAGINATIONOPTIONS']._serialized_end=4782
-  _globals['_PAGINATIONOPTIONS_SORTORDER']._serialized_start=4710
-  _globals['_PAGINATIONOPTIONS_SORTORDER']._serialized_end=4752
-  _globals['_PAGINATIONOPTIONS_SORTKEY']._serialized_start=4754
-  _globals['_PAGINATIONOPTIONS_SORTKEY']._serialized_end=4782
-  _globals['_DATACATALOG']._serialized_start=4785
-  _globals['_DATACATALOG']._serialized_end=5687
+  _globals['_DATASETID']._serialized_end=2864
+  _globals['_ARTIFACT']._serialized_start=2867
+  _globals['_ARTIFACT']._serialized_end=3194
+  _globals['_ARTIFACTDATA']._serialized_start=3196
+  _globals['_ARTIFACTDATA']._serialized_end=3276
+  _globals['_TAG']._serialized_start=3278
+  _globals['_TAG']._serialized_end=3386
+  _globals['_METADATA']._serialized_start=3389
+  _globals['_METADATA']._serialized_end=3518
+  _globals['_METADATA_KEYMAPENTRY']._serialized_start=3461
+  _globals['_METADATA_KEYMAPENTRY']._serialized_end=3518
+  _globals['_FILTEREXPRESSION']._serialized_start=3520
+  _globals['_FILTEREXPRESSION']._serialized_end=3599
+  _globals['_SINGLEPROPERTYFILTER']._serialized_start=3602
+  _globals['_SINGLEPROPERTYFILTER']._serialized_end=4064
+  _globals['_SINGLEPROPERTYFILTER_COMPARISONOPERATOR']._serialized_start=4013
+  _globals['_SINGLEPROPERTYFILTER_COMPARISONOPERATOR']._serialized_end=4045
+  _globals['_ARTIFACTPROPERTYFILTER']._serialized_start=4066
+  _globals['_ARTIFACTPROPERTYFILTER']._serialized_end=4137
+  _globals['_TAGPROPERTYFILTER']._serialized_start=4139
+  _globals['_TAGPROPERTYFILTER']._serialized_end=4199
+  _globals['_PARTITIONPROPERTYFILTER']._serialized_start=4201
+  _globals['_PARTITIONPROPERTYFILTER']._serialized_end=4292
+  _globals['_KEYVALUEPAIR']._serialized_start=4294
+  _globals['_KEYVALUEPAIR']._serialized_end=4348
+  _globals['_DATASETPROPERTYFILTER']._serialized_start=4351
+  _globals['_DATASETPROPERTYFILTER']._serialized_end=4522
+  _globals['_PAGINATIONOPTIONS']._serialized_start=4525
+  _globals['_PAGINATIONOPTIONS']._serialized_end=4800
+  _globals['_PAGINATIONOPTIONS_SORTORDER']._serialized_start=4728
+  _globals['_PAGINATIONOPTIONS_SORTORDER']._serialized_end=4770
+  _globals['_PAGINATIONOPTIONS_SORTKEY']._serialized_start=4772
+  _globals['_PAGINATIONOPTIONS_SORTKEY']._serialized_end=4800
+  _globals['_DATACATALOG']._serialized_start=4803
+  _globals['_DATACATALOG']._serialized_end=5705
 # @@protoc_insertion_point(module_scope)
diff --git a/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.pyi
index dc74bc91af..18d6684d61 100644
--- a/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.pyi
+++ b/flyteidl/gen/pb_python/flyteidl/datacatalog/datacatalog_pb2.pyi
@@ -190,20 +190,22 @@ class Partition(_message.Message):
     def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
 
 class DatasetID(_message.Message):
-    __slots__ = ["project", "name", "domain", "version", "UUID", "partition"]
+    __slots__ = ["project", "name", "domain", "version", "UUID", "partition", "org"]
     PROJECT_FIELD_NUMBER: _ClassVar[int]
     NAME_FIELD_NUMBER: _ClassVar[int]
     DOMAIN_FIELD_NUMBER: _ClassVar[int]
     VERSION_FIELD_NUMBER: _ClassVar[int]
     UUID_FIELD_NUMBER: _ClassVar[int]
     PARTITION_FIELD_NUMBER: _ClassVar[int]
+    ORG_FIELD_NUMBER: _ClassVar[int]
     project: str
     name: str
     domain: str
     version: str
     UUID: str
     partition: str
-    def __init__(self, project: _Optional[str] = ..., name: _Optional[str] = ..., domain: _Optional[str] = ..., version: _Optional[str] = ..., UUID: _Optional[str] = ..., partition: _Optional[str] = ...) -> None: ...
+    org: str
+    def __init__(self, project: _Optional[str] = ..., name: _Optional[str] = ..., domain: _Optional[str] = ..., version: _Optional[str] = ..., UUID: _Optional[str] = ..., partition: _Optional[str] = ..., org: _Optional[str] = ...) -> None: ...
 
 class Artifact(_message.Message):
     __slots__ = ["id", "dataset", "data", "metadata", "partitions", "tags", "created_at"]
diff --git a/flyteidl/gen/pb_rust/datacatalog.rs b/flyteidl/gen/pb_rust/datacatalog.rs
index 8383308137..b24a867be9 100644
--- a/flyteidl/gen/pb_rust/datacatalog.rs
+++ b/flyteidl/gen/pb_rust/datacatalog.rs
@@ -293,6 +293,9 @@ pub struct DatasetId {
     /// Optional, partition key applied to the dataset.
     #[prost(string, tag="6")]
     pub partition: ::prost::alloc::string::String,
+    /// Optional, org key applied to the resource.
+    #[prost(string, tag="7")]
+    pub org: ::prost::alloc::string::String,
 }
 ///
 /// Artifact message. It is composed of several string fields.
diff --git a/flyteidl/protos/flyteidl/datacatalog/datacatalog.proto b/flyteidl/protos/flyteidl/datacatalog/datacatalog.proto
index 437d0e5005..46ef1c8560 100644
--- a/flyteidl/protos/flyteidl/datacatalog/datacatalog.proto
+++ b/flyteidl/protos/flyteidl/datacatalog/datacatalog.proto
@@ -291,6 +291,9 @@ message DatasetID {
     string UUID = 5;     // UUID for the dataset (if set the above fields are optional)
     // Optional, partition key applied to the dataset.
     string partition = 6;
+
+    // Optional, org key applied to the resource.
+    string org = 7;
 }
 
 /*
diff --git a/flytepropeller/pkg/controller/nodes/catalog/datacatalog/transformer.go b/flytepropeller/pkg/controller/nodes/catalog/datacatalog/transformer.go
index 5c0ac0c30b..a5cb2db58d 100644
--- a/flytepropeller/pkg/controller/nodes/catalog/datacatalog/transformer.go
+++ b/flytepropeller/pkg/controller/nodes/catalog/datacatalog/transformer.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"github.com/golang/protobuf/proto"
 	"reflect"
 	"strconv"
 	"strings"
@@ -138,6 +139,7 @@ func GenerateDatasetIDForTask(ctx context.Context, k catalog.Key) (*datacatalog.
 		Domain:  k.Identifier.Domain,
 		Name:    getDatasetNameFromTask(k.Identifier),
 		Version: datasetVersion,
+		Org:     k.Identifier.Org,
 	}
 	return datasetID, nil
 }
@@ -157,6 +159,7 @@ const (
 	execProjectKey     = "exec-project"
 	execNodeIDKey      = "exec-node"
 	execTaskAttemptKey = "exec-attempt"
+	execOrgKey         = "exec-rog"
 )
 
 // Understanding Catalog Identifiers
@@ -187,6 +190,7 @@ func GetArtifactMetadataForSource(taskExecutionID *core.TaskExecutionIdentifier)
 			execNameKey:        taskExecutionID.NodeExecutionId.GetExecutionId().GetName(),
 			execNodeIDKey:      taskExecutionID.NodeExecutionId.GetNodeId(),
 			execTaskAttemptKey: strconv.Itoa(int(taskExecutionID.GetRetryAttempt())),
+			execOrgKey:         taskExecutionID.GetNodeExecutionId().GetExecutionId().GetOrg(),
 		},
 	}
 }
@@ -209,14 +213,11 @@ func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentI
 		return nil, fmt.Errorf("failed to parse [%v] to integer. Error: %w", val, err)
 	}
 
+	taskId := proto.Clone(¤tID).(*core.Identifier)
+	taskId.Version = GetOrDefault(datasetMd.KeyMap, taskVersionKey, "unknown")
+
 	return &core.TaskExecutionIdentifier{
-		TaskId: &core.Identifier{
-			ResourceType: currentID.ResourceType,
-			Project:      currentID.Project,
-			Domain:       currentID.Domain,
-			Name:         currentID.Name,
-			Version:      GetOrDefault(datasetMd.KeyMap, taskVersionKey, "unknown"),
-		},
+		TaskId:       taskId,
 		RetryAttempt: uint32(attempt),
 		NodeExecutionId: &core.NodeExecutionIdentifier{
 			NodeId: GetOrDefault(artifactMd.KeyMap, execNodeIDKey, "unknown"),
@@ -224,6 +225,7 @@ func GetSourceFromMetadata(datasetMd, artifactMd *datacatalog.Metadata, currentI
 				Project: GetOrDefault(artifactMd.KeyMap, execProjectKey, currentID.GetProject()),
 				Domain:  GetOrDefault(artifactMd.KeyMap, execDomainKey, currentID.GetDomain()),
 				Name:    GetOrDefault(artifactMd.KeyMap, execNameKey, "unknown"),
+				Org:     GetOrDefault(artifactMd.KeyMap, execOrgKey, currentID.GetOrg()),
 			},
 		},
 	}, nil