diff --git a/flyteadmin/pkg/common/filters.go b/flyteadmin/pkg/common/filters.go index 57756e7820..697bb75179 100644 --- a/flyteadmin/pkg/common/filters.go +++ b/flyteadmin/pkg/common/filters.go @@ -22,6 +22,7 @@ type GormQueryExpr struct { // Complete set of filters available for database queries. const ( Contains FilterExpression = iota + NotLike GreaterThan GreaterThanOrEqual LessThan @@ -37,6 +38,7 @@ const ( joinArgsFormat = "%s.%s" containsQuery = "%s LIKE ?" containsArgs = "%%%s%%" + notLikeQuery = "%s NOT LIKE ?" greaterThanQuery = "%s > ?" greaterThanOrEqualQuery = "%s >= ?" lessThanQuery = "%s < ?" @@ -50,6 +52,7 @@ const ( // Set of available filters which exclusively accept a single argument value. var singleValueFilters = map[FilterExpression]bool{ Contains: true, + NotLike: true, GreaterThan: true, GreaterThanOrEqual: true, LessThan: true, @@ -68,6 +71,7 @@ const EqualExpression = "eq" var filterNameMappings = map[string]FilterExpression{ "contains": Contains, + "not_like": NotLike, "gt": GreaterThan, "gte": GreaterThanOrEqual, "lt": LessThan, @@ -80,6 +84,7 @@ var filterNameMappings = map[string]FilterExpression{ var filterQueryMappings = map[FilterExpression]string{ Contains: containsQuery, + NotLike: notLikeQuery, GreaterThan: greaterThanQuery, GreaterThanOrEqual: greaterThanOrEqualQuery, LessThan: lessThanQuery, @@ -117,6 +122,8 @@ func getFilterExpressionName(expression FilterExpression) string { switch expression { case Contains: return "contains" + case NotLike: + return "not like" case GreaterThan: return "greater than" case GreaterThanOrEqual: @@ -208,6 +215,12 @@ func (f *inlineFilterImpl) getGormQueryExpr(formattedField string) (GormQueryExp // args renders to something like: "%value%" Args: fmt.Sprintf(containsArgs, f.value), }, nil + case NotLike: + return GormQueryExpr{ + // WHERE field NOT LIKE value + Query: fmt.Sprintf(notLikeQuery, formattedField), + Args: f.value, + }, nil case GreaterThan: return GormQueryExpr{ // WHERE field > value diff --git a/flyteadmin/pkg/common/filters_test.go b/flyteadmin/pkg/common/filters_test.go index 87ba5ac2ac..85092e36b6 100644 --- a/flyteadmin/pkg/common/filters_test.go +++ b/flyteadmin/pkg/common/filters_test.go @@ -107,6 +107,7 @@ func TestGetGormJoinTableQueryExpr(t *testing.T) { var expectedArgsForFilters = map[FilterExpression]string{ Contains: "%value%", + NotLike: "value", GreaterThan: "value", GreaterThanOrEqual: "value", LessThan: "value", @@ -169,3 +170,18 @@ func TestWithDefaultValueFilter(t *testing.T) { assert.Equal(t, "COALESCE(named_entity_metadata.state, 0) = ?", queryExpression.Query) assert.Equal(t, 1, queryExpression.Args) } + +func TestNotLikeFilter(t *testing.T) { + filter, err := NewSingleValueFilter(NamedEntityMetadata, NotLike, "name", ".flytegen%") + assert.NoError(t, err) + + queryExpression, err := filter.GetGormQueryExpr() + assert.NoError(t, err) + assert.Equal(t, "name NOT LIKE ?", queryExpression.Query) + assert.Equal(t, ".flytegen%", queryExpression.Args) + + queryExpression, err = filter.GetGormJoinTableQueryExpr("named_entity_metadata") + assert.NoError(t, err) + assert.Equal(t, "named_entity_metadata.name NOT LIKE ?", queryExpression.Query) + assert.Equal(t, ".flytegen%", queryExpression.Args) +} diff --git a/flyteadmin/pkg/manager/impl/execution_manager.go b/flyteadmin/pkg/manager/impl/execution_manager.go index 27acf152ec..e700a744d8 100644 --- a/flyteadmin/pkg/manager/impl/execution_manager.go +++ b/flyteadmin/pkg/manager/impl/execution_manager.go @@ -480,7 +480,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( return nil, nil, err } - launchPlan, err := util.CreateOrGetLaunchPlan(ctx, m.db, m.config, taskIdentifier, + launchPlan, err := util.CreateOrGetLaunchPlan(ctx, m.db, m.config, m.namedEntityManager, taskIdentifier, workflow.Closure.CompiledWorkflow.Primary.Template.Interface, workflowModel.ID, request.Spec) if err != nil { return nil, nil, err diff --git a/flyteadmin/pkg/manager/impl/named_entity_manager.go b/flyteadmin/pkg/manager/impl/named_entity_manager.go index 329061c4d3..883948318a 100644 --- a/flyteadmin/pkg/manager/impl/named_entity_manager.go +++ b/flyteadmin/pkg/manager/impl/named_entity_manager.go @@ -2,6 +2,7 @@ package impl import ( "context" + "fmt" "strconv" "strings" @@ -17,7 +18,6 @@ import ( "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/transformers" runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/contextutils" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" @@ -25,13 +25,6 @@ import ( const state = "state" -// System-generated workflows are meant to be hidden from the user by default. Therefore we always only show -// workflow-type named entities that have been user generated only. -var nonSystemGeneratedWorkflowsFilter, _ = common.NewSingleValueFilter( - common.NamedEntityMetadata, common.NotEqual, state, admin.NamedEntityState_SYSTEM_GENERATED) -var defaultWorkflowsFilter, _ = common.NewWithDefaultValueFilter( - strconv.Itoa(int(admin.NamedEntityState_NAMED_ENTITY_ACTIVE)), nonSystemGeneratedWorkflowsFilter) - type NamedEntityMetrics struct { Scope promutils.Scope } @@ -75,12 +68,8 @@ func (m *NamedEntityManager) GetNamedEntity(ctx context.Context, request *admin. return util.GetNamedEntity(ctx, m.db, request.ResourceType, request.Id) } -func (m *NamedEntityManager) getQueryFilters(referenceEntity core.ResourceType, requestFilters string) ([]common.InlineFilter, error) { +func (m *NamedEntityManager) getQueryFilters(requestFilters string) ([]common.InlineFilter, error) { filters := make([]common.InlineFilter, 0) - if referenceEntity == core.ResourceType_WORKFLOW { - filters = append(filters, defaultWorkflowsFilter) - } - if len(requestFilters) == 0 { return filters, nil } @@ -111,10 +100,14 @@ func (m *NamedEntityManager) ListNamedEntities(ctx context.Context, request *adm } ctx = contextutils.WithProjectDomain(ctx, request.Project, request.Domain) + if len(request.Filters) == 0 { + // Add implicit filter to exclude system generated workflows + request.Filters = fmt.Sprintf("not_like(name,%s)", ".flytegen%") + } // HACK: In order to filter by state (if requested) - we need to amend the filter to use COALESCE // e.g. eq(state, 1) becomes 'WHERE (COALESCE(state, 0) = '1')' since not every NamedEntity necessarily // has an entry, and therefore the default state value '0' (active), should be assumed. - filters, err := m.getQueryFilters(request.ResourceType, request.Filters) + filters, err := m.getQueryFilters(request.Filters) if err != nil { return nil, err } diff --git a/flyteadmin/pkg/manager/impl/named_entity_manager_test.go b/flyteadmin/pkg/manager/impl/named_entity_manager_test.go index 6bdd5620c5..0d009753c8 100644 --- a/flyteadmin/pkg/manager/impl/named_entity_manager_test.go +++ b/flyteadmin/pkg/manager/impl/named_entity_manager_test.go @@ -87,7 +87,7 @@ func TestNamedEntityManager_Get_BadRequest(t *testing.T) { func TestNamedEntityManager_getQueryFilters(t *testing.T) { repository := getMockRepositoryForNETest() manager := NewNamedEntityManager(repository, getMockConfigForNETest(), mockScope.NewTestScope()) - updatedFilters, err := manager.(*NamedEntityManager).getQueryFilters(core.ResourceType_TASK, "eq(state, 0)") + updatedFilters, err := manager.(*NamedEntityManager).getQueryFilters("eq(state, 0)") assert.NoError(t, err) assert.Len(t, updatedFilters, 1) @@ -97,13 +97,9 @@ func TestNamedEntityManager_getQueryFilters(t *testing.T) { assert.Equal(t, "COALESCE(state, 0) = ?", queryExp.Query) assert.Equal(t, "0", queryExp.Args) - updatedFilters, err = manager.(*NamedEntityManager).getQueryFilters(core.ResourceType_WORKFLOW, "") + updatedFilters, err = manager.(*NamedEntityManager).getQueryFilters("") assert.NoError(t, err) - assert.Len(t, updatedFilters, 1) - queryExp, err = updatedFilters[0].GetGormQueryExpr() - assert.NoError(t, err) - assert.Equal(t, "COALESCE(state, 0) <> ?", queryExp.Query) - assert.Equal(t, admin.NamedEntityState_SYSTEM_GENERATED, queryExp.Args) + assert.Len(t, updatedFilters, 0) } func TestNamedEntityManager_Update(t *testing.T) { diff --git a/flyteadmin/pkg/manager/impl/util/single_task_execution.go b/flyteadmin/pkg/manager/impl/util/single_task_execution.go index a82750f733..036610a9ec 100644 --- a/flyteadmin/pkg/manager/impl/util/single_task_execution.go +++ b/flyteadmin/pkg/manager/impl/util/single_task_execution.go @@ -165,7 +165,7 @@ func CreateOrGetWorkflowModel( } func CreateOrGetLaunchPlan(ctx context.Context, - db repositoryInterfaces.Repository, config runtimeInterfaces.Configuration, taskIdentifier *core.Identifier, + db repositoryInterfaces.Repository, config runtimeInterfaces.Configuration, namedEntityManager interfaces.NamedEntityInterface, taskIdentifier *core.Identifier, workflowInterface *core.TypedInterface, workflowID uint, spec *admin.ExecutionSpec) (*admin.LaunchPlan, error) { var launchPlan *admin.LaunchPlan var err error @@ -226,6 +226,19 @@ func CreateOrGetLaunchPlan(ctx context.Context, logger.Errorf(ctx, "Failed to save launch plan model [%+v] with err: %v", launchPlanIdentifier, err) return nil, err } + _, err = namedEntityManager.UpdateNamedEntity(ctx, &admin.NamedEntityUpdateRequest{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Id: &admin.NamedEntityIdentifier{ + Project: launchPlan.GetId().GetProject(), + Domain: launchPlan.GetId().GetDomain(), + Name: launchPlan.GetId().GetName(), + }, + Metadata: &admin.NamedEntityMetadata{State: admin.NamedEntityState_SYSTEM_GENERATED}, + }) + if err != nil { + logger.Warningf(ctx, "Failed to set launch plan state to system-generated: %v", err) + return nil, err + } } return launchPlan, nil diff --git a/flyteadmin/pkg/manager/impl/util/single_task_execution_test.go b/flyteadmin/pkg/manager/impl/util/single_task_execution_test.go index 7b64f142f9..13ed4a945d 100644 --- a/flyteadmin/pkg/manager/impl/util/single_task_execution_test.go +++ b/flyteadmin/pkg/manager/impl/util/single_task_execution_test.go @@ -217,6 +217,21 @@ func TestCreateOrGetLaunchPlan(t *testing.T) { }, } workflowID := uint(12) + + mockNamedEntityManager := managerMocks.NamedEntityManager{} + mockNamedEntityManager.UpdateNamedEntityFunc = func(ctx context.Context, request *admin.NamedEntityUpdateRequest) (*admin.NamedEntityUpdateResponse, error) { + assert.Equal(t, request.ResourceType, core.ResourceType_LAUNCH_PLAN) + assert.True(t, proto.Equal(request.Id, &admin.NamedEntityIdentifier{ + Project: "flytekit", + Domain: "production", + Name: ".flytegen.app.workflows.MyWorkflow.my_task", + }), fmt.Sprintf("%+v", request.Id)) + assert.True(t, proto.Equal(request.Metadata, &admin.NamedEntityMetadata{ + State: admin.NamedEntityState_SYSTEM_GENERATED, + })) + return &admin.NamedEntityUpdateResponse{}, nil + } + taskIdentifier := &core.Identifier{ ResourceType: core.ResourceType_TASK, Project: "flytekit", @@ -233,7 +248,7 @@ func TestCreateOrGetLaunchPlan(t *testing.T) { }, } launchPlan, err := CreateOrGetLaunchPlan( - context.Background(), repository, config, taskIdentifier, workflowInterface, workflowID, &spec) + context.Background(), repository, config, &mockNamedEntityManager, taskIdentifier, workflowInterface, workflowID, &spec) assert.NoError(t, err) assert.True(t, proto.Equal(&core.Identifier{ ResourceType: core.ResourceType_LAUNCH_PLAN, diff --git a/flyteadmin/pkg/manager/impl/validation/named_entity_validator.go b/flyteadmin/pkg/manager/impl/validation/named_entity_validator.go index 685bdfe4b5..e9af05f527 100644 --- a/flyteadmin/pkg/manager/impl/validation/named_entity_validator.go +++ b/flyteadmin/pkg/manager/impl/validation/named_entity_validator.go @@ -10,7 +10,7 @@ import ( "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) -var archivableResourceTypes = sets.NewInt32(int32(core.ResourceType_WORKFLOW), int32(core.ResourceType_TASK)) +var archivableResourceTypes = sets.NewInt32(int32(core.ResourceType_WORKFLOW), int32(core.ResourceType_TASK), int32(core.ResourceType_LAUNCH_PLAN)) func ValidateNamedEntityGetRequest(request *admin.NamedEntityGetRequest) error { if err := ValidateResourceType(request.ResourceType); err != nil { diff --git a/flyteadmin/pkg/manager/impl/validation/named_entity_validator_test.go b/flyteadmin/pkg/manager/impl/validation/named_entity_validator_test.go index cec2af94ee..025d4eca0c 100644 --- a/flyteadmin/pkg/manager/impl/validation/named_entity_validator_test.go +++ b/flyteadmin/pkg/manager/impl/validation/named_entity_validator_test.go @@ -109,7 +109,7 @@ func TestValidateNamedEntityUpdateRequest(t *testing.T) { }, })) assert.Equal(t, codes.InvalidArgument, ValidateNamedEntityUpdateRequest(&admin.NamedEntityUpdateRequest{ - ResourceType: core.ResourceType_LAUNCH_PLAN, + ResourceType: core.ResourceType_DATASET, Id: &admin.NamedEntityIdentifier{ Project: "project", Domain: "domain", @@ -141,6 +141,30 @@ func TestValidateNamedEntityUpdateRequest(t *testing.T) { State: admin.NamedEntityState_NAMED_ENTITY_ARCHIVED, }, })) + + assert.Nil(t, ValidateNamedEntityUpdateRequest(&admin.NamedEntityUpdateRequest{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Id: &admin.NamedEntityIdentifier{ + Project: "project", + Domain: "domain", + Name: "name", + }, + Metadata: &admin.NamedEntityMetadata{ + Description: "description", + }, + })) + + assert.Nil(t, ValidateNamedEntityUpdateRequest(&admin.NamedEntityUpdateRequest{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Id: &admin.NamedEntityIdentifier{ + Project: "project", + Domain: "domain", + Name: "name", + }, + Metadata: &admin.NamedEntityMetadata{ + State: admin.NamedEntityState_NAMED_ENTITY_ARCHIVED, + }, + })) } func TestValidateNamedEntityListRequest(t *testing.T) {