From 53d497935d25f377130f329a8d1395e91b59b33a Mon Sep 17 00:00:00 2001 From: wayner0628 Date: Wed, 6 Nov 2024 01:34:04 -0800 Subject: [PATCH 1/5] add friendly name execution object column Signed-off-by: wayner0628 --- flyteadmin/go.mod | 1 + flyteadmin/go.sum | 2 ++ .../pkg/repositories/models/execution.go | 1 + .../repositories/transformers/execution.go | 2 ++ .../transformers/friendly_name.go | 17 +++++++++++ .../transformers/friendly_name_test.go | 29 +++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ 8 files changed, 55 insertions(+) create mode 100644 flyteadmin/pkg/repositories/transformers/friendly_name.go create mode 100644 flyteadmin/pkg/repositories/transformers/friendly_name_test.go diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index 5c008a46eb..6c7ccd7bc3 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -221,6 +221,7 @@ require ( github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 // indirect github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2 v2.8.0 github.com/imdario/mergo v0.3.13 // indirect + github.com/wolfeidau/humanhash v1.1.0 k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index ec5e0cdc1c..cf317a4ba9 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1304,6 +1304,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wolfeidau/humanhash v1.1.0 h1:06KgtyyABJGBbrfMONrW7S+b5TTYVyrNB/jss5n7F3E= +github.com/wolfeidau/humanhash v1.1.0/go.mod h1:jkpynR1bfyfkmKEQudIC0osWKynFAoayRjzH9OJdVIg= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= diff --git a/flyteadmin/pkg/repositories/models/execution.go b/flyteadmin/pkg/repositories/models/execution.go index f148c95faf..7998b3b003 100644 --- a/flyteadmin/pkg/repositories/models/execution.go +++ b/flyteadmin/pkg/repositories/models/execution.go @@ -26,6 +26,7 @@ type Execution struct { LaunchPlanID uint `gorm:"index"` WorkflowID uint `gorm:"index"` TaskID uint `gorm:"index"` + FriendlyName string `valid:"length(0|255)"` Phase string `valid:"length(0|255)"` Closure []byte Spec []byte `gorm:"not null"` diff --git a/flyteadmin/pkg/repositories/transformers/execution.go b/flyteadmin/pkg/repositories/transformers/execution.go index 711f6bdddb..2b4867a1e4 100644 --- a/flyteadmin/pkg/repositories/transformers/execution.go +++ b/flyteadmin/pkg/repositories/transformers/execution.go @@ -74,6 +74,7 @@ func CreateExecutionModel(input CreateExecutionModelInput) (*models.Execution, e return nil, flyteErrs.NewFlyteAdminErrorf(codes.Internal, "Failed to serialize execution spec: %v", err) } createdAt := timestamppb.New(input.CreatedAt) + friendlyName := CreateFriendlyName(input.CreatedAt.UnixNano()) closure := admin.ExecutionClosure{ CreatedAt: createdAt, UpdatedAt: createdAt, @@ -119,6 +120,7 @@ func CreateExecutionModel(input CreateExecutionModelInput) (*models.Execution, e Name: input.WorkflowExecutionID.Name, }, Spec: spec, + FriendlyName: friendlyName, Phase: closure.Phase.String(), Closure: closureBytes, WorkflowID: input.WorkflowID, diff --git a/flyteadmin/pkg/repositories/transformers/friendly_name.go b/flyteadmin/pkg/repositories/transformers/friendly_name.go new file mode 100644 index 0000000000..7c976a2ed1 --- /dev/null +++ b/flyteadmin/pkg/repositories/transformers/friendly_name.go @@ -0,0 +1,17 @@ +package transformers + +import ( + "github.com/wolfeidau/humanhash" + "k8s.io/apimachinery/pkg/util/rand" +) + +const RandStringLength = 20 + +/* #nosec */ +func CreateFriendlyName(seed int64) string { + rand.Seed(seed) + hashKey := []byte(rand.String(RandStringLength)) + // Ignoring the error as it's guaranteed hash key longer than result in this context. + result, _ := humanhash.Humanize(hashKey, 4) + return result +} diff --git a/flyteadmin/pkg/repositories/transformers/friendly_name_test.go b/flyteadmin/pkg/repositories/transformers/friendly_name_test.go new file mode 100644 index 0000000000..0da70bac3e --- /dev/null +++ b/flyteadmin/pkg/repositories/transformers/friendly_name_test.go @@ -0,0 +1,29 @@ +package transformers + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const AllowedFriendlyNameStr = "abcdefghijklmnopqrstuvwxyz-" +const FriendlyNameLengthLimit = 255 + +var AllowedFriendlyNameChars = []rune(AllowedFriendlyNameStr) + +func TestGetExecutionName(t *testing.T) { + t.Run("friendly name", func(t *testing.T) { + randString := CreateFriendlyName(time.Now().UnixNano()) + assert.LessOrEqual(t, len(randString), FriendlyNameLengthLimit) + for i := 0; i < len(randString); i++ { + assert.Contains(t, AllowedFriendlyNameChars, rune(randString[i])) + } + hyphenCount := strings.Count(randString, "-") + assert.Equal(t, 3, hyphenCount, "FriendlyName should contain exactly three hyphens") + words := strings.Split(randString, "-") + assert.Equal(t, 4, len(words), "FriendlyName should be split into exactly four words") + }) + +} diff --git a/go.mod b/go.mod index 776eb1abc8..19f187b054 100644 --- a/go.mod +++ b/go.mod @@ -180,6 +180,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/wI2L/jsondiff v0.5.0 // indirect + github.com/wolfeidau/humanhash v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index 59ffb4358b..27c282f97b 100644 --- a/go.sum +++ b/go.sum @@ -1343,6 +1343,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wolfeidau/humanhash v1.1.0 h1:06KgtyyABJGBbrfMONrW7S+b5TTYVyrNB/jss5n7F3E= +github.com/wolfeidau/humanhash v1.1.0/go.mod h1:jkpynR1bfyfkmKEQudIC0osWKynFAoayRjzH9OJdVIg= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= From e59171582f6ee61b978695b32794acab1a35136d Mon Sep 17 00:00:00 2001 From: wayner0628 Date: Wed, 6 Nov 2024 02:14:31 -0800 Subject: [PATCH 2/5] fix execution model db mock Signed-off-by: wayner0628 --- flyteadmin/pkg/repositories/gormimpl/execution_repo_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flyteadmin/pkg/repositories/gormimpl/execution_repo_test.go b/flyteadmin/pkg/repositories/gormimpl/execution_repo_test.go index e1e3117c6e..8220fe4fc7 100644 --- a/flyteadmin/pkg/repositories/gormimpl/execution_repo_test.go +++ b/flyteadmin/pkg/repositories/gormimpl/execution_repo_test.go @@ -96,6 +96,7 @@ func getMockExecutionResponseFromDb(expected models.Execution) map[string]interf execution["execution_name"] = expected.Name execution["launch_plan_id"] = expected.LaunchPlanID execution["workflow_id"] = expected.WorkflowID + execution["friendly_name"] = expected.FriendlyName execution["phase"] = expected.Phase execution["closure"] = expected.Closure execution["spec"] = expected.Spec @@ -381,6 +382,7 @@ func TestListExecutionsForWorkflow(t *testing.T) { }, LaunchPlanID: uint(2), WorkflowID: uint(3), + FriendlyName: "its-a-friendly-name", Phase: core.WorkflowExecution_SUCCEEDED.String(), Closure: []byte{1, 2}, Spec: []byte{3, 4}, @@ -394,7 +396,7 @@ func TestListExecutionsForWorkflow(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true // Only match on queries that append expected filters - GlobalMock.NewMock().WithQuery(`SELECT "executions"."id","executions"."created_at","executions"."updated_at","executions"."deleted_at","executions"."execution_project","executions"."execution_domain","executions"."execution_name","executions"."launch_plan_id","executions"."workflow_id","executions"."task_id","executions"."phase","executions"."closure","executions"."spec","executions"."started_at","executions"."execution_created_at","executions"."execution_updated_at","executions"."duration","executions"."abort_cause","executions"."mode","executions"."source_execution_id","executions"."parent_node_execution_id","executions"."cluster","executions"."inputs_uri","executions"."user_inputs_uri","executions"."error_kind","executions"."error_code","executions"."user","executions"."state","executions"."launch_entity" FROM "executions" INNER JOIN workflows ON executions.workflow_id = workflows.id INNER JOIN tasks ON executions.task_id = tasks.id WHERE executions.execution_project = $1 AND executions.execution_domain = $2 AND executions.execution_name = $3 AND workflows.name = $4 AND tasks.name = $5 AND execution_tags.key in ($6,$7) LIMIT 20`).WithReply(executions) + GlobalMock.NewMock().WithQuery(`SELECT "executions"."id","executions"."created_at","executions"."updated_at","executions"."deleted_at","executions"."execution_project","executions"."execution_domain","executions"."execution_name","executions"."launch_plan_id","executions"."workflow_id","executions"."task_id","executions"."friendly_name","executions"."phase","executions"."closure","executions"."spec","executions"."started_at","executions"."execution_created_at","executions"."execution_updated_at","executions"."duration","executions"."abort_cause","executions"."mode","executions"."source_execution_id","executions"."parent_node_execution_id","executions"."cluster","executions"."inputs_uri","executions"."user_inputs_uri","executions"."error_kind","executions"."error_code","executions"."user","executions"."state","executions"."launch_entity" FROM "executions" INNER JOIN workflows ON executions.workflow_id = workflows.id INNER JOIN tasks ON executions.task_id = tasks.id WHERE executions.execution_project = $1 AND executions.execution_domain = $2 AND executions.execution_name = $3 AND workflows.name = $4 AND tasks.name = $5 AND execution_tags.key in ($6,$7) LIMIT 20`).WithReply(executions) vals := []string{"tag1", "tag2"} tagFilter, err := common.NewRepeatedValueFilter(common.AdminTag, common.ValueIn, "name", vals) assert.NoError(t, err) From ca8943dd7f6d4c0364deac6c9852335c52916bfd Mon Sep 17 00:00:00 2001 From: wayner0628 Date: Wed, 6 Nov 2024 02:32:58 -0800 Subject: [PATCH 3/5] typo Signed-off-by: wayner0628 --- .../pkg/repositories/transformers/friendly_name_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flyteadmin/pkg/repositories/transformers/friendly_name_test.go b/flyteadmin/pkg/repositories/transformers/friendly_name_test.go index 0da70bac3e..a651541712 100644 --- a/flyteadmin/pkg/repositories/transformers/friendly_name_test.go +++ b/flyteadmin/pkg/repositories/transformers/friendly_name_test.go @@ -13,8 +13,8 @@ const FriendlyNameLengthLimit = 255 var AllowedFriendlyNameChars = []rune(AllowedFriendlyNameStr) -func TestGetExecutionName(t *testing.T) { - t.Run("friendly name", func(t *testing.T) { +func TestCreateFriendlyName(t *testing.T) { + t.Run("successful creation", func(t *testing.T) { randString := CreateFriendlyName(time.Now().UnixNano()) assert.LessOrEqual(t, len(randString), FriendlyNameLengthLimit) for i := 0; i < len(randString); i++ { From 9dead1d6d4a370cfc8e3cd6bedc6a8d7c48f36c9 Mon Sep 17 00:00:00 2001 From: wayner0628 Date: Tue, 12 Nov 2024 04:48:24 -0800 Subject: [PATCH 4/5] add comment for random string length Signed-off-by: wayner0628 --- flyteadmin/pkg/repositories/transformers/friendly_name.go | 1 + 1 file changed, 1 insertion(+) diff --git a/flyteadmin/pkg/repositories/transformers/friendly_name.go b/flyteadmin/pkg/repositories/transformers/friendly_name.go index 7c976a2ed1..3e44df7406 100644 --- a/flyteadmin/pkg/repositories/transformers/friendly_name.go +++ b/flyteadmin/pkg/repositories/transformers/friendly_name.go @@ -5,6 +5,7 @@ import ( "k8s.io/apimachinery/pkg/util/rand" ) +// Length of the random string used for generating hash keys; can be any positive integer const RandStringLength = 20 /* #nosec */ From c9914f1caed1f9e5a510c93ec13d5f6635c995d1 Mon Sep 17 00:00:00 2001 From: wayner0628 Date: Sat, 30 Nov 2024 20:45:29 -0800 Subject: [PATCH 5/5] rename const and update migration Signed-off-by: wayner0628 --- .../pkg/repositories/config/migrations.go | 19 +++++++++++++++++++ .../transformers/friendly_name.go | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/flyteadmin/pkg/repositories/config/migrations.go b/flyteadmin/pkg/repositories/config/migrations.go index d48356fe01..bd87700708 100644 --- a/flyteadmin/pkg/repositories/config/migrations.go +++ b/flyteadmin/pkg/repositories/config/migrations.go @@ -1285,6 +1285,25 @@ var ContinuedMigrations = []*gormigrate.Migration{ return nil }, }, + { + ID: "2024-11-30-add-friendly-name-to-execution-tags", + Migrate: func(tx *gorm.DB) error { + // Alter table and add new column `friendly_name` + if err := tx.Exec("ALTER TABLE execution_tags ADD COLUMN friendly_name varchar(255);").Error; err != nil { + return err + } + + return nil + }, + Rollback: func(tx *gorm.DB) error { + // Drop the `friendly_name` column + if err := tx.Exec("ALTER TABLE execution_tags DROP COLUMN IF EXISTS friendly_name;").Error; err != nil { + return err + } + + return nil + }, + }, } var m = append(LegacyMigrations, NoopMigrations...) diff --git a/flyteadmin/pkg/repositories/transformers/friendly_name.go b/flyteadmin/pkg/repositories/transformers/friendly_name.go index 3e44df7406..e7de414fbf 100644 --- a/flyteadmin/pkg/repositories/transformers/friendly_name.go +++ b/flyteadmin/pkg/repositories/transformers/friendly_name.go @@ -6,12 +6,12 @@ import ( ) // Length of the random string used for generating hash keys; can be any positive integer -const RandStringLength = 20 +const defaultFriendlyNameLength = 20 /* #nosec */ func CreateFriendlyName(seed int64) string { rand.Seed(seed) - hashKey := []byte(rand.String(RandStringLength)) + hashKey := []byte(rand.String(defaultFriendlyNameLength)) // Ignoring the error as it's guaranteed hash key longer than result in this context. result, _ := humanhash.Humanize(hashKey, 4) return result