feat: add support for seedProjects with descriptions via seedProjects…

…WithDetails (flyteorg#5864) * feat: add support for seedProjects with descriptions via seedProjectsWithDetails Signed-off-by: terryhung <[email protected]> * fix: update the logging for seeding projects Signed-off-by: terry.hung <[email protected]> * fix: the helm lint error and add more unit test Signed-off-by: terry.hung <[email protected]> * feat: add more unitest and fix the helm diff Signed-off-by: terry.hung <[email protected]> --------- Signed-off-by: terryhung <[email protected]> Signed-off-by: terry.hung <[email protected]>
popojk · Oct 25, 2024 · 1279740 · 1279740
1 parent 7f92953
commit 1279740
Show file tree

Hide file tree

Showing 9 changed files with 371 additions and 15 deletions.
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -116,6 +116,8 @@ Chart for basic single Flyte executable deployment
 | flyte-core-components.admin.disableClusterResourceManager | bool | `false` |  |
 | flyte-core-components.admin.disableScheduler | bool | `false` |  |
 | flyte-core-components.admin.disabled | bool | `false` |  |
+| flyte-core-components.admin.seedProjectsWithDetails[0].description | string | `"Default project setup."` |  |
+| flyte-core-components.admin.seedProjectsWithDetails[0].name | string | `"flytesnacks"` |  |
 | flyte-core-components.admin.seedProjects[0] | string | `"flytesnacks"` |  |
 | flyte-core-components.dataCatalog.disabled | bool | `false` |  |
 | flyte-core-components.propeller.disableWebhook | bool | `false` |  |

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -20,6 +20,14 @@ flyte-core-components:
     # seedProjects flyte projects to create by default
     seedProjects:
       - flytesnacks
+    # seedProjectsWithDetails flyte projects to create by default with description
+    # If there is an overlap between seedProjects and seedProjectsWithDetails,
+    # the description provided in seedProjectsWithDetails will take precedence.
+    # For seedProjects without a corresponding description in seedProjectsWithDetails,
+    # a default description will be auto-generated for the project.
+    seedProjectsWithDetails:
+      - name: flytesnacks
+        description: Default project setup.
   # propeller Configuration to disable propeller or any of its components
   propeller:
     # disabled Disables flytepropeller

diff --git a/cmd/single/config.go b/cmd/single/config.go
@@ -1,6 +1,9 @@
 package single
 
-import "github.com/flyteorg/flyte/flytestdlib/config"
+import (
+	adminRepositoriesConfig "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
+	"github.com/flyteorg/flyte/flytestdlib/config"
+)
 
 //go:generate pflags Config --default-var=DefaultConfig
 
@@ -21,10 +24,11 @@ type Propeller struct {
 }
 
 type Admin struct {
-	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
-	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
-	DisableClusterResourceManager bool     `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
-	SeedProjects                  []string `json:"seedProjects" pflag:",flyte projects to create by default."`
+	Disabled                      bool                                  `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
+	DisableScheduler              bool                                  `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
+	DisableClusterResourceManager bool                                  `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
+	SeedProjects                  []string                              `json:"seedProjects" pflag:",flyte projects to create by default."`
+	SeedProjectsWithDetails       []adminRepositoriesConfig.SeedProject `json:"seedProjectsWithDetails" pflag:",,Detailed configuration for Flyte projects to be created by default."`
 }
 
 type DataCatalog struct {

diff --git a/cmd/single/start.go b/cmd/single/start.go
@@ -22,6 +22,7 @@ import (
 	datacatalog "github.com/flyteorg/flyte/datacatalog/pkg/rpc/datacatalogservice"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/clusterresource"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/common"
+	adminRepositoriesConfig "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/runtime"
 	adminServer "github.com/flyteorg/flyte/flyteadmin/pkg/server"
 	"github.com/flyteorg/flyte/flyteadmin/plugins"
@@ -75,8 +76,9 @@ func startAdmin(ctx context.Context, cfg Admin) error {
 	if len(cfg.SeedProjects) != 0 {
 		projects = cfg.SeedProjects
 	}
-	logger.Infof(ctx, "Seeding default projects...", projects)
-	if err := adminServer.SeedProjects(ctx, projects); err != nil {
+	seedProjects := adminRepositoriesConfig.MergeSeedProjectsWithUniqueNames(projects, cfg.SeedProjectsWithDetails)
+	logger.Infof(ctx, "Seeding default projects... %v", seedProjects)
+	if err := adminServer.SeedProjects(ctx, seedProjects); err != nil {
 		return err
 	}
 

diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml
@@ -92,6 +92,9 @@ data:
         disabled: false
         seedProjects:
         - flytesnacks
+        seedProjectsWithDetails:
+        - description: Default project setup.
+          name: flytesnacks
       dataCatalog:
         disabled: false
       propeller:
@@ -360,7 +363,7 @@ spec:
         app.kubernetes.io/instance: flyte
         app.kubernetes.io/component: flyte-binary
       annotations:
-        checksum/configuration: faaefbd3b3b2ddfd4e718bd77c02c632c75e7111dad0a6e25dc415dc88add73f
+        checksum/configuration: 886440a42b3eeec802cfe60d37885f69e35ffd83e53e625b3c877da5e8c7eb38
         checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:

diff --git a/flyteadmin/cmd/entrypoints/migrate.go b/flyteadmin/cmd/entrypoints/migrate.go
@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	_ "gorm.io/driver/postgres" // Required to import database driver.
 
+	"github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
 	"github.com/flyteorg/flyte/flyteadmin/pkg/server"
 )
 
@@ -40,7 +41,8 @@ var seedProjectsCmd = &cobra.Command{
 	Short: "Seed projects in the database.",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx := context.Background()
-		return server.SeedProjects(ctx, args)
+		seedProjects := config.UniqueProjectsFromNames(args)
+		return server.SeedProjects(ctx, seedProjects)
 	},
 }
 

diff --git a/flyteadmin/pkg/repositories/config/seed_data.go b/flyteadmin/pkg/repositories/config/seed_data.go
@@ -10,16 +10,61 @@ import (
 	"github.com/flyteorg/flyte/flytestdlib/logger"
 )
 
+type SeedProject struct {
+	Name        string `json:"name" pflag:",Name of flyte project to create"`
+	Description string `json:"description" pflag:",Description of flyte project to create"`
+}
+
+func UniqueProjectsFromNames(names []string) []SeedProject {
+	return uniqueProjects(names, nil)
+}
+
+// MergeSeedProjectsWithUniqueNames merges seed projects from names and details while maintaining uniqueness
+func MergeSeedProjectsWithUniqueNames(seedProjects []string, seedProjectsWithDetails []SeedProject) []SeedProject {
+	return uniqueProjects(seedProjects, seedProjectsWithDetails)
+}
+
+func uniqueProjects(seedProjects []string, seedProjectsWithDetails []SeedProject) []SeedProject {
+	// Track unique project names
+	seen := make(map[string]struct{})
+
+	// Create the final result slice
+	var combinedProjects []SeedProject
+
+	// First, add all projects from SeedProjectsWithDetails to the map
+	for _, project := range seedProjectsWithDetails {
+		// Handle the duplication
+		if _, exists := seen[project.Name]; !exists {
+			seen[project.Name] = struct{}{}
+			combinedProjects = append(combinedProjects, project)
+		}
+	}
+
+	// Process SeedProjects
+	for _, projectName := range seedProjects {
+		// Check if project not exists in SeedProjectsWithDetails
+		if _, exists := seen[projectName]; !exists {
+			seen[projectName] = struct{}{}
+			combinedProjects = append(combinedProjects, SeedProject{
+				Name:        projectName,
+				Description: fmt.Sprintf("%s description", projectName),
+			})
+		}
+	}
+
+	return combinedProjects
+}
+
 // Returns a function to seed the database with default values.
-func SeedProjects(db *gorm.DB, projects []string) error {
+func SeedProjects(db *gorm.DB, projects []SeedProject) error {
 	tx := db.Begin()
 	for _, project := range projects {
 		projectModel := models.Project{
-			Identifier:  project,
-			Name:        project,
-			Description: fmt.Sprintf("%s description", project),
+			Identifier:  project.Name,
+			Name:        project.Name,
+			Description: project.Description,
 		}
-		if err := tx.Where(models.Project{Identifier: project}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil {
+		if err := tx.Where(models.Project{Identifier: project.Name}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil {
 			logger.Warningf(context.Background(), "failed to save project [%s]", project)
 			tx.Rollback()
 			return err