Artifacts bring admin monorepo (#4203)

Main pr to bring in the boilerplate code for artifact service into the main pr.

Signed-off-by: Yee Hing Tong <[email protected]>
Signed-off-by: Kevin Su <[email protected]>
Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/bump-tags.yml

@@ -42,6 +42,7 @@ jobs:
             const components = [
               "datacatalog",
               "flyteadmin",
+              "flyteartifacts",
               "flytecopilot",
               "flyteidl",
               "flyteplugins",

.github/workflows/checks.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - 'datacatalog/**'
       - 'flyteadmin/**'
+      - 'flyteartifacts/**'
       - 'flytecopilot/**'
       - 'flyteplugins/**'
       - 'flytepropeller/**'
@@ -15,6 +16,7 @@ on:
     paths:
       - 'datacatalog/**'
       - 'flyteadmin/**'
+      - 'flyteartifacts/**'
       - 'flytecopilot/**'
       - 'flyteidl/**'
       - 'flyteplugins/**'
@@ -80,6 +82,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     name: Docker Build Images
@@ -162,6 +165,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
           - flytescheduler

Dockerfile.flyteartifacts

@@ -0,0 +1,42 @@
+FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
+RUN apk add git openssh-client make curl
+
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+WORKDIR /go/src/github.com/flyteorg/flyte/flyteartifacts/
+
+COPY datacatalog ../datacatalog
+COPY flyteadmin ../flyteadmin
+COPY flyteartifacts .
+COPY flytecopilot ../flytecopilot
+COPY flyteidl ../flyteidl
+COPY flyteplugins ../flyteplugins
+COPY flytepropeller ../flytepropeller
+COPY flytestdlib ../flytestdlib
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/flyteadmin
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.16
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flyte/
+
+COPY --from=builder /artifacts /bin
+
+# Ensure the latest CA certs are present to authenticate SSL connections.
+RUN apk --update add ca-certificates
+
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
+CMD ["artifacts"]

charts/flyte-sandbox/Chart.lock

@@ -14,8 +14,5 @@ dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 12.1.9
-- name: redis
-  repository: https://charts.bitnami.com/bitnami
-  version: 17.15.6
-digest: sha256:1bcc5f102768a19c19d444b3460dabd0f90847b2d4423134f0ce9c7aa0a256ea
-generated: "2023-08-31T16:46:00.478623-07:00"
+digest: sha256:e7155e540bbdb98f690eb12e2bd301a19d8b36833336f6991410cb44d8d9bb5e
+generated: "2023-10-28T10:05:34.269916+08:00"

charts/flyte-sandbox/Chart.yaml

@@ -44,7 +44,3 @@ dependencies:
     version: 12.1.9
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
-  - name: redis
-    version: 17.15.6
-    repository: https://charts.bitnami.com/bitnami
-    condition: redis.enabled

charts/flyte-sandbox/values.yaml

@@ -49,12 +49,10 @@ flyte-binary:
           memory: 0
       cloudEvents:
         enable: true
-        transformToCloudEvents: true
-        type: redis
-        redis:
-          addr: "flyte-sandbox-redis-headless.flyte.svc.cluster.local:6379"
+        cloudEventVersion: v2
+        type: sandbox
       artifacts:
-        host: artifact-service.flyte.svc.cluster.local
+        host: localhost
         port: 50051
         insecure: true
       storage:
@@ -159,21 +157,6 @@ postgresql:
       tag: sandbox
       pullPolicy: Never
 
-redis:
-  enabled: true
-  image:
-    tag: sandbox
-    pullPolicy: Never
-  auth:
-    enabled: false
-  master:
-    service:
-      type: NodePort
-      nodePorts:
-        redis: 30004
-  replica:
-    replicaCount: 0
-
 sandbox:
   # dev Routes requests to an instance of Flyte running locally on a developer's
   # development environment. This is only usable if the flyte-binary chart is disabled.

cmd/single/config.go

@@ -13,13 +13,18 @@ type Config struct {
 	Propeller   Propeller   `json:"propeller" pflag:",Configuration to disable propeller or any of its components."`
 	Admin       Admin       `json:"admin" pflag:",Configuration to disable FlyteAdmin or any of its components"`
 	DataCatalog DataCatalog `json:"dataCatalog" pflag:",Configuration to disable DataCatalog or any of its components"`
+	Artifact    Artifacts   `json:"artifact" pflag:",Configuration to disable Artifact or any of its components"`
 }
 
 type Propeller struct {
 	Disabled       bool `json:"disabled" pflag:",Disables flytepropeller in the single binary mode"`
 	DisableWebhook bool `json:"disableWebhook" pflag:",Disables webhook only"`
 }
 
+type Artifacts struct {
+	Disabled bool `json:"disabled" pflag:",Disables flyteartifacts in the single binary mode"`
+}
+
 type Admin struct {
 	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
 	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`

cmd/single/start.go

@@ -2,6 +2,10 @@ package single
 
 import (
 	"context"
+	sharedCmd "github.com/flyteorg/flyte/flyteartifacts/cmd/shared"
+	"github.com/flyteorg/flyte/flyteartifacts/pkg/configuration"
+	artifactsServer "github.com/flyteorg/flyte/flyteartifacts/pkg/server"
+	"github.com/flyteorg/flyte/flytestdlib/database"
 	"net/http"
 
 	datacatalogConfig "github.com/flyteorg/flyte/datacatalog/pkg/config"
@@ -59,6 +63,40 @@ func startClusterResourceController(ctx context.Context) error {
 	return nil
 }
 
+func startArtifact(ctx context.Context, cfg Artifacts) error {
+	if cfg.Disabled {
+		logger.Infof(ctx, "Artifacts server is disabled. Skipping...")
+		return nil
+	}
+	// Roughly copies main/NewMigrateCmd
+	logger.Infof(ctx, "Artifacts: running database migrations if any...")
+	migs := artifactsServer.GetMigrations(ctx)
+	initializationSql := "create extension if not exists hstore;"
+	dbConfig := artifactsServer.GetDbConfig()
+	err := database.Migrate(context.Background(), dbConfig, migs, initializationSql)
+	if err != nil {
+		logger.Errorf(ctx, "Failed to run Artifacts database migrations. Error: %v", err)
+		return err
+	}
+
+	g, childCtx := errgroup.WithContext(ctx)
+
+	// Rough copy of NewServeCmd
+	g.Go(func() error {
+		cfg := configuration.GetApplicationConfig()
+		serverCfg := &cfg.ArtifactServerConfig
+		err := sharedCmd.ServeGateway(childCtx, "artifacts", serverCfg, artifactsServer.GrpcRegistrationHook,
+			artifactsServer.HttpRegistrationHook)
+		if err != nil {
+			logger.Errorf(childCtx, "Failed to start Artifacts server. Error: %v", err)
+			return err
+		}
+		return nil
+	})
+
+	return g.Wait()
+}
+
 func startAdmin(ctx context.Context, cfg Admin) error {
 	logger.Infof(ctx, "Running Database Migrations...")
 	if err := adminServer.Migrate(ctx); err != nil {
@@ -192,6 +230,16 @@ var startCmd = &cobra.Command{
 			})
 		}
 
+		if !cfg.Artifact.Disabled {
+			g.Go(func() error {
+				err := startArtifact(childCtx, cfg.Artifact)
+				if err != nil {
+					logger.Panicf(childCtx, "Failed to start Artifacts server, err: %v", err)
+				}
+				return nil
+			})
+		}
+
 		if !cfg.Propeller.Disabled {
 			g.Go(func() error {
 				err := startPropeller(childCtx, cfg.Propeller)

datacatalog/go.mod

@@ -9,15 +9,15 @@ require (
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/golang/glog v1.1.0
 	github.com/golang/protobuf v1.5.3
-	github.com/jackc/pgconn v1.10.1
+	github.com/jackc/pgconn v1.14.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
 	google.golang.org/grpc v1.56.1
-	gorm.io/driver/postgres v1.2.3
-	gorm.io/driver/sqlite v1.1.1
-	gorm.io/gorm v1.22.4
+	gorm.io/driver/postgres v1.5.3
+	gorm.io/driver/sqlite v1.5.4
+	gorm.io/gorm v1.25.4
 )
 
 require (
@@ -46,6 +46,7 @@ require (
 	github.com/flyteorg/stow v0.3.7 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -58,17 +59,16 @@ require (
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.9.0 // indirect
-	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.2 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.4.3 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-sqlite3 v1.14.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/ncw/swift v1.0.53 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
@@ -87,10 +87,10 @@ require (
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
-	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.1.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect