Update Buf to publish on artifacts branch push

.github/workflows/checks.yml

@@ -32,6 +32,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
           - flyteidl
@@ -69,6 +70,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     name: Docker Build Images
@@ -112,6 +114,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml

.github/workflows/flyteidl-buf-publish.yml

@@ -3,6 +3,7 @@ name: Publish flyteidl Buf Package
 on:
   push:
     branches:
+      - artifacts
       - master
     paths:
       - 'flyteidl/**'

Dockerfile.flyteartifacts

@@ -0,0 +1,42 @@
+FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
+RUN apk add git openssh-client make curl
+
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+WORKDIR /go/src/github.com/flyteorg/flyte/flyteartifacts/
+
+COPY datacatalog ../datacatalog
+COPY flyteadmin ../flyteadmin
+COPY flyteartifacts .
+COPY flytecopilot ../flytecopilot
+COPY flyteidl ../flyteidl
+COPY flyteplugins ../flyteplugins
+COPY flytepropeller ../flytepropeller
+COPY flytestdlib ../flytestdlib
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/flyteadmin
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.16
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flyte/
+
+COPY --from=builder /artifacts /bin
+
+# Ensure the latest CA certs are present to authenticate SSL connections.
+RUN apk --update add ca-certificates
+
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
+CMD ["artifacts"]

Dockerfile.sandbox-lite

@@ -18,6 +18,7 @@ WORKDIR /app/flyte
 
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
+COPY flyteartifacts flyteartifacts
 COPY flytecopilot flytecopilot
 COPY flyteidl flyteidl
 COPY flyteplugins flyteplugins

charts/flyte-sandbox/Chart.lock

@@ -15,4 +15,4 @@ dependencies:
   repository: https://charts.bitnami.com/bitnami
   version: 12.1.9
 digest: sha256:e7155e540bbdb98f690eb12e2bd301a19d8b36833336f6991410cb44d8d9bb5e
-generated: "2023-03-31T09:25:07.80904-07:00"
+generated: "2023-10-28T10:05:34.269916+08:00"

charts/flyte-sandbox/README.md

@@ -11,6 +11,7 @@ A Helm chart for the Flyte local sandbox
 | file://../flyte-binary | flyte-binary | v0.1.10 |
 | https://charts.bitnami.com/bitnami | minio | 12.1.1 |
 | https://charts.bitnami.com/bitnami | postgresql | 12.1.9 |
+| https://charts.bitnami.com/bitnami | redis | 18.0.1 |
 | https://helm.twun.io/ | docker-registry | 2.2.2 |
 | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 6.0.0 |
 
@@ -100,6 +101,13 @@ A Helm chart for the Flyte local sandbox
 | postgresql.volumePermissions.enabled | bool | `true` |  |
 | postgresql.volumePermissions.image.pullPolicy | string | `"Never"` |  |
 | postgresql.volumePermissions.image.tag | string | `"sandbox"` |  |
+| redis.auth.enabled | bool | `false` |  |
+| redis.enabled | bool | `true` |  |
+| redis.image.pullPolicy | string | `"Never"` |  |
+| redis.image.tag | string | `"sandbox"` |  |
+| redis.master.service.nodePorts.redis | int | `30004` |  |
+| redis.master.service.type | string | `"NodePort"` |  |
+| redis.replica.replicaCount | int | `0` |  |
 | sandbox.buildkit.enabled | bool | `true` |  |
 | sandbox.buildkit.image.pullPolicy | string | `"Never"` |  |
 | sandbox.buildkit.image.repository | string | `"moby/buildkit"` |  |

charts/flyte-sandbox/templates/artifacts/deployment.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: artifact-service
+  name: artifact-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: artifact-service
+  template:
+    metadata:
+      labels:
+        app: artifact-service
+    spec:
+      containers:
+        - name: main
+          image: ghcr.io/unionai/artifacts:sandbox
+          env:
+            - name: DATABASE_URL
+              value: postgresql://postgres:[email protected]:5432/postgres
+            - name: REDIS_HOST
+              value: flyte-sandbox-redis-headless.flyte.svc.cluster.local
+            - name: REDIS_PORT
+              value: "6379"
+          ports:
+            - name: grpc
+              containerPort: 50051
+          livenessProbe:
+            initialDelaySeconds: 30
+            tcpSocket:
+              port: grpc
+          readinessProbe:
+            tcpSocket:
+              port: grpc

charts/flyte-sandbox/templates/artifacts/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: artifact-service
+  labels:
+    app: artifact-service
+spec:
+  ports:
+    - name: grpc
+      port: 50051
+      targetPort: 50051
+  selector:
+    app: artifact-service
+  type: ClusterIP

charts/flyte-sandbox/templates/proxy/configmap.yaml

@@ -108,6 +108,10 @@ data:
                                                   prefix: "/flyteidl.service.SignalService"
                                               route:
                                                   cluster: flyte_grpc
+                                            - match:
+                                                  prefix: "/flyteidl.artifact.ArtifactRegistry"
+                                              route:
+                                                  cluster: artifact
                                             {{- end }}
                                             {{- if index .Values "kubernetes-dashboard" "enabled" }}
                                             - match:
@@ -203,5 +207,19 @@ data:
                                           address: {{ .Release.Name }}-minio
                                           port_value: 9001
             {{- end }}
+            - name: artifact
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: artifact-service
+                                          port_value: 50051
 
 {{- end }}

charts/flyte-sandbox/values.yaml

@@ -29,7 +29,7 @@ flyte-binary:
           accessKey: minio
           secretKey: miniostorage
     logging:
-      level: 6
+      level: 5
       plugins:
         kubernetes:
           enabled: true
@@ -47,6 +47,14 @@ flyte-binary:
           ephemeralStorage: 0
           gpu: 0
           memory: 0
+      cloudEvents:
+        enable: true
+        cloudEventVersion: v2
+        type: sandbox
+      artifacts:
+        host: localhost
+        port: 50051
+        insecure: true
       storage:
         signedURL:
           stowConfigOverride:

cmd/single/config.go

@@ -13,13 +13,18 @@ type Config struct {
 	Propeller   Propeller   `json:"propeller" pflag:",Configuration to disable propeller or any of its components."`
 	Admin       Admin       `json:"admin" pflag:",Configuration to disable FlyteAdmin or any of its components"`
 	DataCatalog DataCatalog `json:"dataCatalog" pflag:",Configuration to disable DataCatalog or any of its components"`
+	Artifact    Artifacts   `json:"artifact" pflag:",Configuration to disable Artifact or any of its components"`
 }
 
 type Propeller struct {
 	Disabled       bool `json:"disabled" pflag:",Disables flytepropeller in the single binary mode"`
 	DisableWebhook bool `json:"disableWebhook" pflag:",Disables webhook only"`
 }
 
+type Artifacts struct {
+	Disabled bool `json:"disabled" pflag:",Disables flyteartifacts in the single binary mode"`
+}
+
 type Admin struct {
 	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
 	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`

cmd/single/start.go

@@ -2,6 +2,10 @@ package single
 
 import (
 	"context"
+	sharedCmd "github.com/flyteorg/flyte/flyteartifacts/cmd/shared"
+	"github.com/flyteorg/flyte/flyteartifacts/pkg/configuration"
+	artifactsServer "github.com/flyteorg/flyte/flyteartifacts/pkg/server"
+	"github.com/flyteorg/flyte/flytestdlib/database"
 	"net/http"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	ctrlWebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -61,6 +65,40 @@ func startClusterResourceController(ctx context.Context) error {
 	return nil
 }
 
+func startArtifact(ctx context.Context, cfg Artifacts) error {
+	if cfg.Disabled {
+		logger.Infof(ctx, "Artifacts server is disabled. Skipping...")
+		return nil
+	}
+	// Roughly copies main/NewMigrateCmd
+	logger.Infof(ctx, "Artifacts: running database migrations if any...")
+	migs := artifactsServer.GetMigrations(ctx)
+	initializationSql := "create extension if not exists hstore;"
+	dbConfig := artifactsServer.GetDbConfig()
+	err := database.Migrate(context.Background(), dbConfig, migs, initializationSql)
+	if err != nil {
+		logger.Errorf(ctx, "Failed to run Artifacts database migrations. Error: %v", err)
+		return err
+	}
+
+	g, childCtx := errgroup.WithContext(ctx)
+
+	// Rough copy of NewServeCmd
+	g.Go(func() error {
+		cfg := configuration.GetApplicationConfig()
+		serverCfg := &cfg.ArtifactServerConfig
+		err := sharedCmd.ServeGateway(childCtx, "artifacts", serverCfg, artifactsServer.GrpcRegistrationHook,
+			artifactsServer.HttpRegistrationHook)
+		if err != nil {
+			logger.Errorf(childCtx, "Failed to start Artifacts server. Error: %v", err)
+			return err
+		}
+		return nil
+	})
+
+	return g.Wait()
+}
+
 func startAdmin(ctx context.Context, cfg Admin) error {
 	logger.Infof(ctx, "Running Database Migrations...")
 	if err := adminServer.Migrate(ctx); err != nil {
@@ -202,6 +240,16 @@ var startCmd = &cobra.Command{
 			})
 		}
 
+		if !cfg.Artifact.Disabled {
+			g.Go(func() error {
+				err := startArtifact(childCtx, cfg.Artifact)
+				if err != nil {
+					logger.Panicf(childCtx, "Failed to start Artifacts server, err: %v", err)
+				}
+				return nil
+			})
+		}
+
 		if !cfg.Propeller.Disabled {
 			g.Go(func() error {
 				err := startPropeller(childCtx, cfg.Propeller)

datacatalog/go.mod

@@ -9,15 +9,15 @@ require (
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/golang/glog v1.1.0
 	github.com/golang/protobuf v1.5.3
-	github.com/jackc/pgconn v1.10.1
+	github.com/jackc/pgconn v1.14.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
 	google.golang.org/grpc v1.56.1
-	gorm.io/driver/postgres v1.2.3
-	gorm.io/driver/sqlite v1.1.1
-	gorm.io/gorm v1.22.4
+	gorm.io/driver/postgres v1.5.3
+	gorm.io/driver/sqlite v1.5.4
+	gorm.io/gorm v1.25.4
 )
 
 require (
@@ -46,6 +46,7 @@ require (
 	github.com/flyteorg/stow v0.3.7 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -58,17 +59,16 @@ require (
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.9.0 // indirect
-	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.2 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.4.3 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-sqlite3 v1.14.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/ncw/swift v1.0.53 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect