[wip] artf svc & debugging

.github/workflows/checks.yml

@@ -32,6 +32,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
           - flyteidl
@@ -69,6 +70,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     name: Docker Build Images
@@ -112,6 +114,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml

.github/workflows/flyteidl-buf-publish.yml

@@ -3,7 +3,7 @@ name: Publish flyteidl Buf Package
 on:
   push:
     branches:
-      - artifacts-shell
+      - artifacts-shell-2
       - artifacts
       - master
     paths:

Dockerfile

@@ -12,6 +12,7 @@ WORKDIR /flyteorg/build
 
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
+COPY flyteartifacts flyteartifacts
 COPY flytecopilot flytecopilot
 COPY flyteidl flyteidl
 COPY flyteplugins flyteplugins

Dockerfile.flyteartifacts

@@ -0,0 +1,42 @@
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
+RUN apk add git openssh-client make curl
+
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+WORKDIR /go/src/github.com/flyteorg/flyte/flyteartifacts/
+
+COPY datacatalog ../datacatalog
+COPY flyteadmin ../flyteadmin
+COPY flyteartifacts .
+COPY flytecopilot ../flytecopilot
+COPY flyteidl ../flyteidl
+COPY flyteplugins ../flyteplugins
+COPY flytepropeller ../flytepropeller
+COPY flytestdlib ../flytestdlib
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/flyteartifacts
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.16
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flyte/
+
+COPY --from=builder /artifacts /bin
+
+# Ensure the latest CA certs are present to authenticate SSL connections.
+RUN apk --update add ca-certificates
+
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
+CMD ["artifacts"]

Dockerfile.sandbox-lite

@@ -18,6 +18,7 @@ WORKDIR /app/flyte
 
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
+COPY flyteartifacts flyteartifacts
 COPY flytecopilot flytecopilot
 COPY flyteidl flyteidl
 COPY flyteplugins flyteplugins

charts/flyte-sandbox/README.md

@@ -27,6 +27,25 @@ A Helm chart for the Flyte local sandbox
 | flyte-binary.clusterResourceTemplates.inlineConfigMap | string | `"{{ include \"flyte-sandbox.clusterResourceTemplates.inlineConfigMap\" . }}"` |  |
 | flyte-binary.configuration.database.host | string | `"{{ printf \"%s-postgresql\" .Release.Name | trunc 63 | trimSuffix \"-\" }}"` |  |
 | flyte-binary.configuration.database.password | string | `"postgres"` |  |
+| flyte-binary.configuration.inline.artifactsProcessor.cloudProvider | string | `"Sandbox"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.access_key_id | string | `"minio"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.auth_type | string | `"accesskey"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.disable_ssl | bool | `true` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.endpoint | string | `"http://flyte-sandbox-minio.flyte:9000"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.secret_key | string | `"miniostorage"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.v2_signing | bool | `true` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.kind | string | `"s3"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.type | string | `"stow"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.dbname | string | `"artifacts"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.host | string | `"{{ printf \"%s-postgresql\" .Release.Name | trunc 63 | trimSuffix \"-\" }}"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.options | string | `"sslmode=disable"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.password | string | `"postgres"` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.port | int | `5432` |  |
+| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.username | string | `"postgres"` |  |
+| flyte-binary.configuration.inline.cloudEvents.cloudEventVersion | string | `"v2"` |  |
+| flyte-binary.configuration.inline.cloudEvents.enable | bool | `true` |  |
+| flyte-binary.configuration.inline.cloudEvents.type | string | `"sandbox"` |  |
+| flyte-binary.configuration.inline.flyteadmin.featureGates.enableArtifacts | bool | `true` |  |
 | flyte-binary.configuration.inline.plugins.k8s.default-env-vars[0].FLYTE_AWS_ENDPOINT | string | `"http://{{ printf \"%s-minio\" .Release.Name | trunc 63 | trimSuffix \"-\" }}.{{ .Release.Namespace }}:9000"` |  |
 | flyte-binary.configuration.inline.plugins.k8s.default-env-vars[1].FLYTE_AWS_ACCESS_KEY_ID | string | `"minio"` |  |
 | flyte-binary.configuration.inline.plugins.k8s.default-env-vars[2].FLYTE_AWS_SECRET_ACCESS_KEY | string | `"miniostorage"` |  |

charts/flyte-sandbox/templates/buildkit/deployment.yaml

@@ -13,6 +13,7 @@ spec:
     metadata:
       labels: {{- include "flyte-sandbox.buildkitSelectorLabels" . | nindent 8 }}
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       containers:
         - name: buildkit

charts/flyte-sandbox/templates/local/endpoint.yaml

@@ -19,4 +19,10 @@ subsets:
       - name: webhook
         port: 9443
         protocol: TCP
+      - name: artifact-http
+        port: 50050
+        protocol: TCP
+      - name: artifact-grpc
+        port: 50051
+        protocol: TCP
 {{- end }}

charts/flyte-sandbox/templates/local/service.yaml

@@ -18,4 +18,10 @@ spec:
     - name: webhook
       port: 9443
       protocol: TCP
+    - name: artifact-http
+      port: 50050
+      protocol: TCP
+    - name: artifact-grpc
+      port: 50051
+      protocol: TCP
 {{- end }}

charts/flyte-sandbox/templates/proxy/configmap.yaml

@@ -84,6 +84,10 @@ data:
                                                   prefix: "/v1"
                                               route:
                                                   cluster: flyte
+                                            - match:
+                                                  prefix: "/artifacts/api"
+                                              route:
+                                                  cluster: artifact_http
                                             - match:
                                                   prefix: "/flyteidl.service.AdminService"
                                               route:
@@ -108,6 +112,10 @@ data:
                                                   prefix: "/flyteidl.service.SignalService"
                                               route:
                                                   cluster: flyte_grpc
+                                            - match:
+                                                  prefix: "/flyteidl.artifact.ArtifactRegistry"
+                                              route:
+                                                  cluster: artifact_grpc
                                             {{- end }}
                                             {{- if index .Values "kubernetes-dashboard" "enabled" }}
                                             - match:
@@ -172,6 +180,34 @@ data:
                                           address: {{ include "flyte-sandbox.localHeadlessService" . }}
                                           {{- end }}
                                           port_value: 8089
+            - name: artifact_http
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact_http
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: flyte-sandbox-local
+                                          port_value: 50050
+            - name: artifact_grpc
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact_grpc
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: flyte-sandbox-local
+                                          port_value: 50051
             {{- end }}
             {{- if index .Values "kubernetes-dashboard" "enabled" }}
             - name: kubernetes-dashboard
@@ -203,5 +239,4 @@ data:
                                           address: {{ .Release.Name }}-minio
                                           port_value: 9001
             {{- end }}
-
 {{- end }}

charts/flyte-sandbox/values.yaml

@@ -47,6 +47,35 @@ flyte-binary:
           ephemeralStorage: 0
           gpu: 0
           memory: 0
+      cloudEvents:
+        enable: true
+        cloudEventVersion: v2
+        type: sandbox
+      flyteadmin:
+        featureGates:
+          enableArtifacts: true
+      artifactsServer:
+        artifactBlobStoreConfig:
+          type: stow
+          stow:
+            kind: s3
+            config:
+              disable_ssl: true
+              v2_signing: true
+              endpoint: http://flyte-sandbox-minio.flyte:9000
+              auth_type: accesskey
+              access_key_id: minio
+              secret_key: miniostorage
+        artifactDatabaseConfig:
+          postgres:
+            username: postgres
+            password: postgres
+            host: '{{ printf "%s-postgresql" .Release.Name | trunc 63 | trimSuffix "-" }}'
+            port: 5432
+            dbname: artifacts
+            options: "sslmode=disable"
+      artifactsProcessor:
+        cloudProvider: Sandbox
       storage:
         signedURL:
           stowConfigOverride:

cmd/single/config.go

@@ -13,13 +13,18 @@ type Config struct {
 	Propeller   Propeller   `json:"propeller" pflag:",Configuration to disable propeller or any of its components."`
 	Admin       Admin       `json:"admin" pflag:",Configuration to disable FlyteAdmin or any of its components"`
 	DataCatalog DataCatalog `json:"dataCatalog" pflag:",Configuration to disable DataCatalog or any of its components"`
+	Artifact    Artifacts   `json:"artifact" pflag:",Configuration to disable Artifact or any of its components"`
 }
 
 type Propeller struct {
 	Disabled       bool `json:"disabled" pflag:",Disables flytepropeller in the single binary mode"`
 	DisableWebhook bool `json:"disableWebhook" pflag:",Disables webhook only"`
 }
 
+type Artifacts struct {
+	Disabled bool `json:"disabled" pflag:",Disables flyteartifacts in the single binary mode"`
+}
+
 type Admin struct {
 	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
 	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`

cmd/single/start.go

@@ -2,6 +2,10 @@ package single
 
 import (
 	"context"
+	sharedCmd "github.com/flyteorg/flyte/flyteartifacts/cmd/shared"
+	"github.com/flyteorg/flyte/flyteartifacts/pkg/configuration"
+	artifactsServer "github.com/flyteorg/flyte/flyteartifacts/pkg/server"
+	"github.com/flyteorg/flyte/flytestdlib/database"
 	"net/http"
 	"os"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -63,6 +67,39 @@ func startClusterResourceController(ctx context.Context) error {
 	return nil
 }
 
+func startArtifact(ctx context.Context, cfg Artifacts) error {
+	if cfg.Disabled {
+		logger.Infof(ctx, "Artifacts server is disabled. Skipping...")
+		return nil
+	}
+	// Roughly copies main/NewMigrateCmd
+	logger.Infof(ctx, "Artifacts: running database migrations if any...")
+	migs := artifactsServer.GetMigrations(ctx)
+	dbConfig := artifactsServer.GetDbConfig()
+	err := database.Migrate(context.Background(), dbConfig, migs)
+	if err != nil {
+		logger.Errorf(ctx, "Failed to run Artifacts database migrations. Error: %v", err)
+		return err
+	}
+
+	g, childCtx := errgroup.WithContext(ctx)
+
+	// Rough copy of NewServeCmd
+	g.Go(func() error {
+		cfg := configuration.GetApplicationConfig()
+		serverCfg := &cfg.ArtifactServerConfig
+		err := sharedCmd.ServeGateway(childCtx, "artifacts", serverCfg, artifactsServer.GrpcRegistrationHook,
+			artifactsServer.HttpRegistrationHook)
+		if err != nil {
+			logger.Errorf(childCtx, "Failed to start Artifacts server. Error: %v", err)
+			return err
+		}
+		return nil
+	})
+
+	return g.Wait()
+}
+
 func startAdmin(ctx context.Context, cfg Admin) error {
 	logger.Infof(ctx, "Running Database Migrations...")
 	if err := adminServer.Migrate(ctx); err != nil {
@@ -232,6 +269,16 @@ var startCmd = &cobra.Command{
 			})
 		}
 
+		if !cfg.Artifact.Disabled {
+			g.Go(func() error {
+				err := startArtifact(childCtx, cfg.Artifact)
+				if err != nil {
+					logger.Panicf(childCtx, "Failed to start Artifacts server, err: %v", err)
+				}
+				return nil
+			})
+		}
+
 		if !cfg.Propeller.Disabled {
 			g.Go(func() error {
 				err := startPropeller(childCtx, cfg.Propeller)

datacatalog/go.mod

@@ -132,7 +132,7 @@ require (
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
-	sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 // indirect
+	sigs.k8s.io/controller-runtime v0.16.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

docker/sandbox-bundled/Makefile

@@ -1,7 +1,7 @@
 define FLYTE_BINARY_BUILD
 mkdir -p images/tar/$(1)
 
-docker buildx build \
+docker buildx build --ssh default \
 	--build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \
 	--platform linux/$(1) \
 	--tag flyte-binary:sandbox \
@@ -18,8 +18,9 @@ flyte:
 .PHONY: manifests
 manifests:
 	mkdir -p manifests
-	helm dependency update ../../charts/flyte-sandbox
 	helm dependency update ../../charts/flyteagent
+	helm dependency update ../../charts/flyte-binary
+	helm dependency update ../../charts/flyte-sandbox
 	kustomize build \
 		--enable-helm \
 		--load-restrictor=LoadRestrictionsNone \
@@ -40,9 +41,20 @@ build: flyte manifests
 		--driver docker-container --driver-opt image=moby/buildkit:master \
 		--buildkitd-flags '--allow-insecure-entitlement security.insecure' \
 		--platform linux/arm64,linux/amd64
-	docker buildx build --builder flyte-sandbox --allow security.insecure --load \
+	docker buildx build --ssh default --builder flyte-sandbox --allow security.insecure --load \
 		--tag flyte-sandbox:latest .
 
+# This is here because we want to be able to push locally, not depend on GH actions
+.PHONY: build_push
+build_push: flyte manifests
+	[ -n "$(shell docker buildx ls | awk '/^flyte-sandbox / {print $$1}')" ] || \
+		docker buildx create --name flyte-sandbox \
+		--driver docker-container --driver-opt image=moby/buildkit:master \
+		--buildkitd-flags '--allow-insecure-entitlement security.insecure' \
+		--platform linux/arm64,linux/amd64
+	docker buildx build --ssh default --builder flyte-sandbox --allow security.insecure --push \
+		--tag ghcr.io/flyteorg/flyte-sandbox:a_v0.1.1 .
+
 # Port map
 # 6443 - k8s API server
 # 30000 - Docker Registry