add back all files according to 9d7e55e

Signed-off-by: Yee Hing Tong <[email protected]>

.github/workflows/checks.yml

@@ -32,6 +32,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
           - flyteidl
@@ -69,6 +70,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     name: Docker Build Images
@@ -112,6 +114,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
+          - flyteartifacts
           - flytecopilot
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml

Dockerfile.flyteartifacts

@@ -0,0 +1,42 @@
+FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
+RUN apk add git openssh-client make curl
+
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+WORKDIR /go/src/github.com/flyteorg/flyte/flyteartifacts/
+
+COPY datacatalog ../datacatalog
+COPY flyteadmin ../flyteadmin
+COPY flyteartifacts .
+COPY flytecopilot ../flytecopilot
+COPY flyteidl ../flyteidl
+COPY flyteplugins ../flyteplugins
+COPY flytepropeller ../flytepropeller
+COPY flytestdlib ../flytestdlib
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/flyteadmin
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.16
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flyte/
+
+COPY --from=builder /artifacts /bin
+
+# Ensure the latest CA certs are present to authenticate SSL connections.
+RUN apk --update add ca-certificates
+
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
+CMD ["artifacts"]

Dockerfile.sandbox-lite

@@ -18,6 +18,7 @@ WORKDIR /app/flyte
 
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
+COPY flyteartifacts flyteartifacts
 COPY flytecopilot flytecopilot
 COPY flyteidl flyteidl
 COPY flyteplugins flyteplugins

charts/flyte-sandbox/README.md

@@ -11,6 +11,7 @@ A Helm chart for the Flyte local sandbox
 | file://../flyte-binary | flyte-binary | v0.1.10 |
 | https://charts.bitnami.com/bitnami | minio | 12.1.1 |
 | https://charts.bitnami.com/bitnami | postgresql | 12.1.9 |
+| https://charts.bitnami.com/bitnami | redis | 18.0.1 |
 | https://helm.twun.io/ | docker-registry | 2.2.2 |
 | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 6.0.0 |
 
@@ -100,6 +101,13 @@ A Helm chart for the Flyte local sandbox
 | postgresql.volumePermissions.enabled | bool | `true` |  |
 | postgresql.volumePermissions.image.pullPolicy | string | `"Never"` |  |
 | postgresql.volumePermissions.image.tag | string | `"sandbox"` |  |
+| redis.auth.enabled | bool | `false` |  |
+| redis.enabled | bool | `true` |  |
+| redis.image.pullPolicy | string | `"Never"` |  |
+| redis.image.tag | string | `"sandbox"` |  |
+| redis.master.service.nodePorts.redis | int | `30004` |  |
+| redis.master.service.type | string | `"NodePort"` |  |
+| redis.replica.replicaCount | int | `0` |  |
 | sandbox.buildkit.enabled | bool | `true` |  |
 | sandbox.buildkit.image.pullPolicy | string | `"Never"` |  |
 | sandbox.buildkit.image.repository | string | `"moby/buildkit"` |  |

charts/flyte-sandbox/templates/artifacts/deployment.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: artifact-service
+  name: artifact-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: artifact-service
+  template:
+    metadata:
+      labels:
+        app: artifact-service
+    spec:
+      containers:
+        - name: main
+          image: ghcr.io/unionai/artifacts:sandbox
+          env:
+            - name: DATABASE_URL
+              value: postgresql://postgres:[email protected]:5432/postgres
+            - name: REDIS_HOST
+              value: flyte-sandbox-redis-headless.flyte.svc.cluster.local
+            - name: REDIS_PORT
+              value: "6379"
+          ports:
+            - name: grpc
+              containerPort: 50051
+          livenessProbe:
+            initialDelaySeconds: 30
+            tcpSocket:
+              port: grpc
+          readinessProbe:
+            tcpSocket:
+              port: grpc

charts/flyte-sandbox/templates/artifacts/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: artifact-service
+  labels:
+    app: artifact-service
+spec:
+  ports:
+    - name: grpc
+      port: 50051
+      targetPort: 50051
+  selector:
+    app: artifact-service
+  type: ClusterIP

charts/flyte-sandbox/templates/proxy/configmap.yaml

@@ -108,6 +108,10 @@ data:
                                                   prefix: "/flyteidl.service.SignalService"
                                               route:
                                                   cluster: flyte_grpc
+                                            - match:
+                                                  prefix: "/flyteidl.artifact.ArtifactRegistry"
+                                              route:
+                                                  cluster: artifact
                                             {{- end }}
                                             {{- if index .Values "kubernetes-dashboard" "enabled" }}
                                             - match:
@@ -203,5 +207,19 @@ data:
                                           address: {{ .Release.Name }}-minio
                                           port_value: 9001
             {{- end }}
+            - name: artifact
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: artifact-service
+                                          port_value: 50051
 
 {{- end }}

charts/flyte-sandbox/values.yaml

@@ -47,6 +47,14 @@ flyte-binary:
           ephemeralStorage: 0
           gpu: 0
           memory: 0
+      cloudEvents:
+        enable: true
+        cloudEventVersion: v2
+        type: sandbox
+      artifacts:
+        host: localhost
+        port: 50051
+        insecure: true
       storage:
         signedURL:
           stowConfigOverride:

cmd/single/config.go

@@ -13,13 +13,18 @@ type Config struct {
 	Propeller   Propeller   `json:"propeller" pflag:",Configuration to disable propeller or any of its components."`
 	Admin       Admin       `json:"admin" pflag:",Configuration to disable FlyteAdmin or any of its components"`
 	DataCatalog DataCatalog `json:"dataCatalog" pflag:",Configuration to disable DataCatalog or any of its components"`
+	Artifact    Artifacts   `json:"artifact" pflag:",Configuration to disable Artifact or any of its components"`
 }
 
 type Propeller struct {
 	Disabled       bool `json:"disabled" pflag:",Disables flytepropeller in the single binary mode"`
 	DisableWebhook bool `json:"disableWebhook" pflag:",Disables webhook only"`
 }
 
+type Artifacts struct {
+	Disabled bool `json:"disabled" pflag:",Disables flyteartifacts in the single binary mode"`
+}
+
 type Admin struct {
 	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
 	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`

cmd/single/start.go

@@ -2,6 +2,10 @@ package single
 
 import (
 	"context"
+	sharedCmd "github.com/flyteorg/flyte/flyteartifacts/cmd/shared"
+	"github.com/flyteorg/flyte/flyteartifacts/pkg/configuration"
+	artifactsServer "github.com/flyteorg/flyte/flyteartifacts/pkg/server"
+	"github.com/flyteorg/flyte/flytestdlib/database"
 	"net/http"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	ctrlWebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -61,6 +65,40 @@ func startClusterResourceController(ctx context.Context) error {
 	return nil
 }
 
+func startArtifact(ctx context.Context, cfg Artifacts) error {
+	if cfg.Disabled {
+		logger.Infof(ctx, "Artifacts server is disabled. Skipping...")
+		return nil
+	}
+	// Roughly copies main/NewMigrateCmd
+	logger.Infof(ctx, "Artifacts: running database migrations if any...")
+	migs := artifactsServer.GetMigrations(ctx)
+	initializationSql := "create extension if not exists hstore;"
+	dbConfig := artifactsServer.GetDbConfig()
+	err := database.Migrate(context.Background(), dbConfig, migs, initializationSql)
+	if err != nil {
+		logger.Errorf(ctx, "Failed to run Artifacts database migrations. Error: %v", err)
+		return err
+	}
+
+	g, childCtx := errgroup.WithContext(ctx)
+
+	// Rough copy of NewServeCmd
+	g.Go(func() error {
+		cfg := configuration.GetApplicationConfig()
+		serverCfg := &cfg.ArtifactServerConfig
+		err := sharedCmd.ServeGateway(childCtx, "artifacts", serverCfg, artifactsServer.GrpcRegistrationHook,
+			artifactsServer.HttpRegistrationHook)
+		if err != nil {
+			logger.Errorf(childCtx, "Failed to start Artifacts server. Error: %v", err)
+			return err
+		}
+		return nil
+	})
+
+	return g.Wait()
+}
+
 func startAdmin(ctx context.Context, cfg Admin) error {
 	logger.Infof(ctx, "Running Database Migrations...")
 	if err := adminServer.Migrate(ctx); err != nil {
@@ -202,6 +240,16 @@ var startCmd = &cobra.Command{
 			})
 		}
 
+		if !cfg.Artifact.Disabled {
+			g.Go(func() error {
+				err := startArtifact(childCtx, cfg.Artifact)
+				if err != nil {
+					logger.Panicf(childCtx, "Failed to start Artifacts server, err: %v", err)
+				}
+				return nil
+			})
+		}
+
 		if !cfg.Propeller.Disabled {
 			g.Go(func() error {
 				err := startPropeller(childCtx, cfg.Propeller)

docker/sandbox-bundled/Makefile

@@ -1,7 +1,7 @@
 define FLYTE_BINARY_BUILD
 mkdir -p images/tar/$(1)
 
-docker buildx build \
+docker buildx build --ssh default \
 	--build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \
 	--platform linux/$(1) \
 	--tag flyte-binary:sandbox \
@@ -40,9 +40,20 @@ build: flyte manifests
 		--driver docker-container --driver-opt image=moby/buildkit:master \
 		--buildkitd-flags '--allow-insecure-entitlement security.insecure' \
 		--platform linux/arm64,linux/amd64
-	docker buildx build --builder flyte-sandbox --allow security.insecure --load \
+	docker buildx build --ssh default --builder flyte-sandbox --allow security.insecure --load \
 		--tag flyte-sandbox:latest .
 
+# This is here because we want to be able to push locally, not depend on GH actions
+.PHONY: build_push
+build_push: flyte manifests
+	[ -n "$(shell docker buildx ls | awk '/^flyte-sandbox / {print $$1}')" ] || \
+		docker buildx create --name flyte-sandbox \
+		--driver docker-container --driver-opt image=moby/buildkit:master \
+		--buildkitd-flags '--allow-insecure-entitlement security.insecure' \
+		--platform linux/arm64,linux/amd64
+	docker buildx build --ssh default --builder flyte-sandbox --allow security.insecure --push \
+		--tag ghcr.io/flyteorg/flyte-sandbox:a_v0.1.1 .
+
 # Port map
 # 6443 - k8s API server
 # 30000 - Docker Registry

flyte-single-binary-local.yaml

@@ -1,5 +1,7 @@
 # This is a sample configuration file for running single-binary Flyte locally against
 # a sandbox.
+# gatepr: revert the local dir to reflect home.
+#   paths were changed to personal to ensure settings didn't get lost.
 admin:
   # This endpoint is used by flytepropeller to talk to admin
   # and artifacts to talk to admin,
@@ -17,22 +19,22 @@ catalog-cache:
 
 cluster_resources:
   standaloneDeployment: false
-  templatePath: $HOME/.flyte/sandbox/cluster-resource-templates
+  templatePath: /Users/ytong/.flyte/sandbox/cluster-resource-templates
 
 logger:
   show-source: true
   level: 3
 
 propeller:
   create-flyteworkflow-crd: true
-  kube-config: $HOME/.flyte/sandbox/kubeconfig
+  kube-config: /Users/ytong/.flyte/sandbox/kubeconfig
   rawoutput-prefix: s3://my-s3-bucket/data
 
 server:
-  kube-config: $HOME/.flyte/sandbox/kubeconfig
+  kube-config: /Users/ytong/.flyte/sandbox/kubeconfig
 
 webhook:
-  certDir: $HOME/.flyte/webhook-certs
+  certDir: /Users/ytong/.flyte/webhook-certs
   localCert: true
   secretName: flyte-sandbox-webhook-secret
   serviceName: flyte-sandbox-local
@@ -75,6 +77,34 @@ database:
     port: 30001
     dbname: flyte
     options: "sslmode=disable"
+  # Point to cloned repo instead.
+  #
+#  postgres:
+#    username: postgres
+#    password: xxx
+#    host: 127.0.0.1
+#    port: 54328
+#    dbname: app
+#    options: "sslmode=disable"
+cloudEvents:
+  enable: true
+  cloudEventVersion: v2
+  type: sandbox
+# For artifact service itself
+artifactsServer:
+  artifactBlobStoreConfig:
+    type: stow
+    stow:
+      kind: s3
+      config:
+        disable_ssl: true
+        v2_signing: true
+        endpoint: http://localhost:30002
+        auth_type: accesskey
+        access_key_id: minio
+        secret_key: miniostorage
+artifactsProcessor:
+  cloudProvider: Sandbox
 storage:
   type: stow
   stow: