From dc28bc43dea407c633c498becf2b7fa53d0cf801 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 29 Dec 2023 18:25:03 +0800 Subject: [PATCH] add a log line, add artifact configuration to inline sandbox values Signed-off-by: Yee Hing Tong --- Dockerfile.flyteartifacts | 1 - charts/flyte-sandbox/values.yaml | 26 +++++++++++--- .../manifests/complete-agent.yaml | 34 ++++++++++++++----- .../sandbox-bundled/manifests/complete.yaml | 32 +++++++++++++---- docker/sandbox-bundled/manifests/dev.yaml | 4 +-- flyteartifacts/pkg/db/querying_test.go | 1 + flytestdlib/database/postgres.go | 1 + 7 files changed, 77 insertions(+), 22 deletions(-) diff --git a/Dockerfile.flyteartifacts b/Dockerfile.flyteartifacts index 7d87a257c7..a3f2cc7df4 100644 --- a/Dockerfile.flyteartifacts +++ b/Dockerfile.flyteartifacts @@ -11,7 +11,6 @@ RUN mkdir /artifacts WORKDIR /go/src/github.com/flyteorg/flyte/flyteartifacts/ -COPY boilerplate ../boilerplate COPY datacatalog ../datacatalog COPY flyteadmin ../flyteadmin COPY flyteartifacts . diff --git a/charts/flyte-sandbox/values.yaml b/charts/flyte-sandbox/values.yaml index 353d44012e..603ea00515 100644 --- a/charts/flyte-sandbox/values.yaml +++ b/charts/flyte-sandbox/values.yaml @@ -51,10 +51,28 @@ flyte-binary: enable: true cloudEventVersion: v2 type: sandbox - artifacts: - host: localhost - port: 50051 - insecure: true + artifactsServer: + artifactBlobStoreConfig: + type: stow + stow: + kind: s3 + config: + disable_ssl: true + v2_signing: true + endpoint: http://localhost:30002 + auth_type: accesskey + access_key_id: minio + secret_key: miniostorage + artifactDatabaseConfig: + postgres: + username: postgres + password: postgres + host: '{{ printf "%s-postgresql" .Release.Name | trunc 63 | trimSuffix "-" }}' + port: 5432 + dbname: artifacts + options: "sslmode=disable" + artifactsProcessor: + cloudProvider: Sandbox storage: signedURL: stowConfigOverride: diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index c7cd0022c6..427c654ec7 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -508,10 +508,28 @@ data: auth_type: accesskey container: my-s3-bucket 100-inline-config.yaml: | - artifacts: - host: localhost - insecure: true - port: 50051 + artifactsProcessor: + cloudProvider: Sandbox + artifactsServer: + artifactBlobStoreConfig: + stow: + config: + access_key_id: minio + auth_type: accesskey + disable_ssl: true + endpoint: http://localhost:30002 + secret_key: miniostorage + v2_signing: true + kind: s3 + type: stow + artifactDatabaseConfig: + postgres: + dbname: artifacts + host: 'flyte-sandbox-postgresql' + options: sslmode=disable + password: postgres + port: 5432 + username: postgres cloudEvents: cloudEventVersion: v2 enable: true @@ -860,7 +878,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: aWlPaFNYOWRSbGdjVlFLeg== + haSharedSecret: YnZWQm1tMm9ETTY3enZUcQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1290,7 +1308,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 0a0002f590cf88cb6728dae902bdacd7d0b5e0d8011bd70d0e4e232bdcbae9cf + checksum/configuration: c3f03f3b03d7909e7fac95f3490d43d9e50cf4759a67ac38d4c1992f08f64ce3 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1455,7 +1473,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 6385cddefb883d2c3cf21197a394b04d7b3e8c6a29d6e023662e21fdee556ba1 + checksum/secret: db210d75941d85245039373f167036040860dde6eb2db965c9a482e496483240 labels: app: docker-registry release: flyte-sandbox @@ -1797,7 +1815,7 @@ spec: value: minio - name: FLYTE_AWS_SECRET_ACCESS_KEY value: miniostorage - image: ghcr.io/flyteorg/flyteagent:1.10.1 + image: ghcr.io/flyteorg/flyteagent:1.10.2 imagePullPolicy: IfNotPresent name: flyteagent ports: diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 1f0c5aa957..0ed370422b 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -488,10 +488,28 @@ data: auth_type: accesskey container: my-s3-bucket 100-inline-config.yaml: | - artifacts: - host: localhost - insecure: true - port: 50051 + artifactsProcessor: + cloudProvider: Sandbox + artifactsServer: + artifactBlobStoreConfig: + stow: + config: + access_key_id: minio + auth_type: accesskey + disable_ssl: true + endpoint: http://localhost:30002 + secret_key: miniostorage + v2_signing: true + kind: s3 + type: stow + artifactDatabaseConfig: + postgres: + dbname: artifacts + host: 'flyte-sandbox-postgresql' + options: sslmode=disable + password: postgres + port: 5432 + username: postgres cloudEvents: cloudEventVersion: v2 enable: true @@ -840,7 +858,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: WkdDR3A3UlFIWENkMWpEcQ== + haSharedSecret: RTRFWEhLQ1c3V3h5N1JpQg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1238,7 +1256,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 93e689ddbfa941c82feb17f470000d27f19135f8cbef340de2640b49f85afc1d + checksum/configuration: 3e7625f0de3a210017c0f2ab5b8ad17a9ccac048a5a06f6466b8d762d8aa27d7 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1403,7 +1421,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: e70faf6ad14b50de7dd3002016d5d375414d99dc2e86f7112f0413f42b6b9613 + checksum/secret: 266445a61b2ef5d294e608c36659f8c3c6605c1b718d73870eb7e0584783504a labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 68ddc8e566..bd867b22d6 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -535,7 +535,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: c3ZITFRJWlF5YW5rWnBJbw== + haSharedSecret: YVdudG42ZklJTWZGTmpUZw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -981,7 +981,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: b064c11cb2d63899ba0ca190ec59e7fc6fad11f73f6b9b73e671cd3a0a59cdd6 + checksum/secret: 460f5a9ea128429763eb9a5cc9e25c1cd84e1a2a5f032853085c8061f0e20da4 labels: app: docker-registry release: flyte-sandbox diff --git a/flyteartifacts/pkg/db/querying_test.go b/flyteartifacts/pkg/db/querying_test.go index 17ffbae46f..9b0f9bce55 100644 --- a/flyteartifacts/pkg/db/querying_test.go +++ b/flyteartifacts/pkg/db/querying_test.go @@ -139,6 +139,7 @@ func TestQuery3_Find(t *testing.T) { res, ct, err := rds.SearchArtifacts(ctx, s) assert.NoError(t, err) assert.Equal(t, "", ct) + assert.Equal(t, nil, res) s = artifact.SearchArtifactsRequest{ ArtifactKey: &core.ArtifactKey{ diff --git a/flytestdlib/database/postgres.go b/flytestdlib/database/postgres.go index be5c118e58..ec43330af1 100644 --- a/flytestdlib/database/postgres.go +++ b/flytestdlib/database/postgres.go @@ -62,6 +62,7 @@ func CreatePostgresDbIfNotExists(ctx context.Context, gormConfig *gorm.Config, p return gormDb, nil } if !IsPgErrorWithCode(err, PqInvalidDBCode) { + logger.Errorf(ctx, "Unhandled error connecting to postgres, pg [%v], gorm [%v]: %v", pgConfig, gormConfig, err) return nil, err } logger.Warningf(ctx, "Database [%v] does not exist", pgConfig.DbName)