From 7211dbb02f803483a5e8236c12f9bac9b5d4906f Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 29 Dec 2023 19:36:53 +0800 Subject: [PATCH] artf/make goimports (#4651) make goimports Update minio address so artifacts can find s3 in sandbox. make helm Signed-off-by: Yee Hing Tong --- charts/flyte-sandbox/README.md | 18 +++++++++++++++--- charts/flyte-sandbox/values.yaml | 2 +- .../manifests/complete-agent.yaml | 8 ++++---- docker/sandbox-bundled/manifests/complete.yaml | 8 ++++---- docker/sandbox-bundled/manifests/dev.yaml | 4 ++-- flyteartifacts/Makefile | 2 +- flyteartifacts/cmd/main.go | 4 ++-- flyteartifacts/cmd/shared/migrate.go | 4 +++- flyteartifacts/cmd/shared/root.go | 8 +++++--- flyteartifacts/cmd/shared/serve.go | 9 +++++---- flyteartifacts/cmd/shared/types.go | 4 +++- flyteartifacts/pkg/blob/artifact_blob_store.go | 4 +++- flyteartifacts/pkg/configuration/config.go | 4 ++-- .../pkg/configuration/shared/config.go | 1 + flyteartifacts/pkg/db/gorm_transformers.go | 8 +++++--- flyteartifacts/pkg/db/querying_test.go | 11 ++++++----- flyteartifacts/pkg/db/storage.go | 10 ++++++---- flyteartifacts/pkg/db/storage_test.go | 10 +++++----- flyteartifacts/pkg/lib/string_converter.go | 3 ++- .../pkg/lib/string_converter_test.go | 8 +++++--- flyteartifacts/pkg/lib/url_parse.go | 5 +++-- flyteartifacts/pkg/lib/url_parse_test.go | 6 ++++-- flyteartifacts/pkg/models/transformers.go | 6 ++++-- flyteartifacts/pkg/server/interfaces.go | 4 +++- flyteartifacts/pkg/server/metrics.go | 6 ++++-- .../pkg/server/processor/channel_processor.go | 8 +++++--- .../pkg/server/processor/events_handler.go | 5 +++-- .../server/processor/events_handler_test.go | 6 ++++-- .../pkg/server/processor/interfaces.go | 1 + .../pkg/server/processor/processor.go | 2 ++ .../pkg/server/processor/sqs_processor.go | 2 +- .../pkg/server/processor/sqs_processor_test.go | 6 ++++-- flyteartifacts/pkg/server/server.go | 12 +++++++----- flyteartifacts/pkg/server/service.go | 1 + flyteartifacts/pkg/server/trigger_engine.go | 18 ++++++++++-------- .../pkg/server/trigger_engine_test.go | 3 ++- 36 files changed, 138 insertions(+), 83 deletions(-) diff --git a/charts/flyte-sandbox/README.md b/charts/flyte-sandbox/README.md index 2506d78a0c..e74f0724e5 100644 --- a/charts/flyte-sandbox/README.md +++ b/charts/flyte-sandbox/README.md @@ -27,9 +27,21 @@ A Helm chart for the Flyte local sandbox | flyte-binary.clusterResourceTemplates.inlineConfigMap | string | `"{{ include \"flyte-sandbox.clusterResourceTemplates.inlineConfigMap\" . }}"` | | | flyte-binary.configuration.database.host | string | `"{{ printf \"%s-postgresql\" .Release.Name | trunc 63 | trimSuffix \"-\" }}"` | | | flyte-binary.configuration.database.password | string | `"postgres"` | | -| flyte-binary.configuration.inline.artifacts.host | string | `"localhost"` | | -| flyte-binary.configuration.inline.artifacts.insecure | bool | `true` | | -| flyte-binary.configuration.inline.artifacts.port | int | `50051` | | +| flyte-binary.configuration.inline.artifactsProcessor.cloudProvider | string | `"Sandbox"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.access_key_id | string | `"minio"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.auth_type | string | `"accesskey"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.disable_ssl | bool | `true` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.endpoint | string | `"http://flyte-sandbox-minio.flyte:9000"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.secret_key | string | `"miniostorage"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.config.v2_signing | bool | `true` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.stow.kind | string | `"s3"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactBlobStoreConfig.type | string | `"stow"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.dbname | string | `"artifacts"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.host | string | `"{{ printf \"%s-postgresql\" .Release.Name | trunc 63 | trimSuffix \"-\" }}"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.options | string | `"sslmode=disable"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.password | string | `"postgres"` | | +| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.port | int | `5432` | | +| flyte-binary.configuration.inline.artifactsServer.artifactDatabaseConfig.postgres.username | string | `"postgres"` | | | flyte-binary.configuration.inline.cloudEvents.cloudEventVersion | string | `"v2"` | | | flyte-binary.configuration.inline.cloudEvents.enable | bool | `true` | | | flyte-binary.configuration.inline.cloudEvents.type | string | `"sandbox"` | | diff --git a/charts/flyte-sandbox/values.yaml b/charts/flyte-sandbox/values.yaml index 603ea00515..b799b9e767 100644 --- a/charts/flyte-sandbox/values.yaml +++ b/charts/flyte-sandbox/values.yaml @@ -59,7 +59,7 @@ flyte-binary: config: disable_ssl: true v2_signing: true - endpoint: http://localhost:30002 + endpoint: http://flyte-sandbox-minio.flyte:9000 auth_type: accesskey access_key_id: minio secret_key: miniostorage diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 427c654ec7..0c753699b5 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -517,7 +517,7 @@ data: access_key_id: minio auth_type: accesskey disable_ssl: true - endpoint: http://localhost:30002 + endpoint: http://flyte-sandbox-minio.flyte:9000 secret_key: miniostorage v2_signing: true kind: s3 @@ -878,7 +878,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: YnZWQm1tMm9ETTY3enZUcQ== + haSharedSecret: eTJZeFhSZWF0TEVQVWlybw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1308,7 +1308,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: c3f03f3b03d7909e7fac95f3490d43d9e50cf4759a67ac38d4c1992f08f64ce3 + checksum/configuration: 45373e78f147407fbee7ae52351e92542475be8fefff767255d1b3bdc48c3a0a checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1473,7 +1473,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: db210d75941d85245039373f167036040860dde6eb2db965c9a482e496483240 + checksum/secret: 101b0e951be7ace5815290755725d39275a0f550892b354b9cfa129e0a750f0f labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 0ed370422b..168627b78d 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -497,7 +497,7 @@ data: access_key_id: minio auth_type: accesskey disable_ssl: true - endpoint: http://localhost:30002 + endpoint: http://flyte-sandbox-minio.flyte:9000 secret_key: miniostorage v2_signing: true kind: s3 @@ -858,7 +858,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: RTRFWEhLQ1c3V3h5N1JpQg== + haSharedSecret: bkVuc0pmTEgwZEp0QUVSbQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1256,7 +1256,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 3e7625f0de3a210017c0f2ab5b8ad17a9ccac048a5a06f6466b8d762d8aa27d7 + checksum/configuration: 9502021d87030cfe424e840eee4c26900579d180008686827db9d9ef3f723a5b checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1421,7 +1421,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 266445a61b2ef5d294e608c36659f8c3c6605c1b718d73870eb7e0584783504a + checksum/secret: 63b35e1a77beeb576299ff75680ecfdebf9034eab5c85af9b0c24646b1e3007c labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index bd867b22d6..0ae038ac5c 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -535,7 +535,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: YVdudG42ZklJTWZGTmpUZw== + haSharedSecret: NDV4Z0JTUVAwRnlCYUFPWQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -981,7 +981,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 460f5a9ea128429763eb9a5cc9e25c1cd84e1a2a5f032853085c8061f0e20da4 + checksum/secret: 17dd6127a719e8f5716b5cbddc9f2c3ccb88560fca3dc5cad286376afc6271c0 labels: app: docker-registry release: flyte-sandbox diff --git a/flyteartifacts/Makefile b/flyteartifacts/Makefile index 62684d152b..db6ed45f1d 100644 --- a/flyteartifacts/Makefile +++ b/flyteartifacts/Makefile @@ -1,4 +1,4 @@ -include ../boilerplate/flyte/golang_test_targets/Makefile +include boilerplate/flyte/golang_test_targets/Makefile .PHONY: linux_compile linux_compile: export CGO_ENABLED ?= 0 diff --git a/flyteartifacts/cmd/main.go b/flyteartifacts/cmd/main.go index 2d4f40c7f8..e34b3b1dce 100644 --- a/flyteartifacts/cmd/main.go +++ b/flyteartifacts/cmd/main.go @@ -2,14 +2,14 @@ package main import ( "context" + _ "net/http/pprof" // Required to serve application. + sharedCmd "github.com/flyteorg/flyte/flyteartifacts/cmd/shared" "github.com/flyteorg/flyte/flyteartifacts/pkg/server" "github.com/flyteorg/flyte/flytestdlib/contextutils" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils/labeled" "github.com/flyteorg/flyte/flytestdlib/storage" - - _ "net/http/pprof" // Required to serve application. ) func main() { diff --git a/flyteartifacts/cmd/shared/migrate.go b/flyteartifacts/cmd/shared/migrate.go index d7fcb9dbcd..e5a102846e 100644 --- a/flyteartifacts/cmd/shared/migrate.go +++ b/flyteartifacts/cmd/shared/migrate.go @@ -2,9 +2,11 @@ package shared import ( "context" - "github.com/flyteorg/flyte/flytestdlib/database" + "github.com/go-gormigrate/gormigrate/v2" "github.com/spf13/cobra" + + "github.com/flyteorg/flyte/flytestdlib/database" ) // NewMigrateCmd represents the migrate command diff --git a/flyteartifacts/cmd/shared/root.go b/flyteartifacts/cmd/shared/root.go index 8bc0390c30..5ffb6eac53 100644 --- a/flyteartifacts/cmd/shared/root.go +++ b/flyteartifacts/cmd/shared/root.go @@ -4,14 +4,16 @@ import ( "context" "flag" "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration" "github.com/flyteorg/flyte/flytestdlib/config" "github.com/flyteorg/flyte/flytestdlib/config/viper" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/profutils" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "os" ) var ( diff --git a/flyteartifacts/cmd/shared/serve.go b/flyteartifacts/cmd/shared/serve.go index 64cb3bfd22..f11b36c706 100644 --- a/flyteartifacts/cmd/shared/serve.go +++ b/flyteartifacts/cmd/shared/serve.go @@ -2,13 +2,9 @@ package shared import ( "context" - "google.golang.org/grpc/reflection" "net" "net/http" - sharedCfg "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration/shared" - "github.com/flyteorg/flyte/flytestdlib/logger" - "github.com/flyteorg/flyte/flytestdlib/promutils" grpcMiddleware "github.com/grpc-ecosystem/go-grpc-middleware" grpcPrometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/grpc-ecosystem/grpc-gateway/runtime" @@ -18,6 +14,11 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" + + sharedCfg "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration/shared" + "github.com/flyteorg/flyte/flytestdlib/logger" + "github.com/flyteorg/flyte/flytestdlib/promutils" ) func NewServeCmd(commandName string, serverCfg sharedCfg.ServerConfiguration, grpcHook GrpcRegistrationHook, httpHook HttpRegistrationHook) *cobra.Command { diff --git a/flyteartifacts/cmd/shared/types.go b/flyteartifacts/cmd/shared/types.go index be1878a6d3..c6f580fcf3 100644 --- a/flyteartifacts/cmd/shared/types.go +++ b/flyteartifacts/cmd/shared/types.go @@ -2,9 +2,11 @@ package shared import ( "context" - "github.com/flyteorg/flyte/flytestdlib/promutils" + "github.com/grpc-ecosystem/grpc-gateway/runtime" "google.golang.org/grpc" + + "github.com/flyteorg/flyte/flytestdlib/promutils" ) type GrpcRegistrationHook func(ctx context.Context, server *grpc.Server, scope promutils.Scope) error diff --git a/flyteartifacts/pkg/blob/artifact_blob_store.go b/flyteartifacts/pkg/blob/artifact_blob_store.go index 36a9c30518..bb8d2f8687 100644 --- a/flyteartifacts/pkg/blob/artifact_blob_store.go +++ b/flyteartifacts/pkg/blob/artifact_blob_store.go @@ -3,11 +3,13 @@ package blob import ( "context" "fmt" + + "github.com/golang/protobuf/ptypes/any" + "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" "github.com/flyteorg/flyte/flytestdlib/storage" - "github.com/golang/protobuf/ptypes/any" ) type ArtifactBlobStore struct { diff --git a/flyteartifacts/pkg/configuration/config.go b/flyteartifacts/pkg/configuration/config.go index 7848789db1..861c01e403 100644 --- a/flyteartifacts/pkg/configuration/config.go +++ b/flyteartifacts/pkg/configuration/config.go @@ -1,12 +1,12 @@ package configuration import ( + "time" + "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration/shared" "github.com/flyteorg/flyte/flytestdlib/config" stdLibDb "github.com/flyteorg/flyte/flytestdlib/database" stdLibStorage "github.com/flyteorg/flyte/flytestdlib/storage" - - "time" ) const artifactsServer = "artifactsServer" diff --git a/flyteartifacts/pkg/configuration/shared/config.go b/flyteartifacts/pkg/configuration/shared/config.go index d4605ea557..f3c23a7b02 100644 --- a/flyteartifacts/pkg/configuration/shared/config.go +++ b/flyteartifacts/pkg/configuration/shared/config.go @@ -2,6 +2,7 @@ package shared import ( "fmt" + "github.com/flyteorg/flyte/flytestdlib/config" ) diff --git a/flyteartifacts/pkg/db/gorm_transformers.go b/flyteartifacts/pkg/db/gorm_transformers.go index bc77eebbb0..659984e52f 100644 --- a/flyteartifacts/pkg/db/gorm_transformers.go +++ b/flyteartifacts/pkg/db/gorm_transformers.go @@ -3,14 +3,16 @@ package db import ( "context" "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "github.com/jackc/pgx/v5/pgtype" + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/logger" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - "github.com/jackc/pgx/v5/pgtype" ) func PartitionsIdlToHstore(idlPartitions *core.Partitions) pgtype.Hstore { diff --git a/flyteartifacts/pkg/db/querying_test.go b/flyteartifacts/pkg/db/querying_test.go index 9b0f9bce55..5625d2e9ea 100644 --- a/flyteartifacts/pkg/db/querying_test.go +++ b/flyteartifacts/pkg/db/querying_test.go @@ -4,17 +4,18 @@ import ( "context" "database/sql" "fmt" + "testing" + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/database" "github.com/flyteorg/flyte/flytestdlib/promutils" - "github.com/stretchr/testify/assert" - "testing" - - "gorm.io/driver/postgres" - "gorm.io/gorm" ) func getMockRds(t *testing.T) (*sql.DB, sqlmock.Sqlmock, RDSStorage) { diff --git a/flyteartifacts/pkg/db/storage.go b/flyteartifacts/pkg/db/storage.go index 184c319a11..a7387312fe 100644 --- a/flyteartifacts/pkg/db/storage.go +++ b/flyteartifacts/pkg/db/storage.go @@ -4,6 +4,12 @@ import ( "context" "errors" "fmt" + "strconv" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "gorm.io/gorm" + "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration" "github.com/flyteorg/flyte/flyteartifacts/pkg/lib" "github.com/flyteorg/flyte/flyteartifacts/pkg/models" @@ -12,10 +18,6 @@ import ( "github.com/flyteorg/flyte/flytestdlib/database" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "gorm.io/gorm" - "strconv" ) // RDSStorage should implement StorageInterface diff --git a/flyteartifacts/pkg/db/storage_test.go b/flyteartifacts/pkg/db/storage_test.go index ef99d160b1..bf74a15c3d 100644 --- a/flyteartifacts/pkg/db/storage_test.go +++ b/flyteartifacts/pkg/db/storage_test.go @@ -7,17 +7,17 @@ package db import ( "context" "fmt" - "github.com/flyteorg/flyte/flyteartifacts/pkg/models" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "os" "testing" - "github.com/flyteorg/flyte/flytestdlib/config" - "github.com/flyteorg/flyte/flytestdlib/config/viper" - "github.com/flyteorg/flyte/flytestdlib/promutils" "github.com/stretchr/testify/assert" + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flyte/flytestdlib/config" + "github.com/flyteorg/flyte/flytestdlib/config/viper" + "github.com/flyteorg/flyte/flytestdlib/promutils" ) func TestBasicWrite(t *testing.T) { diff --git a/flyteartifacts/pkg/lib/string_converter.go b/flyteartifacts/pkg/lib/string_converter.go index b85e56ee2a..a4ffba2170 100644 --- a/flyteartifacts/pkg/lib/string_converter.go +++ b/flyteartifacts/pkg/lib/string_converter.go @@ -2,9 +2,10 @@ package lib import ( "fmt" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "strings" "time" + + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) const DateFormat string = "2006-01-02" diff --git a/flyteartifacts/pkg/lib/string_converter_test.go b/flyteartifacts/pkg/lib/string_converter_test.go index ec50b1f46d..15a7463ab4 100644 --- a/flyteartifacts/pkg/lib/string_converter_test.go +++ b/flyteartifacts/pkg/lib/string_converter_test.go @@ -1,11 +1,13 @@ package lib import ( - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" - "github.com/golang/protobuf/ptypes/timestamp" - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/golang/protobuf/ptypes/timestamp" + "github.com/stretchr/testify/assert" + + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) func TestRenderDate(t *testing.T) { diff --git a/flyteartifacts/pkg/lib/url_parse.go b/flyteartifacts/pkg/lib/url_parse.go index 41bb5df16a..489e416d64 100644 --- a/flyteartifacts/pkg/lib/url_parse.go +++ b/flyteartifacts/pkg/lib/url_parse.go @@ -2,11 +2,12 @@ package lib import ( "errors" - "github.com/flyteorg/flyte/flyteartifacts/pkg/models" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "net/url" "regexp" "strings" + + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) var flyteURLNameRe = regexp.MustCompile(`(?P[\w/-]+)(?:(:(?P\w+))?)(?:(@(?P\w+))?)`) diff --git a/flyteartifacts/pkg/lib/url_parse_test.go b/flyteartifacts/pkg/lib/url_parse_test.go index 6d8b2c8d13..2340d07252 100644 --- a/flyteartifacts/pkg/lib/url_parse_test.go +++ b/flyteartifacts/pkg/lib/url_parse_test.go @@ -2,9 +2,11 @@ package lib import ( "context" - "github.com/flyteorg/flyte/flyteartifacts/pkg/models" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" ) func TestURLParseWithTag(t *testing.T) { diff --git a/flyteartifacts/pkg/models/transformers.go b/flyteartifacts/pkg/models/transformers.go index f10d62694a..833dbea447 100644 --- a/flyteartifacts/pkg/models/transformers.go +++ b/flyteartifacts/pkg/models/transformers.go @@ -3,11 +3,13 @@ package models import ( "context" "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/logger" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" ) func CreateArtifactModelFromRequest(ctx context.Context, key *core.ArtifactKey, spec *artifact.ArtifactSpec, version string, partitions map[string]string, tag string, source *artifact.ArtifactSource) (Artifact, error) { diff --git a/flyteartifacts/pkg/server/interfaces.go b/flyteartifacts/pkg/server/interfaces.go index 275b2d8343..cdb3565d55 100644 --- a/flyteartifacts/pkg/server/interfaces.go +++ b/flyteartifacts/pkg/server/interfaces.go @@ -2,11 +2,13 @@ package server import ( "context" + + "github.com/golang/protobuf/ptypes/any" + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" stdLibStorage "github.com/flyteorg/flyte/flytestdlib/storage" - "github.com/golang/protobuf/ptypes/any" ) type StorageInterface interface { diff --git a/flyteartifacts/pkg/server/metrics.go b/flyteartifacts/pkg/server/metrics.go index 3ae7d5f8ba..cd36ed29c9 100644 --- a/flyteartifacts/pkg/server/metrics.go +++ b/flyteartifacts/pkg/server/metrics.go @@ -1,9 +1,11 @@ package server import ( - "github.com/flyteorg/flyte/flytestdlib/promutils" - "github.com/prometheus/client_golang/prometheus" "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/flyteorg/flyte/flytestdlib/promutils" ) type RequestMetrics struct { diff --git a/flyteartifacts/pkg/server/processor/channel_processor.go b/flyteartifacts/pkg/server/processor/channel_processor.go index 905d3d317c..04dd7d971c 100644 --- a/flyteartifacts/pkg/server/processor/channel_processor.go +++ b/flyteartifacts/pkg/server/processor/channel_processor.go @@ -3,14 +3,16 @@ package processor import ( "bytes" "context" + "time" + pbcloudevents "github.com/cloudevents/sdk-go/binding/format/protobuf/v2" "github.com/cloudevents/sdk-go/v2/event" + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" + flyteEvents "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/sandboxutils" - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/proto" - "time" ) type SandboxCloudEventsReceiver struct { diff --git a/flyteartifacts/pkg/server/processor/events_handler.go b/flyteartifacts/pkg/server/processor/events_handler.go index faf70dfd06..71f7813268 100644 --- a/flyteartifacts/pkg/server/processor/events_handler.go +++ b/flyteartifacts/pkg/server/processor/events_handler.go @@ -3,16 +3,17 @@ package processor import ( "context" "fmt" + event2 "github.com/cloudevents/sdk-go/v2/event" + "github.com/golang/protobuf/proto" + "github.com/flyteorg/flyte/flyteartifacts/pkg/lib" "github.com/flyteorg/flyte/flyteidl/clients/go/admin" adminPb "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" "github.com/flyteorg/flyte/flytestdlib/logger" - "github.com/golang/protobuf/proto" ) // ServiceCallHandler will take events and call the grpc endpoints directly. The service should most likely be local. diff --git a/flyteartifacts/pkg/server/processor/events_handler_test.go b/flyteartifacts/pkg/server/processor/events_handler_test.go index 61d668215b..57c38a539f 100644 --- a/flyteartifacts/pkg/server/processor/events_handler_test.go +++ b/flyteartifacts/pkg/server/processor/events_handler_test.go @@ -1,9 +1,11 @@ package processor import ( - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) func TestMetaDataWrite(t *testing.T) { diff --git a/flyteartifacts/pkg/server/processor/interfaces.go b/flyteartifacts/pkg/server/processor/interfaces.go index 8f5fd8ab8f..90264b2669 100644 --- a/flyteartifacts/pkg/server/processor/interfaces.go +++ b/flyteartifacts/pkg/server/processor/interfaces.go @@ -2,6 +2,7 @@ package processor import ( "context" + "github.com/cloudevents/sdk-go/v2/event" "github.com/golang/protobuf/proto" ) diff --git a/flyteartifacts/pkg/server/processor/processor.go b/flyteartifacts/pkg/server/processor/processor.go index 8e00c3ef22..8995f0132d 100644 --- a/flyteartifacts/pkg/server/processor/processor.go +++ b/flyteartifacts/pkg/server/processor/processor.go @@ -2,8 +2,10 @@ package processor import ( "context" + "github.com/NYTimes/gizmo/pubsub" gizmoAWS "github.com/NYTimes/gizmo/pubsub/aws" + "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" configCommon "github.com/flyteorg/flyte/flytestdlib/config" diff --git a/flyteartifacts/pkg/server/processor/sqs_processor.go b/flyteartifacts/pkg/server/processor/sqs_processor.go index 17bce5e645..0430204570 100644 --- a/flyteartifacts/pkg/server/processor/sqs_processor.go +++ b/flyteartifacts/pkg/server/processor/sqs_processor.go @@ -10,10 +10,10 @@ import ( "github.com/NYTimes/gizmo/pubsub" pbcloudevents "github.com/cloudevents/sdk-go/binding/format/protobuf/v2" "github.com/cloudevents/sdk-go/v2/event" - flyteEvents "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" + flyteEvents "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" ) diff --git a/flyteartifacts/pkg/server/processor/sqs_processor_test.go b/flyteartifacts/pkg/server/processor/sqs_processor_test.go index 8819398b38..9b2fd0012a 100644 --- a/flyteartifacts/pkg/server/processor/sqs_processor_test.go +++ b/flyteartifacts/pkg/server/processor/sqs_processor_test.go @@ -3,12 +3,14 @@ package processor import ( "context" "fmt" + "testing" + gizmoAWS "github.com/NYTimes/gizmo/pubsub/aws" "github.com/cloudevents/sdk-go/v2/event" + "github.com/golang/protobuf/proto" + "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" - "github.com/golang/protobuf/proto" - "testing" ) type EchoEventsHandler struct{} diff --git a/flyteartifacts/pkg/server/server.go b/flyteartifacts/pkg/server/server.go index 5751af52fc..89e9448205 100644 --- a/flyteartifacts/pkg/server/server.go +++ b/flyteartifacts/pkg/server/server.go @@ -3,6 +3,13 @@ package server import ( "context" "fmt" + _ "net/http/pprof" // Required to serve application. + + "github.com/go-gormigrate/gormigrate/v2" + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "github.com/pkg/errors" + "google.golang.org/grpc" + "github.com/flyteorg/flyte/flyteartifacts/pkg/blob" "github.com/flyteorg/flyte/flyteartifacts/pkg/configuration" "github.com/flyteorg/flyte/flyteartifacts/pkg/db" @@ -12,11 +19,6 @@ import ( "github.com/flyteorg/flyte/flytestdlib/database" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" - "github.com/go-gormigrate/gormigrate/v2" - "github.com/grpc-ecosystem/grpc-gateway/runtime" - "github.com/pkg/errors" - "google.golang.org/grpc" - _ "net/http/pprof" // Required to serve application. ) type ArtifactService struct { diff --git a/flyteartifacts/pkg/server/service.go b/flyteartifacts/pkg/server/service.go index e28124a415..abc1b81b6a 100644 --- a/flyteartifacts/pkg/server/service.go +++ b/flyteartifacts/pkg/server/service.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "github.com/flyteorg/flyte/flyteartifacts/pkg/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/artifact" "github.com/flyteorg/flyte/flytestdlib/logger" diff --git a/flyteartifacts/pkg/server/trigger_engine.go b/flyteartifacts/pkg/server/trigger_engine.go index e0d2ae5bdd..49d5593f69 100644 --- a/flyteartifacts/pkg/server/trigger_engine.go +++ b/flyteartifacts/pkg/server/trigger_engine.go @@ -3,6 +3,16 @@ package server import ( "context" "fmt" + "math/rand" + "regexp" + "strconv" + "time" + "unicode/utf8" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + "github.com/flyteorg/flyte/flyteartifacts/pkg/lib" "github.com/flyteorg/flyte/flyteartifacts/pkg/models" admin2 "github.com/flyteorg/flyte/flyteidl/clients/go/admin" @@ -12,14 +22,6 @@ import ( "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/timestamppb" - "math/rand" - "regexp" - "strconv" - "time" - "unicode/utf8" ) type TriggerEngine struct { diff --git a/flyteartifacts/pkg/server/trigger_engine_test.go b/flyteartifacts/pkg/server/trigger_engine_test.go index 3c61d6cc46..e7033b3c41 100644 --- a/flyteartifacts/pkg/server/trigger_engine_test.go +++ b/flyteartifacts/pkg/server/trigger_engine_test.go @@ -1,9 +1,10 @@ package server import ( - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestA(t *testing.T) {