From 45e287ae1e00d6840c3b4661f9ab4d6e0d6a2346 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Thu, 1 Aug 2024 15:19:05 -0700 Subject: [PATCH 01/23] [flyteadmin] Refactor panic recovery into middleware (#5546) * Refactor panic handling to middleware Signed-off-by: Jason Parraga * Remove registration of old panicCounter Signed-off-by: Jason Parraga * Add test coverage Signed-off-by: Jason Parraga --------- Signed-off-by: Jason Parraga --- flyteadmin/pkg/rpc/adminservice/attributes.go | 10 --- flyteadmin/pkg/rpc/adminservice/base.go | 13 --- flyteadmin/pkg/rpc/adminservice/base_test.go | 40 --------- .../rpc/adminservice/description_entity.go | 2 - flyteadmin/pkg/rpc/adminservice/execution.go | 10 --- .../pkg/rpc/adminservice/launch_plan.go | 7 -- flyteadmin/pkg/rpc/adminservice/metrics.go | 7 +- .../middleware/recovery_interceptor.go | 61 +++++++++++++ .../middleware/recovery_interceptor_test.go | 90 +++++++++++++++++++ .../pkg/rpc/adminservice/named_entity.go | 3 - .../pkg/rpc/adminservice/node_execution.go | 6 -- flyteadmin/pkg/rpc/adminservice/project.go | 5 -- flyteadmin/pkg/rpc/adminservice/task.go | 4 - .../pkg/rpc/adminservice/task_execution.go | 4 - flyteadmin/pkg/rpc/adminservice/version.go | 1 - flyteadmin/pkg/rpc/adminservice/workflow.go | 4 - flyteadmin/pkg/server/service.go | 29 +++++- 17 files changed, 177 insertions(+), 119 deletions(-) delete mode 100644 flyteadmin/pkg/rpc/adminservice/base_test.go create mode 100644 flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor.go create mode 100644 flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor_test.go diff --git a/flyteadmin/pkg/rpc/adminservice/attributes.go b/flyteadmin/pkg/rpc/adminservice/attributes.go index 46607da93e..62002a0e6e 100644 --- a/flyteadmin/pkg/rpc/adminservice/attributes.go +++ b/flyteadmin/pkg/rpc/adminservice/attributes.go @@ -12,7 +12,6 @@ import ( func (m *AdminService) UpdateWorkflowAttributes(ctx context.Context, request *admin.WorkflowAttributesUpdateRequest) ( *admin.WorkflowAttributesUpdateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -30,7 +29,6 @@ func (m *AdminService) UpdateWorkflowAttributes(ctx context.Context, request *ad func (m *AdminService) GetWorkflowAttributes(ctx context.Context, request *admin.WorkflowAttributesGetRequest) ( *admin.WorkflowAttributesGetResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -48,7 +46,6 @@ func (m *AdminService) GetWorkflowAttributes(ctx context.Context, request *admin func (m *AdminService) DeleteWorkflowAttributes(ctx context.Context, request *admin.WorkflowAttributesDeleteRequest) ( *admin.WorkflowAttributesDeleteResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -66,7 +63,6 @@ func (m *AdminService) DeleteWorkflowAttributes(ctx context.Context, request *ad func (m *AdminService) UpdateProjectDomainAttributes(ctx context.Context, request *admin.ProjectDomainAttributesUpdateRequest) ( *admin.ProjectDomainAttributesUpdateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -84,7 +80,6 @@ func (m *AdminService) UpdateProjectDomainAttributes(ctx context.Context, reques func (m *AdminService) GetProjectDomainAttributes(ctx context.Context, request *admin.ProjectDomainAttributesGetRequest) ( *admin.ProjectDomainAttributesGetResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -102,7 +97,6 @@ func (m *AdminService) GetProjectDomainAttributes(ctx context.Context, request * func (m *AdminService) DeleteProjectDomainAttributes(ctx context.Context, request *admin.ProjectDomainAttributesDeleteRequest) ( *admin.ProjectDomainAttributesDeleteResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -121,7 +115,6 @@ func (m *AdminService) DeleteProjectDomainAttributes(ctx context.Context, reques func (m *AdminService) UpdateProjectAttributes(ctx context.Context, request *admin.ProjectAttributesUpdateRequest) ( *admin.ProjectAttributesUpdateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -140,7 +133,6 @@ func (m *AdminService) UpdateProjectAttributes(ctx context.Context, request *adm func (m *AdminService) GetProjectAttributes(ctx context.Context, request *admin.ProjectAttributesGetRequest) ( *admin.ProjectAttributesGetResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -159,7 +151,6 @@ func (m *AdminService) GetProjectAttributes(ctx context.Context, request *admin. func (m *AdminService) DeleteProjectAttributes(ctx context.Context, request *admin.ProjectAttributesDeleteRequest) ( *admin.ProjectAttributesDeleteResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -177,7 +168,6 @@ func (m *AdminService) DeleteProjectAttributes(ctx context.Context, request *adm func (m *AdminService) ListMatchableAttributes(ctx context.Context, request *admin.ListMatchableAttributesRequest) ( *admin.ListMatchableAttributesResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/base.go b/flyteadmin/pkg/rpc/adminservice/base.go index 5a2cb2ad89..8df2c595c7 100644 --- a/flyteadmin/pkg/rpc/adminservice/base.go +++ b/flyteadmin/pkg/rpc/adminservice/base.go @@ -5,8 +5,6 @@ import ( "fmt" "runtime/debug" - "github.com/golang/protobuf/proto" - "github.com/flyteorg/flyte/flyteadmin/pkg/async/cloudevent" eventWriter "github.com/flyteorg/flyte/flyteadmin/pkg/async/events/implementations" "github.com/flyteorg/flyte/flyteadmin/pkg/async/notifications" @@ -44,17 +42,6 @@ type AdminService struct { Metrics AdminMetrics } -// Intercepts all admin requests to handle panics during execution. -func (m *AdminService) interceptPanic(ctx context.Context, request proto.Message) { - err := recover() - if err == nil { - return - } - - m.Metrics.PanicCounter.Inc() - logger.Fatalf(ctx, "panic-ed for request: [%+v] with err: %v with Stack: %v", request, err, string(debug.Stack())) -} - const defaultRetries = 3 func NewAdminServer(ctx context.Context, pluginRegistry *plugins.Registry, configuration runtimeIfaces.Configuration, diff --git a/flyteadmin/pkg/rpc/adminservice/base_test.go b/flyteadmin/pkg/rpc/adminservice/base_test.go deleted file mode 100644 index 9b1cb626d5..0000000000 --- a/flyteadmin/pkg/rpc/adminservice/base_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package adminservice - -import ( - "context" - "testing" - - "github.com/golang/protobuf/proto" - "github.com/stretchr/testify/assert" - - "github.com/flyteorg/flyte/flytestdlib/logger" - "github.com/flyteorg/flyte/flytestdlib/promutils" -) - -func Test_interceptPanic(t *testing.T) { - m := AdminService{ - Metrics: InitMetrics(promutils.NewTestScope()), - } - - ctx := context.Background() - - // Mute logs to avoid .Fatal() (called in interceptPanic) causing the process to close - assert.NoError(t, logger.SetConfig(&logger.Config{Mute: true})) - - func() { - defer func() { - if err := recover(); err != nil { - assert.Fail(t, "Unexpected error", err) - } - }() - - a := func() { - defer m.interceptPanic(ctx, proto.Message(nil)) - - var x *int - *x = 10 - } - - a() - }() -} diff --git a/flyteadmin/pkg/rpc/adminservice/description_entity.go b/flyteadmin/pkg/rpc/adminservice/description_entity.go index 1d08234051..bc2d794aed 100644 --- a/flyteadmin/pkg/rpc/adminservice/description_entity.go +++ b/flyteadmin/pkg/rpc/adminservice/description_entity.go @@ -13,7 +13,6 @@ import ( ) func (m *AdminService) GetDescriptionEntity(ctx context.Context, request *admin.ObjectGetRequest) (*admin.DescriptionEntity, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -36,7 +35,6 @@ func (m *AdminService) GetDescriptionEntity(ctx context.Context, request *admin. } func (m *AdminService) ListDescriptionEntities(ctx context.Context, request *admin.DescriptionEntityListRequest) (*admin.DescriptionEntityList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/execution.go b/flyteadmin/pkg/rpc/adminservice/execution.go index 919ed851a3..15caf5aa75 100644 --- a/flyteadmin/pkg/rpc/adminservice/execution.go +++ b/flyteadmin/pkg/rpc/adminservice/execution.go @@ -13,7 +13,6 @@ import ( func (m *AdminService) CreateExecution( ctx context.Context, request *admin.ExecutionCreateRequest) (*admin.ExecutionCreateResponse, error) { - defer m.interceptPanic(ctx, request) requestedAt := time.Now() if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") @@ -32,7 +31,6 @@ func (m *AdminService) CreateExecution( func (m *AdminService) RelaunchExecution( ctx context.Context, request *admin.ExecutionRelaunchRequest) (*admin.ExecutionCreateResponse, error) { - defer m.interceptPanic(ctx, request) requestedAt := time.Now() if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") @@ -51,7 +49,6 @@ func (m *AdminService) RelaunchExecution( func (m *AdminService) RecoverExecution( ctx context.Context, request *admin.ExecutionRecoverRequest) (*admin.ExecutionCreateResponse, error) { - defer m.interceptPanic(ctx, request) requestedAt := time.Now() if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") @@ -70,7 +67,6 @@ func (m *AdminService) RecoverExecution( func (m *AdminService) CreateWorkflowEvent( ctx context.Context, request *admin.WorkflowExecutionEventRequest) (*admin.WorkflowExecutionEventResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -89,7 +85,6 @@ func (m *AdminService) CreateWorkflowEvent( func (m *AdminService) GetExecution( ctx context.Context, request *admin.WorkflowExecutionGetRequest) (*admin.Execution, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -107,7 +102,6 @@ func (m *AdminService) GetExecution( func (m *AdminService) UpdateExecution( ctx context.Context, request *admin.ExecutionUpdateRequest) (*admin.ExecutionUpdateResponse, error) { - defer m.interceptPanic(ctx, request) requestedAt := time.Now() if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") @@ -126,7 +120,6 @@ func (m *AdminService) UpdateExecution( func (m *AdminService) GetExecutionData( ctx context.Context, request *admin.WorkflowExecutionGetDataRequest) (*admin.WorkflowExecutionGetDataResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -144,7 +137,6 @@ func (m *AdminService) GetExecutionData( func (m *AdminService) GetExecutionMetrics( ctx context.Context, request *admin.WorkflowExecutionGetMetricsRequest) (*admin.WorkflowExecutionGetMetricsResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -162,7 +154,6 @@ func (m *AdminService) GetExecutionMetrics( func (m *AdminService) ListExecutions( ctx context.Context, request *admin.ResourceListRequest) (*admin.ExecutionList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -180,7 +171,6 @@ func (m *AdminService) ListExecutions( func (m *AdminService) TerminateExecution( ctx context.Context, request *admin.ExecutionTerminateRequest) (*admin.ExecutionTerminateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/launch_plan.go b/flyteadmin/pkg/rpc/adminservice/launch_plan.go index ff3c2480e0..1586c3f542 100644 --- a/flyteadmin/pkg/rpc/adminservice/launch_plan.go +++ b/flyteadmin/pkg/rpc/adminservice/launch_plan.go @@ -14,7 +14,6 @@ import ( func (m *AdminService) CreateLaunchPlan( ctx context.Context, request *admin.LaunchPlanCreateRequest) (*admin.LaunchPlanCreateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -31,7 +30,6 @@ func (m *AdminService) CreateLaunchPlan( } func (m *AdminService) GetLaunchPlan(ctx context.Context, request *admin.ObjectGetRequest) (*admin.LaunchPlan, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -55,7 +53,6 @@ func (m *AdminService) GetLaunchPlan(ctx context.Context, request *admin.ObjectG } func (m *AdminService) GetActiveLaunchPlan(ctx context.Context, request *admin.ActiveLaunchPlanRequest) (*admin.LaunchPlan, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -73,7 +70,6 @@ func (m *AdminService) GetActiveLaunchPlan(ctx context.Context, request *admin.A func (m *AdminService) UpdateLaunchPlan(ctx context.Context, request *admin.LaunchPlanUpdateRequest) ( *admin.LaunchPlanUpdateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -97,7 +93,6 @@ func (m *AdminService) UpdateLaunchPlan(ctx context.Context, request *admin.Laun func (m *AdminService) ListLaunchPlans(ctx context.Context, request *admin.ResourceListRequest) ( *admin.LaunchPlanList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Empty request. Please rephrase.") } @@ -116,7 +111,6 @@ func (m *AdminService) ListLaunchPlans(ctx context.Context, request *admin.Resou func (m *AdminService) ListActiveLaunchPlans(ctx context.Context, request *admin.ActiveLaunchPlanListRequest) ( *admin.LaunchPlanList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Empty request. Please rephrase.") } @@ -135,7 +129,6 @@ func (m *AdminService) ListActiveLaunchPlans(ctx context.Context, request *admin func (m *AdminService) ListLaunchPlanIds(ctx context.Context, request *admin.NamedEntityIdentifierListRequest) ( *admin.NamedEntityIdentifierList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Empty request. Please rephrase.") } diff --git a/flyteadmin/pkg/rpc/adminservice/metrics.go b/flyteadmin/pkg/rpc/adminservice/metrics.go index 65c6b741f3..f770665ef6 100644 --- a/flyteadmin/pkg/rpc/adminservice/metrics.go +++ b/flyteadmin/pkg/rpc/adminservice/metrics.go @@ -2,8 +2,6 @@ package adminservice import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/flyteorg/flyte/flyteadmin/pkg/rpc/adminservice/util" "github.com/flyteorg/flyte/flytestdlib/promutils" ) @@ -115,8 +113,7 @@ type descriptionEntityEndpointMetrics struct { } type AdminMetrics struct { - Scope promutils.Scope - PanicCounter prometheus.Counter + Scope promutils.Scope executionEndpointMetrics executionEndpointMetrics launchPlanEndpointMetrics launchPlanEndpointMetrics @@ -137,8 +134,6 @@ type AdminMetrics struct { func InitMetrics(adminScope promutils.Scope) AdminMetrics { return AdminMetrics{ Scope: adminScope, - PanicCounter: adminScope.MustNewCounter("handler_panic", - "panics encountered while handling requests to the admin service"), executionEndpointMetrics: executionEndpointMetrics{ scope: adminScope, diff --git a/flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor.go b/flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor.go new file mode 100644 index 0000000000..a0a699a4f0 --- /dev/null +++ b/flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor.go @@ -0,0 +1,61 @@ +package middleware + +import ( + "context" + "runtime/debug" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/flyteorg/flyte/flytestdlib/logger" + "github.com/flyteorg/flyte/flytestdlib/promutils" +) + +// RecoveryInterceptor is a struct for creating gRPC interceptors that handle panics in go +type RecoveryInterceptor struct { + panicCounter prometheus.Counter +} + +// NewRecoveryInterceptor creates a new RecoveryInterceptor with metrics under the provided scope +func NewRecoveryInterceptor(adminScope promutils.Scope) *RecoveryInterceptor { + panicCounter := adminScope.MustNewCounter("handler_panic", "panics encountered while handling gRPC requests") + return &RecoveryInterceptor{ + panicCounter: panicCounter, + } +} + +// UnaryServerInterceptor returns a new unary server interceptor for panic recovery. +func (ri *RecoveryInterceptor) UnaryServerInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ any, err error) { + + defer func() { + if r := recover(); r != nil { + ri.panicCounter.Inc() + logger.Errorf(ctx, "panic-ed for request: [%+v] to %s with err: %v with Stack: %v", req, info.FullMethod, r, string(debug.Stack())) + // Return INTERNAL to client with no info as to not leak implementation details + err = status.Errorf(codes.Internal, "") + } + }() + + return handler(ctx, req) + } +} + +// StreamServerInterceptor returns a new streaming server interceptor for panic recovery. +func (ri *RecoveryInterceptor) StreamServerInterceptor() grpc.StreamServerInterceptor { + return func(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) { + + defer func() { + if r := recover(); r != nil { + ri.panicCounter.Inc() + logger.Errorf(stream.Context(), "panic-ed for stream to %s with err: %v with Stack: %v", info.FullMethod, r, string(debug.Stack())) + // Return INTERNAL to client with no info as to not leak implementation details + err = status.Errorf(codes.Internal, "") + } + }() + + return handler(srv, stream) + } +} diff --git a/flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor_test.go b/flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor_test.go new file mode 100644 index 0000000000..3928856067 --- /dev/null +++ b/flyteadmin/pkg/rpc/adminservice/middleware/recovery_interceptor_test.go @@ -0,0 +1,90 @@ +package middleware + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + mockScope "github.com/flyteorg/flyte/flytestdlib/promutils" +) + +func TestRecoveryInterceptor(t *testing.T) { + ctx := context.Background() + testScope := mockScope.NewTestScope() + recoveryInterceptor := NewRecoveryInterceptor(testScope) + unaryInterceptor := recoveryInterceptor.UnaryServerInterceptor() + streamInterceptor := recoveryInterceptor.StreamServerInterceptor() + unaryInfo := &grpc.UnaryServerInfo{} + streamInfo := &grpc.StreamServerInfo{} + req := "test-request" + + t.Run("unary should recover from panic", func(t *testing.T) { + _, err := unaryInterceptor(ctx, req, unaryInfo, func(ctx context.Context, req any) (any, error) { + panic("synthetic") + }) + expectedErr := status.Errorf(codes.Internal, "") + require.Error(t, err) + require.Equal(t, expectedErr, err) + }) + + t.Run("stream should recover from panic", func(t *testing.T) { + stream := testStream{} + err := streamInterceptor(nil, &stream, streamInfo, func(srv any, stream grpc.ServerStream) error { + panic("synthetic") + }) + expectedErr := status.Errorf(codes.Internal, "") + require.Error(t, err) + require.Equal(t, expectedErr, err) + }) + + t.Run("unary should plumb response without panic", func(t *testing.T) { + mockedResponse := "test" + resp, err := unaryInterceptor(ctx, req, unaryInfo, func(ctx context.Context, req any) (any, error) { + return mockedResponse, nil + }) + require.NoError(t, err) + require.Equal(t, mockedResponse, resp) + }) + + t.Run("stream should plumb response without panic", func(t *testing.T) { + stream := testStream{} + handlerCalled := false + err := streamInterceptor(nil, &stream, streamInfo, func(srv any, stream grpc.ServerStream) error { + handlerCalled = true + return nil + }) + require.NoError(t, err) + require.True(t, handlerCalled) + }) +} + +// testStream is an implementation of grpc.ServerStream for testing. +type testStream struct { +} + +func (s *testStream) SendMsg(m interface{}) error { + return nil +} + +func (s *testStream) RecvMsg(m interface{}) error { + return nil +} + +func (s *testStream) SetHeader(metadata.MD) error { + return nil +} + +func (s *testStream) SendHeader(metadata.MD) error { + return nil +} + +func (s *testStream) SetTrailer(metadata.MD) {} + +func (s *testStream) Context() context.Context { + return context.Background() +} diff --git a/flyteadmin/pkg/rpc/adminservice/named_entity.go b/flyteadmin/pkg/rpc/adminservice/named_entity.go index d48a0485e2..4ef8f3ee0b 100644 --- a/flyteadmin/pkg/rpc/adminservice/named_entity.go +++ b/flyteadmin/pkg/rpc/adminservice/named_entity.go @@ -11,7 +11,6 @@ import ( ) func (m *AdminService) GetNamedEntity(ctx context.Context, request *admin.NamedEntityGetRequest) (*admin.NamedEntity, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -31,7 +30,6 @@ func (m *AdminService) GetNamedEntity(ctx context.Context, request *admin.NamedE func (m *AdminService) UpdateNamedEntity(ctx context.Context, request *admin.NamedEntityUpdateRequest) ( *admin.NamedEntityUpdateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -50,7 +48,6 @@ func (m *AdminService) UpdateNamedEntity(ctx context.Context, request *admin.Nam func (m *AdminService) ListNamedEntities(ctx context.Context, request *admin.NamedEntityListRequest) ( *admin.NamedEntityList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/node_execution.go b/flyteadmin/pkg/rpc/adminservice/node_execution.go index cf17e3ff70..1b187f3a35 100644 --- a/flyteadmin/pkg/rpc/adminservice/node_execution.go +++ b/flyteadmin/pkg/rpc/adminservice/node_execution.go @@ -14,7 +14,6 @@ import ( func (m *AdminService) CreateNodeEvent( ctx context.Context, request *admin.NodeExecutionEventRequest) (*admin.NodeExecutionEventResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -32,7 +31,6 @@ func (m *AdminService) CreateNodeEvent( func (m *AdminService) GetNodeExecution( ctx context.Context, request *admin.NodeExecutionGetRequest) (*admin.NodeExecution, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -49,7 +47,6 @@ func (m *AdminService) GetNodeExecution( } func (m *AdminService) GetDynamicNodeWorkflow(ctx context.Context, request *admin.GetDynamicNodeWorkflowRequest) (*admin.DynamicNodeWorkflowResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -68,7 +65,6 @@ func (m *AdminService) GetDynamicNodeWorkflow(ctx context.Context, request *admi func (m *AdminService) ListNodeExecutions( ctx context.Context, request *admin.NodeExecutionListRequest) (*admin.NodeExecutionList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -86,7 +82,6 @@ func (m *AdminService) ListNodeExecutions( func (m *AdminService) ListNodeExecutionsForTask( ctx context.Context, request *admin.NodeExecutionForTaskListRequest) (*admin.NodeExecutionList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -111,7 +106,6 @@ func (m *AdminService) ListNodeExecutionsForTask( func (m *AdminService) GetNodeExecutionData( ctx context.Context, request *admin.NodeExecutionGetDataRequest) (*admin.NodeExecutionGetDataResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/project.go b/flyteadmin/pkg/rpc/adminservice/project.go index 5e7352ad93..ab8d8e4375 100644 --- a/flyteadmin/pkg/rpc/adminservice/project.go +++ b/flyteadmin/pkg/rpc/adminservice/project.go @@ -12,7 +12,6 @@ import ( func (m *AdminService) RegisterProject(ctx context.Context, request *admin.ProjectRegisterRequest) ( *admin.ProjectRegisterResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -29,7 +28,6 @@ func (m *AdminService) RegisterProject(ctx context.Context, request *admin.Proje } func (m *AdminService) ListProjects(ctx context.Context, request *admin.ProjectListRequest) (*admin.Projects, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -48,7 +46,6 @@ func (m *AdminService) ListProjects(ctx context.Context, request *admin.ProjectL func (m *AdminService) UpdateProject(ctx context.Context, request *admin.Project) ( *admin.ProjectUpdateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -65,7 +62,6 @@ func (m *AdminService) UpdateProject(ctx context.Context, request *admin.Project } func (m *AdminService) GetProject(ctx context.Context, request *admin.ProjectGetRequest) (*admin.Project, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -83,7 +79,6 @@ func (m *AdminService) GetProject(ctx context.Context, request *admin.ProjectGet } func (m *AdminService) GetDomains(ctx context.Context, request *admin.GetDomainRequest) (*admin.GetDomainsResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/task.go b/flyteadmin/pkg/rpc/adminservice/task.go index 8899480489..7db51ed2eb 100644 --- a/flyteadmin/pkg/rpc/adminservice/task.go +++ b/flyteadmin/pkg/rpc/adminservice/task.go @@ -15,7 +15,6 @@ import ( func (m *AdminService) CreateTask( ctx context.Context, request *admin.TaskCreateRequest) (*admin.TaskCreateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -32,7 +31,6 @@ func (m *AdminService) CreateTask( } func (m *AdminService) GetTask(ctx context.Context, request *admin.ObjectGetRequest) (*admin.Task, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -56,7 +54,6 @@ func (m *AdminService) GetTask(ctx context.Context, request *admin.ObjectGetRequ func (m *AdminService) ListTaskIds( ctx context.Context, request *admin.NamedEntityIdentifierListRequest) (*admin.NamedEntityIdentifierList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -74,7 +71,6 @@ func (m *AdminService) ListTaskIds( } func (m *AdminService) ListTasks(ctx context.Context, request *admin.ResourceListRequest) (*admin.TaskList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/task_execution.go b/flyteadmin/pkg/rpc/adminservice/task_execution.go index 0561a1ba36..0638c02aa3 100644 --- a/flyteadmin/pkg/rpc/adminservice/task_execution.go +++ b/flyteadmin/pkg/rpc/adminservice/task_execution.go @@ -15,7 +15,6 @@ import ( func (m *AdminService) CreateTaskEvent( ctx context.Context, request *admin.TaskExecutionEventRequest) (*admin.TaskExecutionEventResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -34,7 +33,6 @@ func (m *AdminService) CreateTaskEvent( func (m *AdminService) GetTaskExecution( ctx context.Context, request *admin.TaskExecutionGetRequest) (*admin.TaskExecution, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -62,7 +60,6 @@ func (m *AdminService) GetTaskExecution( func (m *AdminService) ListTaskExecutions( ctx context.Context, request *admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Nil request") } @@ -84,7 +81,6 @@ func (m *AdminService) ListTaskExecutions( func (m *AdminService) GetTaskExecutionData( ctx context.Context, request *admin.TaskExecutionGetDataRequest) (*admin.TaskExecutionGetDataResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/rpc/adminservice/version.go b/flyteadmin/pkg/rpc/adminservice/version.go index 7fb5861e50..3049a723aa 100644 --- a/flyteadmin/pkg/rpc/adminservice/version.go +++ b/flyteadmin/pkg/rpc/adminservice/version.go @@ -8,7 +8,6 @@ import ( func (m *AdminService) GetVersion(ctx context.Context, request *admin.GetVersionRequest) (*admin.GetVersionResponse, error) { - defer m.interceptPanic(ctx, request) response, err := m.VersionManager.GetVersion(ctx, request) if err != nil { return nil, err diff --git a/flyteadmin/pkg/rpc/adminservice/workflow.go b/flyteadmin/pkg/rpc/adminservice/workflow.go index 9fcf87c453..7f6ecc4c13 100644 --- a/flyteadmin/pkg/rpc/adminservice/workflow.go +++ b/flyteadmin/pkg/rpc/adminservice/workflow.go @@ -15,7 +15,6 @@ import ( func (m *AdminService) CreateWorkflow( ctx context.Context, request *admin.WorkflowCreateRequest) (*admin.WorkflowCreateResponse, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -32,7 +31,6 @@ func (m *AdminService) CreateWorkflow( } func (m *AdminService) GetWorkflow(ctx context.Context, request *admin.ObjectGetRequest) (*admin.Workflow, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -56,7 +54,6 @@ func (m *AdminService) GetWorkflow(ctx context.Context, request *admin.ObjectGet func (m *AdminService) ListWorkflowIds(ctx context.Context, request *admin.NamedEntityIdentifierListRequest) ( *admin.NamedEntityIdentifierList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } @@ -75,7 +72,6 @@ func (m *AdminService) ListWorkflowIds(ctx context.Context, request *admin.Named } func (m *AdminService) ListWorkflows(ctx context.Context, request *admin.ResourceListRequest) (*admin.WorkflowList, error) { - defer m.interceptPanic(ctx, request) if request == nil { return nil, status.Errorf(codes.InvalidArgument, "Incorrect request, nil requests not allowed") } diff --git a/flyteadmin/pkg/server/service.go b/flyteadmin/pkg/server/service.go index ff80c343d3..bb09f9f615 100644 --- a/flyteadmin/pkg/server/service.go +++ b/flyteadmin/pkg/server/service.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/handlers" grpcmiddleware "github.com/grpc-ecosystem/go-grpc-middleware" grpcauth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/pkg/errors" @@ -35,6 +36,7 @@ import ( "github.com/flyteorg/flyte/flyteadmin/pkg/config" "github.com/flyteorg/flyte/flyteadmin/pkg/rpc" "github.com/flyteorg/flyte/flyteadmin/pkg/rpc/adminservice" + "github.com/flyteorg/flyte/flyteadmin/pkg/rpc/adminservice/middleware" runtime2 "github.com/flyteorg/flyte/flyteadmin/pkg/runtime" runtimeIfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flyte/flyteadmin/plugins" @@ -98,11 +100,18 @@ func newGRPCServer(ctx context.Context, pluginRegistry *plugins.Registry, cfg *c otelgrpc.WithPropagators(propagation.TraceContext{}), ) + adminScope := scope.NewSubScope("admin") + recoveryInterceptor := middleware.NewRecoveryInterceptor(adminScope) + var chainedUnaryInterceptors grpc.UnaryServerInterceptor if cfg.Security.UseAuth { logger.Infof(ctx, "Creating gRPC server with authentication") middlewareInterceptors := plugins.Get[grpc.UnaryServerInterceptor](pluginRegistry, plugins.PluginIDUnaryServiceMiddleware) - chainedUnaryInterceptors = grpcmiddleware.ChainUnaryServer(grpcprometheus.UnaryServerInterceptor, + chainedUnaryInterceptors = grpcmiddleware.ChainUnaryServer( + // recovery interceptor should always be first in order to handle any panics in the middleware or server + recoveryInterceptor.UnaryServerInterceptor(), + grpcrecovery.UnaryServerInterceptor(), + grpcprometheus.UnaryServerInterceptor, otelUnaryServerInterceptor, auth.GetAuthenticationCustomMetadataInterceptor(authCtx), grpcauth.UnaryServerInterceptor(auth.GetAuthenticationInterceptor(authCtx)), @@ -111,11 +120,23 @@ func newGRPCServer(ctx context.Context, pluginRegistry *plugins.Registry, cfg *c ) } else { logger.Infof(ctx, "Creating gRPC server without authentication") - chainedUnaryInterceptors = grpcmiddleware.ChainUnaryServer(grpcprometheus.UnaryServerInterceptor, otelUnaryServerInterceptor) + chainedUnaryInterceptors = grpcmiddleware.ChainUnaryServer( + // recovery interceptor should always be first in order to handle any panics in the middleware or server + recoveryInterceptor.UnaryServerInterceptor(), + grpcprometheus.UnaryServerInterceptor, + otelUnaryServerInterceptor, + ) } + chainedStreamInterceptors := grpcmiddleware.ChainStreamServer( + // recovery interceptor should always be first in order to handle any panics in the middleware or server + recoveryInterceptor.StreamServerInterceptor(), + grpcprometheus.StreamServerInterceptor, + ) + serverOpts := []grpc.ServerOption{ - grpc.StreamInterceptor(grpcprometheus.StreamServerInterceptor), + // recovery interceptor should always be first in order to handle any panics in the middleware or server + grpc.StreamInterceptor(chainedStreamInterceptors), grpc.UnaryInterceptor(chainedUnaryInterceptors), } if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { @@ -131,7 +152,7 @@ func newGRPCServer(ctx context.Context, pluginRegistry *plugins.Registry, cfg *c } configuration := runtime2.NewConfigurationProvider() - adminServer := adminservice.NewAdminServer(ctx, pluginRegistry, configuration, cfg.KubeConfig, cfg.Master, dataStorageClient, scope.NewSubScope("admin")) + adminServer := adminservice.NewAdminServer(ctx, pluginRegistry, configuration, cfg.KubeConfig, cfg.Master, dataStorageClient, adminScope) grpcService.RegisterAdminServiceServer(grpcServer, adminServer) if cfg.Security.UseAuth { grpcService.RegisterAuthMetadataServiceServer(grpcServer, authCtx.AuthMetadataService()) From b18449e2f441216e7eb50de4d2317302f7759232 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Fri, 2 Aug 2024 11:41:16 +0800 Subject: [PATCH 02/23] Snowflake agent Doc (#5620) * TEST build Signed-off-by: Future-Outlier * remove emphasize-lines Signed-off-by: Future-Outlier * test build Signed-off-by: Future-Outlier * revert Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier --- docs/deployment/agents/index.md | 6 ++++-- docs/deployment/agents/snowflake.rst | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/deployment/agents/index.md b/docs/deployment/agents/index.md index 912ab8613c..11ce607788 100644 --- a/docs/deployment/agents/index.md +++ b/docs/deployment/agents/index.md @@ -25,10 +25,12 @@ If you are using a managed deployment of Flyte, you will need to contact your de - Configuring your Flyte deployment for the BigQuery agent. * - {ref}`MMCloud Agent ` - Configuring your Flyte deployment for the MMCloud agent. -* - {ref}`Sensor Agent ` - - Configuring your Flyte deployment for the sensor agent. * - {ref}`SageMaker Inference ` - Deploy models and create, as well as trigger inference endpoints on SageMaker. +* - {ref}`Sensor Agent ` + - Configuring your Flyte deployment for the sensor agent. +* - {ref}`Snowflake Agent ` + - Configuring your Flyte deployment for the SnowFlake agent. * - {ref}`OpenAI Batch ` - Submit requests to OpenAI GPT models for asynchronous batch processing. ``` diff --git a/docs/deployment/agents/snowflake.rst b/docs/deployment/agents/snowflake.rst index fe1c8482ae..a689c748bf 100644 --- a/docs/deployment/agents/snowflake.rst +++ b/docs/deployment/agents/snowflake.rst @@ -1,16 +1,25 @@ .. _deployment-agent-setup-snowflake: Snowflake agent -================= +=============== This guide provides an overview of how to set up the Snowflake agent in your Flyte deployment. 1. Set up the key pair authentication in Snowflake. For more details, see the `Snowflake key-pair authentication and key-pair rotation guide `__. -2. Create a secret with the group "snowflake" and the key "private_key". For more details, see `"Using Secrets in a Task" `__. +2. Create a secret with the group "private_key" and the key "snowflake". + This is hardcoded in the flytekit sdk, since we can't know the group and key name in advance. + This is for permission to upload and download data with structured dataset in python task pod. .. code-block:: bash - kubectl create secret generic snowflake-private-key --namespace=flytesnacks-development --from-file=your_private_key_above + kubectl create secret generic private-key --from-file=snowflake= --namespace=flytesnacks-development + +3. Create a secret in the flyteagent's pod, this is for execution snowflake query in the agent pod. + +.. code-block:: bash + + ENCODED_VALUE=$(cat | base64) && kubectl patch secret flyteagent -n flyte --patch "{\"data\":{\"snowflake_private_key\":\"$ENCODED_VALUE\"}}" + Specify agent configuration ---------------------------- @@ -73,7 +82,7 @@ Specify agent configuration supportedTaskTypes: - snowflake -Ensure that the propeller has the correct service account for BigQuery. +Ensure that the propeller has the correct service account for Snowflake. Upgrade the Flyte Helm release ------------------------------ @@ -97,7 +106,6 @@ Upgrade the Flyte Helm release helm upgrade flyte/flyte-core -n --values values-override.yaml Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). For Snowflake agent on the Flyte cluster, see `Snowflake agent `_. From 17719e2d94f469a35b69fe5d11e388f33fbdb2a9 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Fri, 2 Aug 2024 14:24:36 +0800 Subject: [PATCH 03/23] [flytepropeller][compiler] Error Handling when Type is not found (#5612) * FlytePropeller Compiler Avoid Crash when Type not found Signed-off-by: Future-Outlier * Update pingsu's error message advices Signed-off-by: Future-Outlier Co-authored-by: pingsutw * fix lint Signed-off-by: Future-Outlier * Trigger CI Signed-off-by: Future-Outlier * Trigger CI Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier Co-authored-by: pingsutw --- .../validation/launch_plan_validator_test.go | 14 ++++--- .../pkg/manager/impl/validation/validation.go | 17 +++++++++ .../impl/validation/validation_test.go | 38 +++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/flyteadmin/pkg/manager/impl/validation/launch_plan_validator_test.go b/flyteadmin/pkg/manager/impl/validation/launch_plan_validator_test.go index 86bfc5c6b7..178c2b497b 100644 --- a/flyteadmin/pkg/manager/impl/validation/launch_plan_validator_test.go +++ b/flyteadmin/pkg/manager/impl/validation/launch_plan_validator_test.go @@ -13,6 +13,10 @@ import ( "github.com/flyteorg/flyte/flytestdlib/utils" ) +const ( + foo = "foo" +) + var lpApplicationConfig = testutils.GetApplicationConfigWithDefaultDomains() func getWorkflowInterface() *core.TypedInterface { @@ -344,7 +348,7 @@ func TestValidateSchedule_KickoffTimeArgPointsAtWrongType(t *testing.T) { request := testutils.GetLaunchPlanRequestWithDeprecatedCronSchedule("* * * * * *") inputMap := &core.ParameterMap{ Parameters: map[string]*core.Parameter{ - "foo": { + foo: { Var: &core.Variable{ Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}, }, @@ -354,7 +358,7 @@ func TestValidateSchedule_KickoffTimeArgPointsAtWrongType(t *testing.T) { }, }, } - request.Spec.EntityMetadata.Schedule.KickoffTimeInputArg = "foo" + request.Spec.EntityMetadata.Schedule.KickoffTimeInputArg = foo err := validateSchedule(request, inputMap) assert.NotNil(t, err) @@ -364,7 +368,7 @@ func TestValidateSchedule_NoRequired(t *testing.T) { request := testutils.GetLaunchPlanRequestWithDeprecatedCronSchedule("* * * * * *") inputMap := &core.ParameterMap{ Parameters: map[string]*core.Parameter{ - "foo": { + foo: { Var: &core.Variable{ Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}, }, @@ -383,7 +387,7 @@ func TestValidateSchedule_KickoffTimeBound(t *testing.T) { request := testutils.GetLaunchPlanRequestWithDeprecatedCronSchedule("* * * * * *") inputMap := &core.ParameterMap{ Parameters: map[string]*core.Parameter{ - "foo": { + foo: { Var: &core.Variable{ Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_DATETIME}}, }, @@ -393,7 +397,7 @@ func TestValidateSchedule_KickoffTimeBound(t *testing.T) { }, }, } - request.Spec.EntityMetadata.Schedule.KickoffTimeInputArg = "foo" + request.Spec.EntityMetadata.Schedule.KickoffTimeInputArg = foo err := validateSchedule(request, inputMap) assert.Nil(t, err) diff --git a/flyteadmin/pkg/manager/impl/validation/validation.go b/flyteadmin/pkg/manager/impl/validation/validation.go index 1958f25021..55c45db9bb 100644 --- a/flyteadmin/pkg/manager/impl/validation/validation.go +++ b/flyteadmin/pkg/manager/impl/validation/validation.go @@ -1,6 +1,7 @@ package validation import ( + "fmt" "net/url" "strconv" "strings" @@ -282,11 +283,27 @@ func validateParameterMap(inputMap *core.ParameterMap, fieldName string) error { defaultValue := defaultInput.GetDefault() if defaultValue != nil { inputType := validators.LiteralTypeForLiteral(defaultValue) + + if inputType == nil { + return errors.NewFlyteAdminErrorf(codes.InvalidArgument, + fmt.Sprintf( + "Flyte encountered an issue while determining\n"+ + "the type of the default value for Parameter '%s' in '%s'.\n"+ + "Registered type: [%s].\n"+ + "Flyte needs to support the latest FlyteIDL to support this type.\n"+ + "Suggested solution: Please update all of your Flyte images to the latest version and "+ + "try again.", + name, fieldName, defaultInput.GetVar().GetType().String(), + ), + ) + } + if !validators.AreTypesCastable(inputType, defaultInput.GetVar().GetType()) { return errors.NewFlyteAdminErrorf(codes.InvalidArgument, "Type mismatch for Parameter %s in %s has type %s, expected %s", name, fieldName, defaultInput.GetVar().GetType().String(), inputType.String()) } + if defaultInput.GetVar().GetType().GetSimple() == core.SimpleType_DATETIME { // Make datetime specific validations return ValidateDatetime(defaultValue) diff --git a/flyteadmin/pkg/manager/impl/validation/validation_test.go b/flyteadmin/pkg/manager/impl/validation/validation_test.go index a9fed38ee9..1aa0bc7bab 100644 --- a/flyteadmin/pkg/manager/impl/validation/validation_test.go +++ b/flyteadmin/pkg/manager/impl/validation/validation_test.go @@ -320,6 +320,44 @@ func TestValidateParameterMap(t *testing.T) { err := validateParameterMap(&exampleMap, "some text") assert.NoError(t, err) }) + t.Run("invalid because inputType is nil", func(t *testing.T) { + // Create a literal that will cause LiteralTypeForLiteral to return nil. + // For example, a scalar with no value. + unsupportedLiteral := &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{}, + }, + } + + name := "foo" + fieldName := "test_field_name" + exampleMap := core.ParameterMap{ + Parameters: map[string]*core.Parameter{ + name: { + Var: &core.Variable{ + // 1000 means an unsupported type + Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: 1000}}, + }, + Behavior: &core.Parameter_Default{ + Default: unsupportedLiteral, + }, + }, + }, + } + err := validateParameterMap(&exampleMap, fieldName) + assert.Error(t, err) + fmt.Println(err.Error()) + expectedErrMsg := fmt.Sprintf( + "Flyte encountered an issue while determining\n"+ + "the type of the default value for Parameter '%s' in '%s'.\n"+ + "Registered type: [%s].\n"+ + "Flyte needs to support the latest FlyteIDL to support this type.\n"+ + "Suggested solution: Please update all of your Flyte images to the latest version and "+ + "try again.", + name, fieldName, exampleMap.Parameters[name].GetVar().GetType().String(), + ) + assert.Equal(t, expectedErrMsg, err.Error()) + }) } func TestValidateToken(t *testing.T) { From 89fd084101875ee26fe8c5312d9e621669d2eeb8 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Thu, 1 Aug 2024 23:46:52 -0700 Subject: [PATCH 04/23] Fix nil pointer when task plugin load returns error (#5622) --- .../pkg/controller/nodes/task/handler.go | 7 ++++--- .../pkg/controller/nodes/task/handler_test.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/flytepropeller/pkg/controller/nodes/task/handler.go b/flytepropeller/pkg/controller/nodes/task/handler.go index d1595890d8..9ec47985c9 100644 --- a/flytepropeller/pkg/controller/nodes/task/handler.go +++ b/flytepropeller/pkg/controller/nodes/task/handler.go @@ -248,13 +248,14 @@ func (t *Handler) Setup(ctx context.Context, sCtx interfaces.SetupContext) error logger.Infof(ctx, "Loading Plugin [%s] ENABLED", p.ID) cp, err := pluginCore.LoadPlugin(ctx, sCtxFinal, p) + if err != nil { + return regErrors.Wrapf(err, "failed to load plugin - %s", p.ID) + } + if cp.GetID() == agent.ID { t.agentService.CorePlugin = cp } - if err != nil { - return regErrors.Wrapf(err, "failed to load plugin - %s", p.ID) - } // For every default plugin for a task type specified in flytepropeller config we validate that the plugin's // static definition includes that task type as something it is registered to handle. for _, tt := range p.RegisteredTaskTypes { diff --git a/flytepropeller/pkg/controller/nodes/task/handler_test.go b/flytepropeller/pkg/controller/nodes/task/handler_test.go index 4e6798cfef..31e1be9a7f 100644 --- a/flytepropeller/pkg/controller/nodes/task/handler_test.go +++ b/flytepropeller/pkg/controller/nodes/task/handler_test.go @@ -126,6 +126,8 @@ func Test_task_Setup(t *testing.T) { k8sPluginDefault := &pluginK8sMocks.Plugin{} k8sPluginDefault.OnGetProperties().Return(pluginK8s.PluginProperties{}) + loadErrorPluginType := "loadError" + corePluginEntry := pluginCore.PluginEntry{ ID: corePluginType, RegisteredTaskTypes: []pluginCore.TaskType{corePluginType}, @@ -154,6 +156,13 @@ func Test_task_Setup(t *testing.T) { RegisteredTaskTypes: []pluginCore.TaskType{k8sPluginDefaultType}, ResourceToWatch: &v1.Pod{}, } + loadErrorPluginEntry := pluginCore.PluginEntry{ + ID: loadErrorPluginType, + RegisteredTaskTypes: []pluginCore.TaskType{loadErrorPluginType}, + LoadPlugin: func(ctx context.Context, iCtx pluginCore.SetupContext) (pluginCore.Plugin, error) { + return nil, fmt.Errorf("test") + }, + } type wantFields struct { pluginIDs map[pluginCore.TaskType]string @@ -232,6 +241,15 @@ func Test_task_Setup(t *testing.T) { }, }, false}, + {"load-error", + testPluginRegistry{ + core: []pluginCore.PluginEntry{loadErrorPluginEntry}, + k8s: []pluginK8s.PluginEntry{}, + }, + []string{loadErrorPluginType}, + map[string]string{corePluginType: loadErrorPluginType}, + wantFields{}, + true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 89a70fb1ba33beab06784cbeb0c7ec8ff08a1011 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Thu, 1 Aug 2024 23:48:45 -0700 Subject: [PATCH 05/23] Log stack trace when refresh cache sync recovers from panic (#5623) --- flytestdlib/cache/auto_refresh.go | 5 ++-- flytestdlib/cache/auto_refresh_test.go | 37 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/flytestdlib/cache/auto_refresh.go b/flytestdlib/cache/auto_refresh.go index bb23ef9369..8218e577a8 100644 --- a/flytestdlib/cache/auto_refresh.go +++ b/flytestdlib/cache/auto_refresh.go @@ -3,6 +3,7 @@ package cache import ( "context" "fmt" + "runtime/debug" "sync" "time" @@ -290,9 +291,9 @@ func (w *autoRefresh) sync(ctx context.Context) (err error) { } if err, isErr = rVal.(error); isErr { - err = fmt.Errorf("worker panic'd and is shutting down. Error: %w", err) + err = fmt.Errorf("worker panic'd and is shutting down. Error: %w with Stack: %v", err, string(debug.Stack())) } else { - err = fmt.Errorf("worker panic'd and is shutting down. Panic value: %v", rVal) + err = fmt.Errorf("worker panic'd and is shutting down. Panic value: %v with Stack: %v", rVal, string(debug.Stack())) } logger.Error(ctx, err) diff --git a/flytestdlib/cache/auto_refresh_test.go b/flytestdlib/cache/auto_refresh_test.go index e798300f5d..5e1c49777e 100644 --- a/flytestdlib/cache/auto_refresh_test.go +++ b/flytestdlib/cache/auto_refresh_test.go @@ -64,6 +64,15 @@ func syncTerminalItem(_ context.Context, batch Batch) ([]ItemSyncResponse, error panic("This should never be called") } +type panickingSyncer struct { + callCount atomic.Int32 +} + +func (p *panickingSyncer) sync(_ context.Context, _ Batch) ([]ItemSyncResponse, error) { + p.callCount.Inc() + panic("testing") +} + func TestCacheFour(t *testing.T) { testResyncPeriod := 10 * time.Millisecond rateLimiter := workqueue.DefaultControllerRateLimiter() @@ -172,6 +181,34 @@ func TestCacheFour(t *testing.T) { cancel() }) + + t.Run("Test panic on sync and shutdown", func(t *testing.T) { + syncer := &panickingSyncer{} + cache, err := NewAutoRefreshCache("fake3", syncer.sync, rateLimiter, testResyncPeriod, 10, 2, promutils.NewTestScope()) + assert.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + assert.NoError(t, cache.Start(ctx)) + + itemID := "dummy_id" + _, err = cache.GetOrCreate(itemID, fakeCacheItem{ + val: 0, + }) + assert.NoError(t, err) + + // wait for all workers to run + assert.Eventually(t, func() bool { + return syncer.callCount.Load() == int32(10) + }, 5*time.Second, time.Millisecond) + + // wait some more time + time.Sleep(500 * time.Millisecond) + + // all workers should have shut down. + assert.Equal(t, int32(10), syncer.callCount.Load()) + + cancel() + }) } func TestQueueBuildUp(t *testing.T) { From 7e16ff402d0b6e46434ccb8ada601feeb3b297f1 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Fri, 2 Aug 2024 17:48:36 +0800 Subject: [PATCH 06/23] use private-key (#5626) --- docs/deployment/agents/snowflake.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/agents/snowflake.rst b/docs/deployment/agents/snowflake.rst index a689c748bf..d6ee74125b 100644 --- a/docs/deployment/agents/snowflake.rst +++ b/docs/deployment/agents/snowflake.rst @@ -6,7 +6,7 @@ Snowflake agent This guide provides an overview of how to set up the Snowflake agent in your Flyte deployment. 1. Set up the key pair authentication in Snowflake. For more details, see the `Snowflake key-pair authentication and key-pair rotation guide `__. -2. Create a secret with the group "private_key" and the key "snowflake". +2. Create a secret with the group "private-key" and the key "snowflake". This is hardcoded in the flytekit sdk, since we can't know the group and key name in advance. This is for permission to upload and download data with structured dataset in python task pod. From b6bc90254849d424126b30f5e3e3d4449e7fca43 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Fri, 2 Aug 2024 17:55:47 +0800 Subject: [PATCH 07/23] Explain how Agent Secret Works (#5625) * first version Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier --- docs/flyte_agents/how_secret_works_in_agent.md | 17 +++++++++++++++++ docs/flyte_agents/index.md | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 docs/flyte_agents/how_secret_works_in_agent.md diff --git a/docs/flyte_agents/how_secret_works_in_agent.md b/docs/flyte_agents/how_secret_works_in_agent.md new file mode 100644 index 0000000000..7abada46ac --- /dev/null +++ b/docs/flyte_agents/how_secret_works_in_agent.md @@ -0,0 +1,17 @@ +--- +jupytext: + formats: md:myst + text_representation: + extension: .md + format_name: myst +--- + +(how_secret_works_in_agent)= +# How Secret Works in Agent + +In Flyte agent's deployment, we mount secrets in Kubernetes with the namespace `flyte` and the name `flyteagent`. +If you want to add secrets for agents, you can use the following command: + +```bash +SECRET_VALUE=$( | base64) && kubectl patch secret flyteagent -n flyte --patch "{\"data\":{\"your_agent_secret_name\":\"$SECRET_VALUE\"}}" +``` diff --git a/docs/flyte_agents/index.md b/docs/flyte_agents/index.md index e7d627a670..a32200cde6 100644 --- a/docs/flyte_agents/index.md +++ b/docs/flyte_agents/index.md @@ -36,6 +36,8 @@ You can create different agent services that host different agents, e.g., a prod - Once you have tested your new agent in a local development cluster and want to use it in production, you should test it in the Flyte sandbox. * - {doc}`Implementing the agent metadata service ` - If you want to develop an agent server in a language other than Python (e.g., Rust or Java), you must implement the agent metadata service in your agent server. +* - {doc}`How secret works in agent ` + - Explain how secret works in your agent server. ``` ```{toctree} @@ -48,4 +50,5 @@ developing_agents testing_agents_in_a_local_development_cluster deploying_agents_to_the_flyte_sandbox implementing_the_agent_metadata_service +how_secret_works_in_agent ``` From 1124ea94055e05facbb09da720bcf670f7b1849a Mon Sep 17 00:00:00 2001 From: ddl-rliu <140021987+ddl-rliu@users.noreply.github.com> Date: Fri, 2 Aug 2024 03:03:48 -0700 Subject: [PATCH 08/23] Fix typo in execution manager (#5619) Signed-off-by: ddl-rliu <140021987+ddl-rliu@users.noreply.github.com> --- flyteadmin/pkg/manager/impl/execution_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyteadmin/pkg/manager/impl/execution_manager.go b/flyteadmin/pkg/manager/impl/execution_manager.go index 337301977e..13521cedbb 100644 --- a/flyteadmin/pkg/manager/impl/execution_manager.go +++ b/flyteadmin/pkg/manager/impl/execution_manager.go @@ -1719,7 +1719,7 @@ func (m *ExecutionManager) TerminateExecution( } if common.IsExecutionTerminal(core.WorkflowExecution_Phase(core.WorkflowExecution_Phase_value[executionModel.Phase])) { - return nil, errors.NewAlreadyInTerminalStateError(ctx, "Cannot abort an already terminate workflow execution", executionModel.Phase) + return nil, errors.NewAlreadyInTerminalStateError(ctx, "Cannot abort an already terminated workflow execution", executionModel.Phase) } err = transformers.SetExecutionAborting(&executionModel, request.Cause, getUser(ctx)) From 0a441a97c990024c0f6e07a39939fb3aa1f4b7f0 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 2 Aug 2024 12:53:33 -0700 Subject: [PATCH 09/23] Amend Admin to use grpc message size (#5628) * add send arg Signed-off-by: Yee Hing Tong * Add acction to remove cache in gh runner Signed-off-by: Eduardo Apolinario * Use correct checked out path Signed-off-by: Eduardo Apolinario * Path in strings Signed-off-by: Eduardo Apolinario * Checkout repo in root Signed-off-by: Eduardo Apolinario * Use the correct path to new action Signed-off-by: Eduardo Apolinario * Do not use gh var in path to clear-action-cache Signed-off-by: Eduardo Apolinario * Remove wrong invocation of clear-action-cache Signed-off-by: Eduardo Apolinario * GITHUB_WORKSPACE is implicit in the checkout action Signed-off-by: Eduardo Apolinario * Refer to local `flyte` directory Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Yee Hing Tong Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/actions/clear-action-cache/action.yml | 11 +++++++++++ .github/workflows/tests.yml | 6 ++++-- flyteadmin/pkg/server/service.go | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 .github/actions/clear-action-cache/action.yml diff --git a/.github/actions/clear-action-cache/action.yml b/.github/actions/clear-action-cache/action.yml new file mode 100644 index 0000000000..a29347b61c --- /dev/null +++ b/.github/actions/clear-action-cache/action.yml @@ -0,0 +1,11 @@ +name: 'Clear action cache' +description: 'As suggested by GitHub to prevent low disk space: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173' +runs: + using: 'composite' + steps: + - shell: bash + run: | + rm -rf /usr/share/dotnet + rm -rf /opt/ghc + rm -rf "/usr/local/share/boost" + rm -rf "$AGENT_TOOLSDIRECTORY" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cbce9cd054..1d69466464 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,12 +30,14 @@ jobs: - name: Fetch flyte code uses: actions/checkout@v4 with: - path: "${{ github.workspace }}/flyte" + path: flyte + - name: 'Clear action cache' + uses: ./flyte/.github/actions/clear-action-cache - name: Fetch flytekit code uses: actions/checkout@v4 with: repository: flyteorg/flytekit - path: "${{ github.workspace }}/flytekit" + path: flytekit - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true diff --git a/flyteadmin/pkg/server/service.go b/flyteadmin/pkg/server/service.go index bb09f9f615..0a7371ef68 100644 --- a/flyteadmin/pkg/server/service.go +++ b/flyteadmin/pkg/server/service.go @@ -140,7 +140,7 @@ func newGRPCServer(ctx context.Context, pluginRegistry *plugins.Registry, cfg *c grpc.UnaryInterceptor(chainedUnaryInterceptors), } if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { - serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes)) + serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes), grpc.MaxSendMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes)) } serverOpts = append(serverOpts, opts...) grpcServer := grpc.NewServer(serverOpts...) From 4014bbdfba50293714fe5aa531093fb0987b0e91 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 6 Aug 2024 12:50:06 +0800 Subject: [PATCH 10/23] document the process of setting ttl for a ray cluster (#5636) Signed-off-by: Kevin Su --- docs/deployment/plugins/k8s/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/deployment/plugins/k8s/index.rst b/docs/deployment/plugins/k8s/index.rst index a46ec23815..64fbb41136 100644 --- a/docs/deployment/plugins/k8s/index.rst +++ b/docs/deployment/plugins/k8s/index.rst @@ -272,6 +272,10 @@ Specify plugin configuration - container: container - container_array: k8s-array - ray: ray + plugins: + ray: + // Shutdown Ray cluster after 1 hour of inactivity + ttlSecondsAfterFinished: 3600 .. group-tab:: Flyte core @@ -294,6 +298,10 @@ Specify plugin configuration sidecar: sidecar container_array: k8s-array ray: ray + plugins: + ray: + // Shutdown Ray cluster after 1 hour of inactivity + ttlSecondsAfterFinished: 3600 .. group-tab:: Spark From 8156b1cf8b99831b5eece8b32cf3ec627a709eab Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Wed, 7 Aug 2024 16:24:11 -0700 Subject: [PATCH 11/23] Add CustomHeaderMatcher to pass additional headers (#5563) Signed-off-by: Andrew Dye --- flyteadmin/auth/handlers.go | 20 ++++++++++++++++++++ flyteadmin/pkg/server/service.go | 3 +++ flyteadmin/plugins/registry.go | 9 +++++---- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/flyteadmin/auth/handlers.go b/flyteadmin/auth/handlers.go index a6220db6e3..b839cf26d0 100644 --- a/flyteadmin/auth/handlers.go +++ b/flyteadmin/auth/handlers.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" "net/http" + "net/textproto" "net/url" "strings" "time" "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "golang.org/x/oauth2" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -22,6 +24,7 @@ import ( "github.com/flyteorg/flyte/flyteadmin/pkg/common" "github.com/flyteorg/flyte/flyteadmin/plugins" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service" + "github.com/flyteorg/flyte/flytestdlib/contextutils" "github.com/flyteorg/flyte/flytestdlib/errors" "github.com/flyteorg/flyte/flytestdlib/logger" ) @@ -32,6 +35,8 @@ const ( FromHTTPVal = "true" ) +var XRequestID = textproto.CanonicalMIMEHeaderKey(contextutils.RequestIDKey.String()) + type PreRedirectHookError struct { Message string Code int @@ -533,3 +538,18 @@ func GetUserInfoForwardResponseHandler() UserInfoForwardResponseHandler { return nil } } + +func GetCustomHeaderMatcher(pluginRegistry *plugins.Registry) runtime.HeaderMatcherFunc { + if fn := plugins.Get[runtime.HeaderMatcherFunc](pluginRegistry, plugins.PluginIDCustomerHeaderMatcher); fn != nil { + return fn + } + return func(key string) (string, bool) { + canonicalKey := textproto.CanonicalMIMEHeaderKey(key) + switch canonicalKey { + case XRequestID: + return canonicalKey, true + default: + return runtime.DefaultHeaderMatcher(key) + } + } +} diff --git a/flyteadmin/pkg/server/service.go b/flyteadmin/pkg/server/service.go index 0a7371ef68..587ea86e3b 100644 --- a/flyteadmin/pkg/server/service.go +++ b/flyteadmin/pkg/server/service.go @@ -240,6 +240,9 @@ func newHTTPServer(ctx context.Context, pluginRegistry *plugins.Registry, cfg *c // This option sets subject in the user info response gwmuxOptions = append(gwmuxOptions, runtime.WithForwardResponseOption(auth.GetUserInfoForwardResponseHandler())) + // Use custom header matcher to allow additional headers to be passed through + gwmuxOptions = append(gwmuxOptions, runtime.WithIncomingHeaderMatcher(auth.GetCustomHeaderMatcher(pluginRegistry))) + if cfg.Security.UseAuth { // Add HTTP handlers for OIDC endpoints auth.RegisterHandlers(ctx, mux, authCtx, pluginRegistry) diff --git a/flyteadmin/plugins/registry.go b/flyteadmin/plugins/registry.go index 92644b1367..a89a8dfeae 100644 --- a/flyteadmin/plugins/registry.go +++ b/flyteadmin/plugins/registry.go @@ -9,12 +9,13 @@ import ( type PluginID = string const ( - PluginIDWorkflowExecutor PluginID = "WorkflowExecutor" + PluginIDAdditionalGRPCService PluginID = "AdditionalGRPCService" + PluginIDCustomerHeaderMatcher PluginID = "CustomerHeaderMatcher" PluginIDDataProxy PluginID = "DataProxy" - PluginIDUnaryServiceMiddleware PluginID = "UnaryServiceMiddleware" - PluginIDPreRedirectHook PluginID = "PreRedirectHook" PluginIDLogoutHook PluginID = "LogoutHook" - PluginIDAdditionalGRPCService PluginID = "AdditionalGRPCService" + PluginIDPreRedirectHook PluginID = "PreRedirectHook" + PluginIDUnaryServiceMiddleware PluginID = "UnaryServiceMiddleware" + PluginIDWorkflowExecutor PluginID = "WorkflowExecutor" ) type AtomicRegistry struct { From 43c9d94c2898dd2cf88117ef63a0fd217d437ae0 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:57:03 -0700 Subject: [PATCH 12/23] Turn flyteidl and flytectl releases into manual gh workflows (#5635) * Make flyteidl releases go through a manual gh workflow Signed-off-by: Eduardo Apolinario * Make flytectl releases go through a manual gh workflow Signed-off-by: Eduardo Apolinario * Rewrite the documentation for `version` and clarify wording in RELEASE.md Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/workflows/create_release.yml | 1 - .github/workflows/flytectl-release.yml | 27 +++++++++++++++++++--- .github/workflows/flyteidl-release.yml | 31 ++++++++++++++++++++++---- flytectl/RELEASE.md | 2 +- flyteidl/RELEASE.md | 4 ++++ 5 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 flyteidl/RELEASE.md diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 1db5986925..e00c09f2d7 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -28,7 +28,6 @@ jobs: "datacatalog", "flyteadmin", "flytecopilot", - "flyteidl", "flyteplugins", "flytepropeller", "flytestdlib", diff --git a/.github/workflows/flytectl-release.yml b/.github/workflows/flytectl-release.yml index 2bfa6f28eb..2aba67dbe9 100644 --- a/.github/workflows/flytectl-release.yml +++ b/.github/workflows/flytectl-release.yml @@ -1,13 +1,34 @@ name: Flytectl release on: - push: - tags: - - flytectl/v*.*.* + workflow_dispatch: + inputs: + version: + description: "version. Do *not* use the `flytectl/` prefix, e.g. `flytectl/v1.2.3`, instead use only `v1.2.3` (including the `v`)" + required: true jobs: + push-flytectl-tag: + name: Push git tag containing the `flyteidl/` prefix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: '0' + - uses: actions/github-script@v6 + with: + github-token: ${{ secrets.FLYTE_BOT_PAT }} + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/flytectl/${{ github.event.inputs.version }}`, + sha: context.sha + }) release: name: Goreleaser + needs: + - push-flytectl-tag runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/flyteidl-release.yml b/.github/workflows/flyteidl-release.yml index 94c13645b2..2b19f7942d 100644 --- a/.github/workflows/flyteidl-release.yml +++ b/.github/workflows/flyteidl-release.yml @@ -1,12 +1,33 @@ -name: Upload flyteidl to PyPI and npm +name: Release flyteidl on: - push: - tags: - - flyteidl/v*.*.* + workflow_dispatch: + inputs: + version: + description: "version. Do *not* use the `flyteidl/` prefix, e.g. `flyteidl/v1.2.3`, instead use only `v1.2.3` (including the `v`)" + required: true jobs: + push-flyteidl-tag: + name: Push git tag containing the `flyteidl/` prefix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: '0' + - uses: actions/github-script@v6 + with: + github-token: ${{ secrets.FLYTE_BOT_PAT }} + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/flyteidl/${{ github.event.inputs.version }}`, + sha: context.sha + }) deploy-to-pypi: + needs: + - push-flyteidl-tag runs-on: ubuntu-latest defaults: run: @@ -29,6 +50,8 @@ jobs: python -m build twine upload dist/* deploy-to-npm: + needs: + - push-flyteidl-tag runs-on: ubuntu-latest defaults: run: diff --git a/flytectl/RELEASE.md b/flytectl/RELEASE.md index 646f7465a6..414aa24199 100644 --- a/flytectl/RELEASE.md +++ b/flytectl/RELEASE.md @@ -2,4 +2,4 @@ Flytectl releases map to git tags with the prefix `flytectl/` followed by a semver string, e.g. [flytectl/v0.9.0](https://github.com/flyteorg/flyte/releases/tag/flytectl%2Fv0.9.0). -To release a new version of flytectl push a new git tag in the format described above. This will kick off a <[github workflow](https://github.com/flyteorg/flyte/blob/master/.github/workflows/flytectl-release.yml) responsible for releasing this new version. Note how the git tag has to be formatted a certain way for the workflow to run. +To release a new version of flytectl run the <[github workflow](https://github.com/flyteorg/flyte/blob/master/.github/workflows/flytectl-release.yml), which is responsible for releasing this new version. Remember to use valid semver versions, including adding the prefix `v`, e.g. `v1.2.3`. diff --git a/flyteidl/RELEASE.md b/flyteidl/RELEASE.md new file mode 100644 index 0000000000..eaaa4d51f6 --- /dev/null +++ b/flyteidl/RELEASE.md @@ -0,0 +1,4 @@ +# Release Process + +To release a new version of flyteidl run the <[github workflow](https://github.com/flyteorg/flyte/blob/master/.github/workflows/flyteidl-release.yml), which is responsible for releasing this new version. Remember to use valid semver versions, including adding the prefix `v`, e.g. `v1.2.3`. + From 91d14b79b9cfc5d1d303126461a5c5e41e8dd4e2 Mon Sep 17 00:00:00 2001 From: Christina <156356273+cratiu222@users.noreply.github.com> Date: Thu, 8 Aug 2024 22:42:44 +0300 Subject: [PATCH 13/23] docs: fix typo (#5643) * fix CHANGELOG-v0.2.0.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> * fix CHANGELOG-v1.0.2-b1.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> * fix CHANGELOG-v1.1.0.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> * fix CHANGELOG-v1.3.0.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> --------- Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> --- CHANGELOG/CHANGELOG-v0.2.0.md | 2 +- CHANGELOG/CHANGELOG-v1.0.2-b1.md | 2 +- CHANGELOG/CHANGELOG-v1.1.0.md | 2 +- CHANGELOG/CHANGELOG-v1.3.0.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG/CHANGELOG-v0.2.0.md b/CHANGELOG/CHANGELOG-v0.2.0.md index 4c16bb0742..d3c85ff0fb 100644 --- a/CHANGELOG/CHANGELOG-v0.2.0.md +++ b/CHANGELOG/CHANGELOG-v0.2.0.md @@ -16,7 +16,7 @@ - RawOutputDirectories created in FlytePropeller - Improve visibility and observability - User/System error differentiation -- Optional interruptible tasks (lets use spot instances, reduce cost) +- Optional interruptible tasks (let's use spot instances, to reduce cost) - Caps on queue time for workflows - Multi cluster improvements - Visibility into execution cluster for the execution diff --git a/CHANGELOG/CHANGELOG-v1.0.2-b1.md b/CHANGELOG/CHANGELOG-v1.0.2-b1.md index 3ade2ab25e..ef216a0c25 100644 --- a/CHANGELOG/CHANGELOG-v1.0.2-b1.md +++ b/CHANGELOG/CHANGELOG-v1.0.2-b1.md @@ -5,7 +5,7 @@ 1. [Bugfix](https://github.com/flyteorg/flyte/issues/2444) With GRPC v1.46.0 non-ascii chars are not permitted in grpc metadata 1. [Housekeeping](https://github.com/flyteorg/flyte/issues/1698) Configure grpc_health_prob in admin 1. [Feature](https://github.com/flyteorg/flyte/issues/2329) In Flytectl use Launchplan with latest version for scheduled workflows -1. [Bugfix](https://github.com/flyteorg/flyte/issues/2262) Pods started before InjectFinalizer is disabled are never deleted +1. [Bugfix](https://github.com/flyteorg/flyte/issues/2262) Pods started before InjectFinalizer was disabled are never deleted 1. [Housekeeping](https://github.com/flyteorg/flyte/issues/2504) Checksum grpc_health_probe 1. [Feature](https://github.com/flyteorg/flyte/issues/2284) Allow to choose Spot Instances at workflow start time 1. [Feature](https://github.com/flyteorg/flyte/pull/2439) Use the same pod annotation formatting in syncresources cronjob diff --git a/CHANGELOG/CHANGELOG-v1.1.0.md b/CHANGELOG/CHANGELOG-v1.1.0.md index 9236270965..1cbad29584 100644 --- a/CHANGELOG/CHANGELOG-v1.1.0.md +++ b/CHANGELOG/CHANGELOG-v1.1.0.md @@ -17,7 +17,7 @@ Support for [Optional types](https://github.com/flyteorg/flyte/issues/2426). Wit ### Bug Fixes * [Propeller](https://github.com/flyteorg/flyte/issues/2298) calling finalize rather than abort -* [Propeller](https://github.com/flyteorg/flyte/issues/2404) correctly identify error when requesting a launch plan that does not exist. +* [Propeller](https://github.com/flyteorg/flyte/issues/2404) correctly identifies an error when requesting a launch plan that does not exist. * Better handle [execution CRDs](https://github.com/flyteorg/flyte/issues/2275) that don't exist in Admin. * [Fix panic](https://github.com/flyteorg/flyte/issues/2597) when creating additional label options. * Check [validity](https://github.com/flyteorg/flyte/issues/2601) of notifications. diff --git a/CHANGELOG/CHANGELOG-v1.3.0.md b/CHANGELOG/CHANGELOG-v1.3.0.md index c15224d4a6..0591f676ef 100644 --- a/CHANGELOG/CHANGELOG-v1.3.0.md +++ b/CHANGELOG/CHANGELOG-v1.3.0.md @@ -7,7 +7,7 @@ The main features of this 1.3 release are * Signaling/gate node support (human in the loop tasks) * User documentation support (backend and flytekit only, limited types) -The latter two are pending some work in Flyte console, they will be piped through fully by the end of Q1. Support for setting and approving gate nodes is supported in `FlyteRemote` however, though only a limited set of types can be passed in. +The latter two are pending some work in Flyte console, they will be piped through fully by the end of Q1. Support for setting and approving gate nodes is supported in `FlyteRemote` however, only a limited set of types can be passed in. ## Notes There are a couple things to point out with this release. From 1de1b500ca4dc8086e15efc45f7225562b54127d Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 8 Aug 2024 15:53:22 -0400 Subject: [PATCH 14/23] Use enable_deck=True in docs (#5645) --- docs/core_use_cases/analytics.md | 2 +- docs/core_use_cases/machine_learning.md | 2 +- .../visualizing_task_input_and_output.md | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/core_use_cases/analytics.md b/docs/core_use_cases/analytics.md index 886b75618d..71b5530c03 100644 --- a/docs/core_use_cases/analytics.md +++ b/docs/core_use_cases/analytics.md @@ -48,7 +48,7 @@ of the map. In this case, we normalize the `people_vaccinated` by the `population` count of each country: ```{code-cell} ipython3 -@task(disable_deck=False) +@task(enable_deck=True) def plot(df: pd.DataFrame): """Render a Choropleth map.""" df["text"] = df["location"] + "
" + "Last updated on: " + df["date"] diff --git a/docs/core_use_cases/machine_learning.md b/docs/core_use_cases/machine_learning.md index 489b8b05f9..6368b0aa54 100644 --- a/docs/core_use_cases/machine_learning.md +++ b/docs/core_use_cases/machine_learning.md @@ -112,7 +112,7 @@ There are many ways to extend your workloads: {ref}`Kubeflow Pytorch` and {doc}`more <_tags/DistributedComputing>` to do distributed training. * - **🔎 Experiment Tracking** - Auto-capture training logs with the {py:func}`~flytekitplugins.mlflow.mlflow_autolog` - decorator, which can be viewed as Flyte Decks with `@task(disable_decks=False)`. + decorator, which can be viewed as Flyte Decks with `@task(enable_deck=True)`. * - **⏩ Inference Acceleration** - Serialize your models in ONNX format using the {ref}`ONNX plugin `, which supports ScikitLearn, TensorFlow, and PyTorch. diff --git a/docs/flyte_fundamentals/visualizing_task_input_and_output.md b/docs/flyte_fundamentals/visualizing_task_input_and_output.md index 487d1627c9..0390d6cf44 100644 --- a/docs/flyte_fundamentals/visualizing_task_input_and_output.md +++ b/docs/flyte_fundamentals/visualizing_task_input_and_output.md @@ -22,14 +22,14 @@ how to generate an HTML report from some Python object. ## Enabling Flyte decks -To enable Flyte decks, simply set `disable_deck=False` in the `@task` decorator: +To enable Flyte decks, simply set `enable_deck=True` in the `@task` decorator: ```{code-cell} ipython3 import pandas as pd from flytekit import task, workflow -@task(disable_deck=False) +@task(enable_deck=True) def iris_data() -> pd.DataFrame: ... ``` @@ -51,7 +51,7 @@ from typing import Optional from flytekit import task, workflow -@task(disable_deck=False) +@task(enable_deck=True) def iris_data( sample_frac: Optional[float] = None, random_state: Optional[int] = None, @@ -168,7 +168,7 @@ function. In the following example, we extend the `iris_data` task with: import flytekit from flytekitplugins.deck.renderer import MarkdownRenderer, BoxRenderer -@task(disable_deck=False) +@task(enable_deck=True) def iris_data( sample_frac: Optional[float] = None, random_state: Optional[int] = None, @@ -220,7 +220,7 @@ except ImportError: from typing_extensions import Annotated -@task(disable_deck=False) +@task(enable_deck=True) def iris_data( sample_frac: Optional[float] = None, random_state: Optional[int] = None, From 52322d021de0ece8362244062abc029ce9337366 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:44:11 -0700 Subject: [PATCH 15/23] Fix flyteidl release checkout all tags (#5646) * Fetch all tags in flyteidl-release.yml Signed-off-by: Eduardo Apolinario * Fix sed expression for npm job Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/workflows/flyteidl-release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flyteidl-release.yml b/.github/workflows/flyteidl-release.yml index 2b19f7942d..c895beba4b 100644 --- a/.github/workflows/flyteidl-release.yml +++ b/.github/workflows/flyteidl-release.yml @@ -34,6 +34,8 @@ jobs: working-directory: flyteidl steps: - uses: actions/checkout@v4 + with: + fetch-depth: '0' - name: Set up Python uses: actions/setup-python@v1 with: @@ -64,8 +66,8 @@ jobs: registry-url: "https://registry.npmjs.org" - name: Set version in npm package run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + # v1.2.3 get 1.2.3 + VERSION=$(echo ${{ inputs.version }} | sed 's#.*v##') VERSION=$VERSION make update_npmversion shell: bash - run: | From 96bbf7ed5331f7d629763a3721e90a1e35215da9 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:57:30 -0700 Subject: [PATCH 16/23] Install pyarrow in sandbox functional tests (#5647) Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/workflows/single-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 23d438b322..d4cb79f4d5 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -175,7 +175,7 @@ jobs: run: | python -m pip install --upgrade pip pip install uv - uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0" + uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0" pyarrow uv pip freeze - name: Checkout flytesnacks uses: actions/checkout@v4 From 5be45452d730f16edd9b5e33a3ee9528bff21b94 Mon Sep 17 00:00:00 2001 From: desihsu <43691987+desihsu@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:19:30 -0700 Subject: [PATCH 17/23] docs: add documentation for configuring notifications in GCP (#5545) * update Signed-off-by: Desi Hsu * dco Signed-off-by: Desi Hsu * dco Signed-off-by: Desi Hsu * typo Signed-off-by: Desi Hsu --------- Signed-off-by: Desi Hsu --- .../configuration/notifications.rst | 81 +++++++++++++++++-- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/docs/deployment/configuration/notifications.rst b/docs/deployment/configuration/notifications.rst index 2e4a77ac53..fecad482fe 100644 --- a/docs/deployment/configuration/notifications.rst +++ b/docs/deployment/configuration/notifications.rst @@ -1,7 +1,8 @@ .. _deployment-configuration-notifications: +############# Notifications -------------- +############# .. tags:: Infrastructure, Advanced @@ -62,10 +63,10 @@ The ``notifications`` top-level portion of the FlyteAdmin config specifies how t As with schedules, the notifications handling is composed of two parts. One handles enqueuing notifications asynchronously and the second part handles processing pending notifications and actually firing off emails and alerts. -This is only supported for Flyte instances running on AWS. +This is only supported for Flyte instances running on AWS or GCP. -Config -======= +AWS Config +========== To publish notifications, you'll need to set up an `SNS topic `_. @@ -80,9 +81,7 @@ Let's look at the following config section and explain what each value represent .. code-block:: yaml notifications: - # Because AWS is the only cloud back-end supported for executing scheduled - # workflows in this case, only ``"aws"`` is a valid value. By default, the - #no-op executor is used. + # By default, the no-op executor is used. type: "aws" # This specifies which region AWS clients will use when creating SNS and SQS clients. @@ -126,10 +125,76 @@ into `code `__. .. rli:: https://raw.githubusercontent.com/flyteorg/flyteadmin/master/flyteadmin_config.yaml :caption: flyteadmin/flyteadmin_config.yaml :lines: 91-105 + +GCP Config +========== + +You'll need to set up a `Pub/Sub topic `__ to publish notifications to, +and a `Pub/Sub subscriber `__ to consume from that topic +and process notifications. The GCP service account used by FlyteAdmin must also have Pub/Sub publish and subscribe permissions. + +Email service +------------- + +In order to actually publish notifications, you'll need an account with an external email service which will be +used to send notification emails and alerts using email APIs. + +Currently, `SendGrid `__ is the only supported external email service, +and you will need to have a verified SendGrid sender. Create a SendGrid API key with ``Mail Send`` permissions +and save it to a file ``key``. + +Create a K8s secret in FlyteAdmin's cluster with that file: + +.. prompt:: bash $ + + kubectl create secret generic -n flyte --from-file key sendgrid-key + +Mount the secret by adding the following to the ``flyte-core`` values YAML: + +.. code-block:: yaml + + flyteadmin: + additionalVolumes: + - name: sendgrid-key + secret: + secretName: sendgrid-key + items: + - key: key + path: key + additionalVolumeMounts: + - name: sendgrid-key + mountPath: /sendgrid + +Config +------ + +In the ``flyte-core`` values YAML, the top-level ``notifications`` config should be +placed under ``workflow_notifications``. + +.. code-block:: yaml + + workflow_notifications: + enabled: true + config: + notifications: + type: gcp + gcp: + projectId: "{{ YOUR PROJECT ID }}" + publisher: + topicName: "{{ YOUR PUB/SUB TOPIC NAME }}" + processor: + queueName: "{{ YOUR PUB/SUB SUBSCRIBER NAME }}" + emailer: + emailServerConfig: + serviceName: sendgrid + apiKeyFilePath: /sendgrid/key + subject: "Flyte execution \"{{ name }}\" has {{ phase }} in \"{{ project }}\"." + sender: "{{ YOUR SENDGRID SENDER EMAIL }}" + body: View details at https://{{ YOUR FLYTE HOST }}/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }} From a13a63dfd24470f566b11c7c107cc9655d3ec3c0 Mon Sep 17 00:00:00 2001 From: ShengYu Date: Mon, 12 Aug 2024 14:12:02 +0800 Subject: [PATCH 18/23] Correct "sucessfile" to "successfile" (#5652) --- flytecopilot/cmd/sidecar_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytecopilot/cmd/sidecar_test.go b/flytecopilot/cmd/sidecar_test.go index 6d261e2c48..a7cc1c964a 100644 --- a/flytecopilot/cmd/sidecar_test.go +++ b/flytecopilot/cmd/sidecar_test.go @@ -90,7 +90,7 @@ func TestUploadOptions_Upload(t *testing.T) { assert.NoError(t, ioutil.WriteFile(success, []byte("done"), os.ModePerm)) ok, err := containerwatcher.FileExists(success) assert.NoError(t, err) - assert.True(t, ok, "sucessfile not created") + assert.True(t, ok, "successfile not created") assert.NoError(t, uopts.Sidecar(ctx)) v, err := store.Head(ctx, "/output/errors.pb") assert.NoError(t, err) From 9cfb4e3c2b884f82dd938f3f33a74439170aa14b Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Mon, 12 Aug 2024 18:10:16 +0200 Subject: [PATCH 19/23] Fix ordering for custom template values in cluster resource controller (#5648) Signed-off-by: Katrina Rogan --- flyteadmin/pkg/clusterresource/controller.go | 5 ++- .../pkg/clusterresource/controller_test.go | 32 +++++++++++++++++++ .../imagepullsecrets_templatized.yaml | 7 ++++ 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 flyteadmin/pkg/clusterresource/testdata/imagepullsecrets_templatized.yaml diff --git a/flyteadmin/pkg/clusterresource/controller.go b/flyteadmin/pkg/clusterresource/controller.go index daad2600e8..6ea1731909 100644 --- a/flyteadmin/pkg/clusterresource/controller.go +++ b/flyteadmin/pkg/clusterresource/controller.go @@ -485,14 +485,13 @@ func (c *controller) createResourceFromTemplate(ctx context.Context, templateDir templateValues[fmt.Sprintf(templateVariableFormat, domainVariable)] = domain.Id var k8sManifest = string(template) - for templateKey, templateValue := range templateValues { + for templateKey, templateValue := range customTemplateValues { k8sManifest = strings.Replace(k8sManifest, templateKey, templateValue, replaceAllInstancesOfString) } // Replace remaining template variables from domain specific defaults. - for templateKey, templateValue := range customTemplateValues { + for templateKey, templateValue := range templateValues { k8sManifest = strings.Replace(k8sManifest, templateKey, templateValue, replaceAllInstancesOfString) } - return k8sManifest, nil } diff --git a/flyteadmin/pkg/clusterresource/controller_test.go b/flyteadmin/pkg/clusterresource/controller_test.go index dc3239cdc2..f6a966d6ef 100644 --- a/flyteadmin/pkg/clusterresource/controller_test.go +++ b/flyteadmin/pkg/clusterresource/controller_test.go @@ -293,6 +293,38 @@ kind: IAMServiceAccount metadata: name: my-project-dev-gsa namespace: my-project-dev +`, + wantErr: false, + }, + { + name: "test create resource from templatized imagepullsecrets.yaml", + args: args{ + ctx: context.Background(), + templateDir: "testdata", + templateFileName: "imagepullsecrets_templatized.yaml", + project: &admin.Project{ + Name: "my-project", + Id: "my-project", + }, + domain: &admin.Domain{ + Id: "dev", + Name: "dev", + }, + namespace: "my-project-dev", + templateValues: templateValuesType{ + "{{ imagePullSecretsName }}": "default", + }, + customTemplateValues: templateValuesType{ + "{{ imagePullSecretsName }}": "custom", + }, + }, + wantK8sManifest: `apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: my-project-dev +imagePullSecrets: + - name: custom `, wantErr: false, }, diff --git a/flyteadmin/pkg/clusterresource/testdata/imagepullsecrets_templatized.yaml b/flyteadmin/pkg/clusterresource/testdata/imagepullsecrets_templatized.yaml new file mode 100644 index 0000000000..5c9d267382 --- /dev/null +++ b/flyteadmin/pkg/clusterresource/testdata/imagepullsecrets_templatized.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: {{ namespace }} +imagePullSecrets: + - name: {{ imagePullSecretsName }} From 37b4e13ac4a3594ac63b7a35058f4b2220e51282 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Mon, 12 Aug 2024 18:57:23 +0200 Subject: [PATCH 20/23] Don't error when attempting to trigger schedules for inactive projects (#5649) * Don't error when attempting to trigger schedules for inactive projects Signed-off-by: Katrina Rogan * regen Signed-off-by: Katrina Rogan --------- Signed-off-by: Katrina Rogan --- flyteadmin/pkg/errors/errors.go | 12 ++ flyteadmin/pkg/errors/errors_test.go | 12 ++ .../impl/validation/project_validator.go | 3 +- .../scheduler/executor/executor_impl.go | 19 +++ .../scheduler/executor/executor_impl_test.go | 12 ++ .../gen/pb-es/flyteidl/admin/project_pb.ts | 50 +++++++ .../gen/pb-go/flyteidl/admin/project.pb.go | 114 +++++++++++++--- flyteidl/gen/pb-js/flyteidl.d.ts | 58 ++++++++ flyteidl/gen/pb-js/flyteidl.js | 127 ++++++++++++++++++ .../pb_python/flyteidl/admin/project_pb2.py | 4 +- .../pb_python/flyteidl/admin/project_pb2.pyi | 8 ++ flyteidl/gen/pb_rust/flyteidl.admin.rs | 12 ++ flyteidl/protos/flyteidl/admin/project.proto | 12 ++ 13 files changed, 421 insertions(+), 22 deletions(-) diff --git a/flyteadmin/pkg/errors/errors.go b/flyteadmin/pkg/errors/errors.go index 51e5ede579..78727a7305 100644 --- a/flyteadmin/pkg/errors/errors.go +++ b/flyteadmin/pkg/errors/errors.go @@ -202,3 +202,15 @@ func IsDoesNotExistError(err error) bool { adminError, ok := err.(FlyteAdminError) return ok && adminError.Code() == codes.NotFound } + +func NewInactiveProjectError(ctx context.Context, id string) FlyteAdminError { + errMsg := fmt.Sprintf("project [%s] is not active", id) + statusErr, transformationErr := NewFlyteAdminError(codes.InvalidArgument, errMsg).WithDetails(&admin.InactiveProject{ + Id: id, + }) + if transformationErr != nil { + logger.Errorf(ctx, "failed to wrap grpc status in type 'Error': %v", transformationErr) + return NewFlyteAdminErrorf(codes.InvalidArgument, errMsg) + } + return statusErr +} diff --git a/flyteadmin/pkg/errors/errors_test.go b/flyteadmin/pkg/errors/errors_test.go index c126f96d6d..daaa060340 100644 --- a/flyteadmin/pkg/errors/errors_test.go +++ b/flyteadmin/pkg/errors/errors_test.go @@ -310,3 +310,15 @@ func TestIsNotDoesNotExistError(t *testing.T) { func TestIsNotDoesNotExistErrorBecauseOfNoneAdminError(t *testing.T) { assert.False(t, IsDoesNotExistError(errors.New("foo"))) } + +func TestNewInactiveProjectError(t *testing.T) { + err := NewInactiveProjectError(context.TODO(), identifier.GetProject()) + statusErr, ok := status.FromError(err) + + assert.True(t, ok) + + details, ok := statusErr.Details()[0].(*admin.InactiveProject) + + assert.True(t, ok) + assert.Equal(t, identifier.GetProject(), details.Id) +} diff --git a/flyteadmin/pkg/manager/impl/validation/project_validator.go b/flyteadmin/pkg/manager/impl/validation/project_validator.go index 8577c13e2b..8a76ce889d 100644 --- a/flyteadmin/pkg/manager/impl/validation/project_validator.go +++ b/flyteadmin/pkg/manager/impl/validation/project_validator.go @@ -71,8 +71,7 @@ func ValidateProjectAndDomain( projectID, domainID, err) } if *project.State != int32(admin.Project_ACTIVE) { - return errors.NewFlyteAdminErrorf(codes.InvalidArgument, - "project [%s] is not active", projectID) + return errors.NewInactiveProjectError(ctx, projectID) } var validDomain bool domains := config.GetDomainsConfig() diff --git a/flyteadmin/scheduler/executor/executor_impl.go b/flyteadmin/scheduler/executor/executor_impl.go index 30ab7f0677..dffb98e1b6 100644 --- a/flyteadmin/scheduler/executor/executor_impl.go +++ b/flyteadmin/scheduler/executor/executor_impl.go @@ -114,6 +114,10 @@ func (w *executor) Execute(ctx context.Context, scheduledTime time.Time, s model }, func() error { _, execErr := w.adminServiceClient.CreateExecution(context.Background(), executionRequest) + if isInactiveProjectError(execErr) { + logger.Debugf(ctx, "project %+v is inactive, ignoring schedule create failure for %+v", s.Project, s) + return nil + } return execErr }, ) @@ -144,3 +148,18 @@ func getExecutorMetrics(scope promutils.Scope) executorMetrics { "count of successful attempts to fire execution for a schedules"), } } + +func isInactiveProjectError(err error) bool { + statusErr, ok := status.FromError(err) + if !ok { + return false + } + if len(statusErr.Details()) > 0 { + for _, detail := range statusErr.Details() { + if _, ok := detail.(*admin.InactiveProject); ok { + return true + } + } + } + return false +} diff --git a/flyteadmin/scheduler/executor/executor_impl_test.go b/flyteadmin/scheduler/executor/executor_impl_test.go index e864d68d79..fc75367ca9 100644 --- a/flyteadmin/scheduler/executor/executor_impl_test.go +++ b/flyteadmin/scheduler/executor/executor_impl_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/scheduler/repositories/models" @@ -98,3 +99,14 @@ func TestExecutorInactiveSchedule(t *testing.T) { err := executor.Execute(context.Background(), time.Now(), schedule) assert.Nil(t, err) } + +func TestIsInactiveProjectError(t *testing.T) { + statusErr := status.New(codes.InvalidArgument, "foo") + var transformationErr error + statusErr, transformationErr = statusErr.WithDetails(&admin.InactiveProject{ + Id: "project", + }) + assert.NoError(t, transformationErr) + + assert.True(t, isInactiveProjectError(statusErr.Err())) +} diff --git a/flyteidl/gen/pb-es/flyteidl/admin/project_pb.ts b/flyteidl/gen/pb-es/flyteidl/admin/project_pb.ts index 11f2726e08..a6fc913c03 100644 --- a/flyteidl/gen/pb-es/flyteidl/admin/project_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/admin/project_pb.ts @@ -540,3 +540,53 @@ export class ProjectGetRequest extends Message { } } +/** + * Error returned for inactive projects + * + * @generated from message flyteidl.admin.InactiveProject + */ +export class InactiveProject extends Message { + /** + * Indicates a unique project. + * +required + * + * @generated from field: string id = 1; + */ + id = ""; + + /** + * Optional, org key applied to the resource. + * + * @generated from field: string org = 2; + */ + org = ""; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "flyteidl.admin.InactiveProject"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "org", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): InactiveProject { + return new InactiveProject().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): InactiveProject { + return new InactiveProject().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): InactiveProject { + return new InactiveProject().fromJsonString(jsonString, options); + } + + static equals(a: InactiveProject | PlainMessage | undefined, b: InactiveProject | PlainMessage | undefined): boolean { + return proto3.util.equals(InactiveProject, a, b); + } +} + diff --git a/flyteidl/gen/pb-go/flyteidl/admin/project.pb.go b/flyteidl/gen/pb-go/flyteidl/admin/project.pb.go index 243f46bf5d..d34451452b 100644 --- a/flyteidl/gen/pb-go/flyteidl/admin/project.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/admin/project.pb.go @@ -661,6 +661,65 @@ func (x *ProjectGetRequest) GetOrg() string { return "" } +// Error returned for inactive projects +type InactiveProject struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Indicates a unique project. + // +required + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Optional, org key applied to the resource. + Org string `protobuf:"bytes,2,opt,name=org,proto3" json:"org,omitempty"` +} + +func (x *InactiveProject) Reset() { + *x = InactiveProject{} + if protoimpl.UnsafeEnabled { + mi := &file_flyteidl_admin_project_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *InactiveProject) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InactiveProject) ProtoMessage() {} + +func (x *InactiveProject) ProtoReflect() protoreflect.Message { + mi := &file_flyteidl_admin_project_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InactiveProject.ProtoReflect.Descriptor instead. +func (*InactiveProject) Descriptor() ([]byte, []int) { + return file_flyteidl_admin_project_proto_rawDescGZIP(), []int{10} +} + +func (x *InactiveProject) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *InactiveProject) GetOrg() string { + if x != nil { + return x.Org + } + return "" +} + var File_flyteidl_admin_project_proto protoreflect.FileDescriptor var file_flyteidl_admin_project_proto_rawDesc = []byte{ @@ -725,19 +784,23 @@ var file_flyteidl_admin_project_proto_rawDesc = []byte{ 0x73, 0x65, 0x22, 0x35, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x42, 0xb8, 0x01, 0x0a, 0x12, 0x63, 0x6f, - 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, - 0x42, 0x0c, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, - 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, - 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, - 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0xa2, 0x02, 0x03, - 0x46, 0x41, 0x58, 0xaa, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x41, - 0x64, 0x6d, 0x69, 0x6e, 0xca, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, - 0x41, 0x64, 0x6d, 0x69, 0x6e, 0xe2, 0x02, 0x1a, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0xea, 0x02, 0x0f, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x41, - 0x64, 0x6d, 0x69, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x22, 0x33, 0x0a, 0x0f, 0x49, 0x6e, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x6f, 0x72, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x42, 0xb8, + 0x01, 0x0a, 0x12, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, + 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x42, 0x0c, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, + 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, + 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x61, 0x64, 0x6d, + 0x69, 0x6e, 0xa2, 0x02, 0x03, 0x46, 0x41, 0x58, 0xaa, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x2e, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0xca, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, + 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0xe2, 0x02, 0x1a, 0x46, 0x6c, 0x79, + 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x47, 0x50, 0x42, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0f, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x3a, 0x3a, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -753,7 +816,7 @@ func file_flyteidl_admin_project_proto_rawDescGZIP() []byte { } var file_flyteidl_admin_project_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_flyteidl_admin_project_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_flyteidl_admin_project_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_flyteidl_admin_project_proto_goTypes = []interface{}{ (Project_ProjectState)(0), // 0: flyteidl.admin.Project.ProjectState (*GetDomainRequest)(nil), // 1: flyteidl.admin.GetDomainRequest @@ -766,16 +829,17 @@ var file_flyteidl_admin_project_proto_goTypes = []interface{}{ (*ProjectRegisterResponse)(nil), // 8: flyteidl.admin.ProjectRegisterResponse (*ProjectUpdateResponse)(nil), // 9: flyteidl.admin.ProjectUpdateResponse (*ProjectGetRequest)(nil), // 10: flyteidl.admin.ProjectGetRequest - (*Labels)(nil), // 11: flyteidl.admin.Labels - (*Sort)(nil), // 12: flyteidl.admin.Sort + (*InactiveProject)(nil), // 11: flyteidl.admin.InactiveProject + (*Labels)(nil), // 12: flyteidl.admin.Labels + (*Sort)(nil), // 13: flyteidl.admin.Sort } var file_flyteidl_admin_project_proto_depIdxs = []int32{ 2, // 0: flyteidl.admin.GetDomainsResponse.domains:type_name -> flyteidl.admin.Domain 2, // 1: flyteidl.admin.Project.domains:type_name -> flyteidl.admin.Domain - 11, // 2: flyteidl.admin.Project.labels:type_name -> flyteidl.admin.Labels + 12, // 2: flyteidl.admin.Project.labels:type_name -> flyteidl.admin.Labels 0, // 3: flyteidl.admin.Project.state:type_name -> flyteidl.admin.Project.ProjectState 4, // 4: flyteidl.admin.Projects.projects:type_name -> flyteidl.admin.Project - 12, // 5: flyteidl.admin.ProjectListRequest.sort_by:type_name -> flyteidl.admin.Sort + 13, // 5: flyteidl.admin.ProjectListRequest.sort_by:type_name -> flyteidl.admin.Sort 4, // 6: flyteidl.admin.ProjectRegisterRequest.project:type_name -> flyteidl.admin.Project 7, // [7:7] is the sub-list for method output_type 7, // [7:7] is the sub-list for method input_type @@ -911,6 +975,18 @@ func file_flyteidl_admin_project_proto_init() { return nil } } + file_flyteidl_admin_project_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*InactiveProject); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -918,7 +994,7 @@ func file_flyteidl_admin_project_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_flyteidl_admin_project_proto_rawDesc, NumEnums: 1, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 0, }, diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index a1d9a34637..ada71f1f09 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -18184,6 +18184,64 @@ export namespace flyteidl { public static verify(message: { [k: string]: any }): (string|null); } + /** Properties of an InactiveProject. */ + interface IInactiveProject { + + /** InactiveProject id */ + id?: (string|null); + + /** InactiveProject org */ + org?: (string|null); + } + + /** Represents an InactiveProject. */ + class InactiveProject implements IInactiveProject { + + /** + * Constructs a new InactiveProject. + * @param [properties] Properties to set + */ + constructor(properties?: flyteidl.admin.IInactiveProject); + + /** InactiveProject id. */ + public id: string; + + /** InactiveProject org. */ + public org: string; + + /** + * Creates a new InactiveProject instance using the specified properties. + * @param [properties] Properties to set + * @returns InactiveProject instance + */ + public static create(properties?: flyteidl.admin.IInactiveProject): flyteidl.admin.InactiveProject; + + /** + * Encodes the specified InactiveProject message. Does not implicitly {@link flyteidl.admin.InactiveProject.verify|verify} messages. + * @param message InactiveProject message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: flyteidl.admin.IInactiveProject, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes an InactiveProject message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns InactiveProject + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.admin.InactiveProject; + + /** + * Verifies an InactiveProject message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + } + /** Properties of a ProjectAttributes. */ interface IProjectAttributes { diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index 3402b1bdbb..8f446b4aba 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -43952,6 +43952,133 @@ return ProjectGetRequest; })(); + admin.InactiveProject = (function() { + + /** + * Properties of an InactiveProject. + * @memberof flyteidl.admin + * @interface IInactiveProject + * @property {string|null} [id] InactiveProject id + * @property {string|null} [org] InactiveProject org + */ + + /** + * Constructs a new InactiveProject. + * @memberof flyteidl.admin + * @classdesc Represents an InactiveProject. + * @implements IInactiveProject + * @constructor + * @param {flyteidl.admin.IInactiveProject=} [properties] Properties to set + */ + function InactiveProject(properties) { + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * InactiveProject id. + * @member {string} id + * @memberof flyteidl.admin.InactiveProject + * @instance + */ + InactiveProject.prototype.id = ""; + + /** + * InactiveProject org. + * @member {string} org + * @memberof flyteidl.admin.InactiveProject + * @instance + */ + InactiveProject.prototype.org = ""; + + /** + * Creates a new InactiveProject instance using the specified properties. + * @function create + * @memberof flyteidl.admin.InactiveProject + * @static + * @param {flyteidl.admin.IInactiveProject=} [properties] Properties to set + * @returns {flyteidl.admin.InactiveProject} InactiveProject instance + */ + InactiveProject.create = function create(properties) { + return new InactiveProject(properties); + }; + + /** + * Encodes the specified InactiveProject message. Does not implicitly {@link flyteidl.admin.InactiveProject.verify|verify} messages. + * @function encode + * @memberof flyteidl.admin.InactiveProject + * @static + * @param {flyteidl.admin.IInactiveProject} message InactiveProject message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + InactiveProject.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.id != null && message.hasOwnProperty("id")) + writer.uint32(/* id 1, wireType 2 =*/10).string(message.id); + if (message.org != null && message.hasOwnProperty("org")) + writer.uint32(/* id 2, wireType 2 =*/18).string(message.org); + return writer; + }; + + /** + * Decodes an InactiveProject message from the specified reader or buffer. + * @function decode + * @memberof flyteidl.admin.InactiveProject + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {flyteidl.admin.InactiveProject} InactiveProject + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + InactiveProject.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.admin.InactiveProject(); + while (reader.pos < end) { + var tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.id = reader.string(); + break; + case 2: + message.org = reader.string(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Verifies an InactiveProject message. + * @function verify + * @memberof flyteidl.admin.InactiveProject + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + InactiveProject.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.id != null && message.hasOwnProperty("id")) + if (!$util.isString(message.id)) + return "id: string expected"; + if (message.org != null && message.hasOwnProperty("org")) + if (!$util.isString(message.org)) + return "org: string expected"; + return null; + }; + + return InactiveProject; + })(); + admin.ProjectAttributes = (function() { /** diff --git a/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.py b/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.py index 885ef84716..c04fdb67e1 100644 --- a/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.py @@ -14,7 +14,7 @@ from flyteidl.admin import common_pb2 as flyteidl_dot_admin_dot_common__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/admin/project.proto\x12\x0e\x66lyteidl.admin\x1a\x1b\x66lyteidl/admin/common.proto\"\x12\n\x10GetDomainRequest\",\n\x06\x44omain\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"F\n\x12GetDomainsResponse\x12\x30\n\x07\x64omains\x18\x01 \x03(\x0b\x32\x16.flyteidl.admin.DomainR\x07\x64omains\"\xd4\x02\n\x07Project\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x30\n\x07\x64omains\x18\x03 \x03(\x0b\x32\x16.flyteidl.admin.DomainR\x07\x64omains\x12 \n\x0b\x64\x65scription\x18\x04 \x01(\tR\x0b\x64\x65scription\x12.\n\x06labels\x18\x05 \x01(\x0b\x32\x16.flyteidl.admin.LabelsR\x06labels\x12:\n\x05state\x18\x06 \x01(\x0e\x32$.flyteidl.admin.Project.ProjectStateR\x05state\x12\x10\n\x03org\x18\x07 \x01(\tR\x03org\"S\n\x0cProjectState\x12\n\n\x06\x41\x43TIVE\x10\x00\x12\x0c\n\x08\x41RCHIVED\x10\x01\x12\x14\n\x10SYSTEM_GENERATED\x10\x02\x12\x13\n\x0fSYSTEM_ARCHIVED\x10\x03\"U\n\x08Projects\x12\x33\n\x08projects\x18\x01 \x03(\x0b\x32\x17.flyteidl.admin.ProjectR\x08projects\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"\x9b\x01\n\x12ProjectListRequest\x12\x14\n\x05limit\x18\x01 \x01(\rR\x05limit\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\x12\x18\n\x07\x66ilters\x18\x03 \x01(\tR\x07\x66ilters\x12-\n\x07sort_by\x18\x04 \x01(\x0b\x32\x14.flyteidl.admin.SortR\x06sortBy\x12\x10\n\x03org\x18\x05 \x01(\tR\x03org\"K\n\x16ProjectRegisterRequest\x12\x31\n\x07project\x18\x01 \x01(\x0b\x32\x17.flyteidl.admin.ProjectR\x07project\"\x19\n\x17ProjectRegisterResponse\"\x17\n\x15ProjectUpdateResponse\"5\n\x11ProjectGetRequest\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x10\n\x03org\x18\x02 \x01(\tR\x03orgB\xb8\x01\n\x12\x63om.flyteidl.adminB\x0cProjectProtoP\x01Z;github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin\xa2\x02\x03\x46\x41X\xaa\x02\x0e\x46lyteidl.Admin\xca\x02\x0e\x46lyteidl\\Admin\xe2\x02\x1a\x46lyteidl\\Admin\\GPBMetadata\xea\x02\x0f\x46lyteidl::Adminb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/admin/project.proto\x12\x0e\x66lyteidl.admin\x1a\x1b\x66lyteidl/admin/common.proto\"\x12\n\x10GetDomainRequest\",\n\x06\x44omain\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"F\n\x12GetDomainsResponse\x12\x30\n\x07\x64omains\x18\x01 \x03(\x0b\x32\x16.flyteidl.admin.DomainR\x07\x64omains\"\xd4\x02\n\x07Project\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x30\n\x07\x64omains\x18\x03 \x03(\x0b\x32\x16.flyteidl.admin.DomainR\x07\x64omains\x12 \n\x0b\x64\x65scription\x18\x04 \x01(\tR\x0b\x64\x65scription\x12.\n\x06labels\x18\x05 \x01(\x0b\x32\x16.flyteidl.admin.LabelsR\x06labels\x12:\n\x05state\x18\x06 \x01(\x0e\x32$.flyteidl.admin.Project.ProjectStateR\x05state\x12\x10\n\x03org\x18\x07 \x01(\tR\x03org\"S\n\x0cProjectState\x12\n\n\x06\x41\x43TIVE\x10\x00\x12\x0c\n\x08\x41RCHIVED\x10\x01\x12\x14\n\x10SYSTEM_GENERATED\x10\x02\x12\x13\n\x0fSYSTEM_ARCHIVED\x10\x03\"U\n\x08Projects\x12\x33\n\x08projects\x18\x01 \x03(\x0b\x32\x17.flyteidl.admin.ProjectR\x08projects\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"\x9b\x01\n\x12ProjectListRequest\x12\x14\n\x05limit\x18\x01 \x01(\rR\x05limit\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\x12\x18\n\x07\x66ilters\x18\x03 \x01(\tR\x07\x66ilters\x12-\n\x07sort_by\x18\x04 \x01(\x0b\x32\x14.flyteidl.admin.SortR\x06sortBy\x12\x10\n\x03org\x18\x05 \x01(\tR\x03org\"K\n\x16ProjectRegisterRequest\x12\x31\n\x07project\x18\x01 \x01(\x0b\x32\x17.flyteidl.admin.ProjectR\x07project\"\x19\n\x17ProjectRegisterResponse\"\x17\n\x15ProjectUpdateResponse\"5\n\x11ProjectGetRequest\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x10\n\x03org\x18\x02 \x01(\tR\x03org\"3\n\x0fInactiveProject\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x10\n\x03org\x18\x02 \x01(\tR\x03orgB\xb8\x01\n\x12\x63om.flyteidl.adminB\x0cProjectProtoP\x01Z;github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin\xa2\x02\x03\x46\x41X\xaa\x02\x0e\x46lyteidl.Admin\xca\x02\x0e\x46lyteidl\\Admin\xe2\x02\x1a\x46lyteidl\\Admin\\GPBMetadata\xea\x02\x0f\x46lyteidl::Adminb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -45,4 +45,6 @@ _globals['_PROJECTUPDATERESPONSE']._serialized_end=930 _globals['_PROJECTGETREQUEST']._serialized_start=932 _globals['_PROJECTGETREQUEST']._serialized_end=985 + _globals['_INACTIVEPROJECT']._serialized_start=987 + _globals['_INACTIVEPROJECT']._serialized_end=1038 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.pyi index 12750a8959..c775c5aac8 100644 --- a/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/admin/project_pb2.pyi @@ -96,3 +96,11 @@ class ProjectGetRequest(_message.Message): id: str org: str def __init__(self, id: _Optional[str] = ..., org: _Optional[str] = ...) -> None: ... + +class InactiveProject(_message.Message): + __slots__ = ["id", "org"] + ID_FIELD_NUMBER: _ClassVar[int] + ORG_FIELD_NUMBER: _ClassVar[int] + id: str + org: str + def __init__(self, id: _Optional[str] = ..., org: _Optional[str] = ...) -> None: ... diff --git a/flyteidl/gen/pb_rust/flyteidl.admin.rs b/flyteidl/gen/pb_rust/flyteidl.admin.rs index dcbf3b5df7..ca3270264b 100644 --- a/flyteidl/gen/pb_rust/flyteidl.admin.rs +++ b/flyteidl/gen/pb_rust/flyteidl.admin.rs @@ -2615,6 +2615,18 @@ pub struct ProjectGetRequest { #[prost(string, tag="2")] pub org: ::prost::alloc::string::String, } +/// Error returned for inactive projects +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InactiveProject { + /// Indicates a unique project. + /// +required + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + /// Optional, org key applied to the resource. + #[prost(string, tag="2")] + pub org: ::prost::alloc::string::String, +} /// Defines a set of custom matching attributes at the project level. /// For more info on matchable attributes, see :ref:`ref_flyteidl.admin.MatchableAttributesConfiguration` #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/flyteidl/protos/flyteidl/admin/project.proto b/flyteidl/protos/flyteidl/admin/project.proto index bbaccd70ff..8b994b7267 100644 --- a/flyteidl/protos/flyteidl/admin/project.proto +++ b/flyteidl/protos/flyteidl/admin/project.proto @@ -118,3 +118,15 @@ message ProjectGetRequest { // Optional, org key applied to the resource. string org = 2; } + + +// Error returned for inactive projects +message InactiveProject { + // Indicates a unique project. + // +required + string id = 1; + + // Optional, org key applied to the resource. + string org = 2; +} + From 4f9227a39f2affb937db1921712bf33a29b20959 Mon Sep 17 00:00:00 2001 From: Flyte Bot Date: Tue, 13 Aug 2024 13:50:55 -0700 Subject: [PATCH 21/23] Update Flyte components - v1.13.1-rc0 (#5656) * Update Flyte Components Signed-off-by: Flyte-Bot * Bump conf.py and add changelog Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Flyte-Bot Signed-off-by: Eduardo Apolinario Co-authored-by: eapolinario --- CHANGELOG/CHANGELOG-v1.13.1-rc0.md | 65 +++++++++++++++++++ charts/flyte-binary/README.md | 2 +- charts/flyte-binary/values.yaml | 2 +- charts/flyte-core/README.md | 14 ++-- charts/flyte-core/values.yaml | 12 ++-- charts/flyte/README.md | 18 ++--- charts/flyte/values.yaml | 12 ++-- charts/flyteagent/README.md | 2 +- charts/flyteagent/values.yaml | 2 +- .../agent/flyte_agent_helm_generated.yaml | 2 +- .../flyte_aws_scheduler_helm_generated.yaml | 32 ++++----- .../flyte_helm_controlplane_generated.yaml | 22 +++---- .../eks/flyte_helm_dataplane_generated.yaml | 14 ++-- deployment/eks/flyte_helm_generated.yaml | 36 +++++----- .../flyte_helm_controlplane_generated.yaml | 22 +++---- .../gcp/flyte_helm_dataplane_generated.yaml | 14 ++-- deployment/gcp/flyte_helm_generated.yaml | 36 +++++----- .../flyte_sandbox_binary_helm_generated.yaml | 4 +- deployment/sandbox/flyte_helm_generated.yaml | 36 +++++----- .../manifests/complete-agent.yaml | 10 +-- .../sandbox-bundled/manifests/complete.yaml | 8 +-- docker/sandbox-bundled/manifests/dev.yaml | 4 +- docs/conf.py | 2 +- .../generated/flyteadmin_config.rst | 12 ++++ .../generated/scheduler_config.rst | 12 ++++ 25 files changed, 242 insertions(+), 153 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-v1.13.1-rc0.md diff --git a/CHANGELOG/CHANGELOG-v1.13.1-rc0.md b/CHANGELOG/CHANGELOG-v1.13.1-rc0.md new file mode 100644 index 0000000000..f5481309a6 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.13.1-rc0.md @@ -0,0 +1,65 @@ +# Flyte v1.13.1-rc0 Release Notes + +## Changelog +* chore: update runllm widget configuration by @agiron123 in https://github.com/flyteorg/flyte/pull/5530 +* Use jsonpb AllowUnknownFields everywhere by @andrewwdye in https://github.com/flyteorg/flyte/pull/5521 +* [flyteadmin] Use WithContext in all DB calls by @Sovietaced in https://github.com/flyteorg/flyte/pull/5538 +* add new project state: SYSTEM_ARCHIVED by @troychiu in https://github.com/flyteorg/flyte/pull/5544 +* Pryce/doc 434 clarify how code is pushed into a given image during pyflyte by @pryce-turner in https://github.com/flyteorg/flyte/pull/5548 +* Increase more memory limits in flyteagent by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5550 +* Updated map task information to indicate array node is now the default and optional return type by @pryce-turner in https://github.com/flyteorg/flyte/pull/5561 +* reverted mockery-v2 on ExecutionContext by @hamersaw in https://github.com/flyteorg/flyte/pull/5562 +* Fix issues with helm chart release process by @Sovietaced in https://github.com/flyteorg/flyte/pull/5560 +* Add FlyteDirectory to file_types.rst template by @ppiegaze in https://github.com/flyteorg/flyte/pull/5564 +* Fully populate Abort task event fields by @va6996 in https://github.com/flyteorg/flyte/pull/5551 +* [fix] Add blob typechecker by @ddl-rliu in https://github.com/flyteorg/flyte/pull/5519 +* Refactor echo plugin by @pingsutw in https://github.com/flyteorg/flyte/pull/5565 +* [Bug] fix ArrayNode state's TaskPhase reset by @pvditt in https://github.com/flyteorg/flyte/pull/5451 +* Remove confusing prometheus configuration options in helm charts by @Sovietaced in https://github.com/flyteorg/flyte/pull/5549 +* [Housekeeping] Bump Go version to 1.22 by @lowc1012 in https://github.com/flyteorg/flyte/pull/5032 +* Fix typos by @omahs in https://github.com/flyteorg/flyte/pull/5571 +* Respect original task definition retry strategy for single task executions by @katrogan in https://github.com/flyteorg/flyte/pull/5577 +* Clarify the support for the Java/Scala SDK in the docs by @eapolinario in https://github.com/flyteorg/flyte/pull/5582 +* Fix spelling issues by @nnsW3 in https://github.com/flyteorg/flyte/pull/5580 +* Give flyte binary cluster role permission to create service accounts by @shreyas44 in https://github.com/flyteorg/flyte/pull/5579 +* Simplify single task retry strategy check by @eapolinario in https://github.com/flyteorg/flyte/pull/5584 +* Fix failures in `generate_helm` CI check by @eapolinario in https://github.com/flyteorg/flyte/pull/5587 +* Update GPU docs by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5515 +* Update azblob 1.1.0 -> 1.4.0 / azcore 1.7.2 -> 1.13.0 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5590 +* Bump github.com/go-jose/go-jose/v3 from 3.0.0 to 3.0.3 in /flyteadmin by @dependabot in https://github.com/flyteorg/flyte/pull/5591 +* add execution mode to ArrayNode proto by @pvditt in https://github.com/flyteorg/flyte/pull/5512 +* Fix incorrect YAML for unpartitoned GPU by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5595 +* Another YAML fix by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5596 +* DOC-431 Document pyflyte option --overwrite-cache by @ppiegaze in https://github.com/flyteorg/flyte/pull/5567 +* Upgrade docker dependency to address vulnerability by @katrogan in https://github.com/flyteorg/flyte/pull/5614 +* Support offloading workflow CRD inputs by @katrogan in https://github.com/flyteorg/flyte/pull/5609 +* [flyteadmin] Refactor panic recovery into middleware by @Sovietaced in https://github.com/flyteorg/flyte/pull/5546 +* Snowflake agent Doc by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5620 +* [flytepropeller][compiler] Error Handling when Type is not found by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5612 +* Fix nil pointer when task plugin load returns error by @Sovietaced in https://github.com/flyteorg/flyte/pull/5622 +* Log stack trace when refresh cache sync recovers from panic by @Sovietaced in https://github.com/flyteorg/flyte/pull/5623 +* [Doc] Fix snowflake agent secret documentation error by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5626 +* [Doc] Explain how Agent Secret Works by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5625 +* Fix typo in execution manager by @ddl-rliu in https://github.com/flyteorg/flyte/pull/5619 +* Amend Admin to use grpc message size by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5628 +* [Docs] document the process of setting ttl for a ray cluster by @pingsutw in https://github.com/flyteorg/flyte/pull/5636 +* Add CustomHeaderMatcher to pass additional headers by @andrewwdye in https://github.com/flyteorg/flyte/pull/5563 +* Turn flyteidl and flytectl releases into manual gh workflows by @eapolinario in https://github.com/flyteorg/flyte/pull/5635 +* docs: fix typo by @cratiu222 in https://github.com/flyteorg/flyte/pull/5643 +* Use enable_deck=True in docs by @thomasjpfan in https://github.com/flyteorg/flyte/pull/5645 +* Fix flyteidl release checkout all tags by @eapolinario in https://github.com/flyteorg/flyte/pull/5646 +* Install pyarrow in sandbox functional tests by @eapolinario in https://github.com/flyteorg/flyte/pull/5647 +* docs: add documentation for configuring notifications in GCP by @desihsu in https://github.com/flyteorg/flyte/pull/5545 +* Correct "sucessfile" to "successfile" by @shengyu7697 in https://github.com/flyteorg/flyte/pull/5652 +* Fix ordering for custom template values in cluster resource controller by @katrogan in https://github.com/flyteorg/flyte/pull/5648 +* Don't error when attempting to trigger schedules for inactive projects by @katrogan in https://github.com/flyteorg/flyte/pull/5649 + +## New Contributors +* @omahs made their first contribution in https://github.com/flyteorg/flyte/pull/5571 +* @nnsW3 made their first contribution in https://github.com/flyteorg/flyte/pull/5580 +* @shreyas44 made their first contribution in https://github.com/flyteorg/flyte/pull/5579 +* @cratiu222 made their first contribution in https://github.com/flyteorg/flyte/pull/5643 +* @desihsu made their first contribution in https://github.com/flyteorg/flyte/pull/5545 +* @shengyu7697 made their first contribution in https://github.com/flyteorg/flyte/pull/5652 + +**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.13.0...v1.13.1-rc0 diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md index 63a351d0f5..64feee5d89 100644 --- a/charts/flyte-binary/README.md +++ b/charts/flyte-binary/README.md @@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment | configuration.auth.oidc.clientId | string | `""` | | | configuration.auth.oidc.clientSecret | string | `""` | | | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` | | -| configuration.co-pilot.image.tag | string | `"v1.13.0"` | | +| configuration.co-pilot.image.tag | string | `"v1.13.1-rc0"` | | | configuration.database.dbname | string | `"flyte"` | | | configuration.database.host | string | `"127.0.0.1"` | | | configuration.database.options | string | `"sslmode=disable"` | | diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml index 66d1e1feae..b3ea6877f7 100644 --- a/charts/flyte-binary/values.yaml +++ b/charts/flyte-binary/values.yaml @@ -159,7 +159,7 @@ configuration: # repository CoPilot sidecar image repository repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE # tag CoPilot sidecar image tag - tag: v1.13.0 # FLYTECOPILOT_TAG + tag: v1.13.1-rc0 # FLYTECOPILOT_TAG # agentService Flyte Agent configuration agentService: defaultAgent: diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 8aaab01172..cf4f511a2e 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -95,8 +95,8 @@ helm install gateway bitnami/contour -n flyte | configmap.clusters.clusterConfigs | list | `[]` | | | configmap.clusters.labelClusterMap | object | `{}` | | | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). | | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | @@ -130,7 +130,7 @@ helm install gateway bitnami/contour -n flyte | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v1.13.0"` | Docker image tag | +| datacatalog.image.tag | string | `"v1.13.1-rc0"` | Docker image tag | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables | @@ -166,7 +166,7 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v1.13.0"` | | +| flyteadmin.image.tag | string | `"v1.13.1-rc0"` | | | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -207,7 +207,7 @@ helm install gateway bitnami/contour -n flyte | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` | | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v1.15.1"` | | +| flyteconsole.image.tag | string | `"v1.17.1"` | | | flyteconsole.imagePullSecrets | list | `[]` | ImagePullSecrets to assign to the Flyteconsole deployment | | flyteconsole.livenessProbe | object | `{}` | | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | @@ -238,7 +238,7 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v1.13.0"` | | +| flytepropeller.image.tag | string | `"v1.13.1-rc0"` | | | flytepropeller.manager | bool | `false` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | @@ -270,7 +270,7 @@ helm install gateway bitnami/contour -n flyte | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flytescheduler.image.tag | string | `"v1.13.0"` | Docker image tag | +| flytescheduler.image.tag | string | `"v1.13.1-rc0"` | Docker image tag | | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables | diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 01508385e9..2c9e02427e 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE - tag: v1.13.0 # FLYTEADMIN_TAG + tag: v1.13.1-rc0 # FLYTEADMIN_TAG pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables # @@ -144,7 +144,7 @@ flytescheduler: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.13.0 # FLYTESCHEDULER_TAG + tag: v1.13.1-rc0 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -210,7 +210,7 @@ datacatalog: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.13.0 # DATACATALOG_TAG + tag: v1.13.1-rc0 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -309,7 +309,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE - tag: v1.13.0 # FLYTEPROPELLER_TAG + tag: v1.13.1-rc0 # FLYTEPROPELLER_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -398,7 +398,7 @@ flyteconsole: image: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE - tag: v1.15.1 # FLYTECONSOLE_TAG + tag: v1.17.1 # FLYTECONSOLE_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -801,7 +801,7 @@ configmap: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration diff --git a/charts/flyte/README.md b/charts/flyte/README.md index 4df222dbd4..321b9e22a3 100644 --- a/charts/flyte/README.md +++ b/charts/flyte/README.md @@ -71,7 +71,7 @@ helm upgrade -f values-sandbox.yaml flyte . | contour.tolerations | list | `[]` | tolerations for Contour deployment | | daskoperator | object | `{"enabled":false}` | Optional: Dask Plugin using the Dask Operator | | daskoperator.enabled | bool | `false` | - enable or disable the dask operator deployment installation | -| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.13.0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.13.0"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.15.1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.13.0"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.13.0"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | +| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.13.1-rc0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.13.1-rc0"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.17.1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.13.1-rc0"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.13.1-rc0"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | | flyte.cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | | flyte.cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | | flyte.cluster_resource_manager.config.cluster_resources.standaloneDeployment | bool | `false` | Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints | @@ -91,15 +91,15 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.common.ingress.separateGrpcIngressAnnotations | object | `{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"}` | - Extra Ingress annotations applied only to the GRPC ingress. Only makes sense if `separateGrpcIngress` is enabled. | | flyte.common.ingress.tls | object | `{"enabled":false}` | - TLS Settings | | flyte.common.ingress.webpackHMR | bool | `true` | - Enable or disable HMR route to flyteconsole. This is useful only for frontend development. | -| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | +| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | | flyte.configmap.adminServer | object | `{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | | flyte.configmap.adminServer.auth | object | `{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}}` | Authentication configuration | | flyte.configmap.adminServer.server.security.secure | bool | `false` | Controls whether to serve requests over SSL/TLS. | | flyte.configmap.adminServer.server.security.useAuth | bool | `false` | Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. | | flyte.configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | | flyte.configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | flyte.configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | flyte.configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | flyte.configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | @@ -120,7 +120,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | flyte.datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| flyte.datacatalog.image.tag | string | `"v1.13.0"` | Docker image tag | +| flyte.datacatalog.image.tag | string | `"v1.13.1-rc0"` | Docker image tag | | flyte.datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | flyte.datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | flyte.datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | @@ -136,7 +136,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteadmin.env | list | `[]` | Additional flyteadmin container environment variables e.g. SendGrid's API key - name: SENDGRID_API_KEY value: "" e.g. secret environment variable (you can combine it with .additionalVolumes): - name: SENDGRID_API_KEY valueFrom: secretKeyRef: name: sendgrid-secret key: api_key | | flyte.flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyte.flyteadmin.image.tag | string | `"v1.13.0"` | Docker image tag | +| flyte.flyteadmin.image.tag | string | `"v1.13.1-rc0"` | Docker image tag | | flyte.flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyte.flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyte.flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -151,7 +151,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyte.flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyte.flyteconsole.image.tag | string | `"v1.15.1"` | Docker image tag | +| flyte.flyteconsole.image.tag | string | `"v1.17.1"` | Docker image tag | | flyte.flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyte.flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyte.flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -162,7 +162,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flyte.flytepropeller.image.tag | string | `"v1.13.0"` | Docker image tag | +| flyte.flytepropeller.image.tag | string | `"v1.13.1-rc0"` | Docker image tag | | flyte.flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flyte.flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flyte.flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | @@ -176,7 +176,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flyte.flytescheduler.image.tag | string | `"v1.13.0"` | Docker image tag | +| flyte.flytescheduler.image.tag | string | `"v1.13.1-rc0"` | Docker image tag | | flyte.flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flyte.flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flyte.flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment | diff --git a/charts/flyte/values.yaml b/charts/flyte/values.yaml index 9d4940537f..acd3df9050 100755 --- a/charts/flyte/values.yaml +++ b/charts/flyte/values.yaml @@ -15,7 +15,7 @@ flyte: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE # -- Docker image tag - tag: v1.13.0 # FLYTEADMIN_TAG + tag: v1.13.1-rc0 # FLYTEADMIN_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables @@ -83,7 +83,7 @@ flyte: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.13.0 # FLYTESCHEDULER_TAG + tag: v1.13.1-rc0 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -128,7 +128,7 @@ flyte: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.13.0 # DATACATALOG_TAG + tag: v1.13.1-rc0 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -177,7 +177,7 @@ flyte: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE # -- Docker image tag - tag: v1.13.0 # FLYTEPROPELLER_TAG + tag: v1.13.1-rc0 # FLYTEPROPELLER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment @@ -222,7 +222,7 @@ flyte: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE # -- Docker image tag - tag: v1.15.1 # FLYTECONSOLE_TAG + tag: v1.17.1 # FLYTECONSOLE_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment @@ -471,7 +471,7 @@ flyte: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration diff --git a/charts/flyteagent/README.md b/charts/flyteagent/README.md index d5a33165f9..e6851b5758 100644 --- a/charts/flyteagent/README.md +++ b/charts/flyteagent/README.md @@ -20,7 +20,7 @@ A Helm chart for Flyte agent | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | image.repository | string | `"cr.flyte.org/flyteorg/flyteagent"` | Docker image for flyteagent deployment | -| image.tag | string | `"1.13.0"` | Docker image tag | +| image.tag | string | `"1.13.3"` | Docker image tag | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | nodeSelector for flyteagent deployment | | podAnnotations | object | `{}` | Annotations for flyteagent pods | diff --git a/charts/flyteagent/values.yaml b/charts/flyteagent/values.yaml index d960a90e8c..ce23995df0 100755 --- a/charts/flyteagent/values.yaml +++ b/charts/flyteagent/values.yaml @@ -23,7 +23,7 @@ image: # -- Docker image for flyteagent deployment repository: cr.flyte.org/flyteorg/flyteagent # FLYTEAGENT_IMAGE # -- Docker image tag - tag: 1.13.0 # FLYTEAGENT_TAG + tag: 1.13.3 # FLYTEAGENT_TAG # -- Docker image pull policy pullPolicy: IfNotPresent ports: diff --git a/deployment/agent/flyte_agent_helm_generated.yaml b/deployment/agent/flyte_agent_helm_generated.yaml index 4c9eec59e3..c8244070d5 100644 --- a/deployment/agent/flyte_agent_helm_generated.yaml +++ b/deployment/agent/flyte_agent_helm_generated.yaml @@ -79,7 +79,7 @@ spec: - pyflyte - serve - agent - image: "cr.flyte.org/flyteorg/flyteagent:1.13.0" + image: "cr.flyte.org/flyteorg/flyteagent:1.13.3" imagePullPolicy: "IfNotPresent" name: flyteagent volumeMounts: diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 0ce940cfa9..7e2f9fa395 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -430,7 +430,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -876,7 +876,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -897,7 +897,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -915,7 +915,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -932,7 +932,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -959,7 +959,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1066,7 +1066,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1122,7 +1122,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.15.1" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.17.1" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1196,7 +1196,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1214,7 +1214,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1277,7 +1277,7 @@ spec: template: metadata: annotations: - configChecksum: "38c8d0c5ce2cb50f839d3ecda13d881b6bf5a503ac264f0a9a821a9b5a70ca6" + configChecksum: "de81ec89079e3abbbc351f1e9dd5f918ac37e7a302dfe32a4ce4da1083980cd" prometheus.io/path: "/metrics" prometheus.io/port: "10254" labels: @@ -1305,7 +1305,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1359,9 +1359,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.13.0 + app.kubernetes.io/version: v1.13.1-rc0 annotations: - configChecksum: "38c8d0c5ce2cb50f839d3ecda13d881b6bf5a503ac264f0a9a821a9b5a70ca6" + configChecksum: "de81ec89079e3abbbc351f1e9dd5f918ac37e7a302dfe32a4ce4da1083980cd" prometheus.io/path: "/metrics" prometheus.io/port: "10254" spec: @@ -1375,7 +1375,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1402,7 +1402,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index d9ebf1d7bd..b6fd465b34 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -581,7 +581,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -602,7 +602,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -620,7 +620,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -637,7 +637,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -664,7 +664,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -771,7 +771,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -827,7 +827,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.15.1" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.17.1" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -901,7 +901,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -919,7 +919,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1002,7 +1002,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1022,7 +1022,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 040cb007f7..421c16dae3 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -428,7 +428,7 @@ spec: template: metadata: annotations: - configChecksum: "38c8d0c5ce2cb50f839d3ecda13d881b6bf5a503ac264f0a9a821a9b5a70ca6" + configChecksum: "de81ec89079e3abbbc351f1e9dd5f918ac37e7a302dfe32a4ce4da1083980cd" prometheus.io/path: "/metrics" prometheus.io/port: "10254" labels: @@ -456,7 +456,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -510,9 +510,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.13.0 + app.kubernetes.io/version: v1.13.1-rc0 annotations: - configChecksum: "38c8d0c5ce2cb50f839d3ecda13d881b6bf5a503ac264f0a9a821a9b5a70ca6" + configChecksum: "de81ec89079e3abbbc351f1e9dd5f918ac37e7a302dfe32a4ce4da1083980cd" prometheus.io/path: "/metrics" prometheus.io/port: "10254" spec: @@ -526,7 +526,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -553,7 +553,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index 5fc562963e..1e9d9a5bf1 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -461,7 +461,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -907,7 +907,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -928,7 +928,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -946,7 +946,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -963,7 +963,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -990,7 +990,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1097,7 +1097,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1153,7 +1153,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.15.1" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.17.1" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1227,7 +1227,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1245,7 +1245,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1328,7 +1328,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1348,7 +1348,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1407,7 +1407,7 @@ spec: template: metadata: annotations: - configChecksum: "38c8d0c5ce2cb50f839d3ecda13d881b6bf5a503ac264f0a9a821a9b5a70ca6" + configChecksum: "de81ec89079e3abbbc351f1e9dd5f918ac37e7a302dfe32a4ce4da1083980cd" prometheus.io/path: "/metrics" prometheus.io/port: "10254" labels: @@ -1435,7 +1435,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1489,9 +1489,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.13.0 + app.kubernetes.io/version: v1.13.1-rc0 annotations: - configChecksum: "38c8d0c5ce2cb50f839d3ecda13d881b6bf5a503ac264f0a9a821a9b5a70ca6" + configChecksum: "de81ec89079e3abbbc351f1e9dd5f918ac37e7a302dfe32a4ce4da1083980cd" prometheus.io/path: "/metrics" prometheus.io/port: "10254" spec: @@ -1505,7 +1505,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1532,7 +1532,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index e83e4ebe24..7d999c54fa 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -596,7 +596,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -617,7 +617,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -635,7 +635,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -652,7 +652,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -679,7 +679,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -786,7 +786,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -842,7 +842,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.15.1" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.17.1" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -916,7 +916,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -934,7 +934,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1017,7 +1017,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1037,7 +1037,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index b5ad82d3f0..a189c612a2 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -436,7 +436,7 @@ spec: template: metadata: annotations: - configChecksum: "e63d877e1ff27d18ac7c4af4c6e4ca6106fc337a13ce00d9beb8efed8548493" + configChecksum: "407e317f4b09b0311b506d7bf60e23bd6e98acd501f0301c2d78d71ea108983" prometheus.io/path: "/metrics" prometheus.io/port: "10254" labels: @@ -463,7 +463,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -517,9 +517,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.13.0 + app.kubernetes.io/version: v1.13.1-rc0 annotations: - configChecksum: "e63d877e1ff27d18ac7c4af4c6e4ca6106fc337a13ce00d9beb8efed8548493" + configChecksum: "407e317f4b09b0311b506d7bf60e23bd6e98acd501f0301c2d78d71ea108983" prometheus.io/path: "/metrics" prometheus.io/port: "10254" spec: @@ -533,7 +533,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -560,7 +560,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 4e3fe06e38..c51e6972ce 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -474,7 +474,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -930,7 +930,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -951,7 +951,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -969,7 +969,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -986,7 +986,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -1013,7 +1013,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1120,7 +1120,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1176,7 +1176,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.15.1" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.17.1" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1250,7 +1250,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1268,7 +1268,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1351,7 +1351,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1371,7 +1371,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1430,7 +1430,7 @@ spec: template: metadata: annotations: - configChecksum: "e63d877e1ff27d18ac7c4af4c6e4ca6106fc337a13ce00d9beb8efed8548493" + configChecksum: "407e317f4b09b0311b506d7bf60e23bd6e98acd501f0301c2d78d71ea108983" prometheus.io/path: "/metrics" prometheus.io/port: "10254" labels: @@ -1457,7 +1457,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1511,9 +1511,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.13.0 + app.kubernetes.io/version: v1.13.1-rc0 annotations: - configChecksum: "e63d877e1ff27d18ac7c4af4c6e4ca6106fc337a13ce00d9beb8efed8548493" + configChecksum: "407e317f4b09b0311b506d7bf60e23bd6e98acd501f0301c2d78d71ea108983" prometheus.io/path: "/metrics" prometheus.io/port: "10254" spec: @@ -1527,7 +1527,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1554,7 +1554,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml index 08599c13f7..74a37957ea 100644 --- a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml +++ b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml @@ -116,7 +116,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.13.0" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0" k8s-array: logs: config: @@ -359,7 +359,7 @@ spec: app.kubernetes.io/instance: flyte app.kubernetes.io/component: flyte-binary annotations: - checksum/configuration: cc9ac3ae89908d4036cebe1d3b50ce9bf564167ca808afdec67c90260665a58e + checksum/configuration: 58c26a7a95c9edce075726e132dac345f0aafb69dea2b21f6445dc2615ee61fe checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929 checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae spec: diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index f53025f850..28ab47df17 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -586,7 +586,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.13.0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -6714,7 +6714,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -6734,7 +6734,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -6751,7 +6751,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -6767,7 +6767,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -6794,7 +6794,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -6891,7 +6891,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -6944,7 +6944,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.15.1" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.17.1" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -7016,7 +7016,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -7033,7 +7033,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.13.0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -7106,7 +7106,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -7125,7 +7125,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -7181,7 +7181,7 @@ spec: template: metadata: annotations: - configChecksum: "7ee7db423358a7296a24baf63ddb60c73b3c38c663d92731caae21e75d89fbc" + configChecksum: "87f8dd83145c058839fbf440c688d131d5917282ae935b2fe02147df47ef3a7" prometheus.io/path: "/metrics" prometheus.io/port: "10254" labels: @@ -7208,7 +7208,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -7255,9 +7255,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.13.0 + app.kubernetes.io/version: v1.13.1-rc0 annotations: - configChecksum: "7ee7db423358a7296a24baf63ddb60c73b3c38c663d92731caae21e75d89fbc" + configChecksum: "87f8dd83145c058839fbf440c688d131d5917282ae935b2fe02147df47ef3a7" prometheus.io/path: "/metrics" prometheus.io/port: "10254" spec: @@ -7271,7 +7271,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -7298,7 +7298,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.13.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 4b0dc5c172..9e10ae09fb 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -469,7 +469,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.13.0" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0" k8s-array: logs: config: @@ -817,7 +817,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: RVJtRHVFeUtzZHdDMjRTQQ== + haSharedSecret: cEJlTDl0bXN6NVE4ZEdFag== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1248,7 +1248,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 35a84a5044b9c9ffb7963f8ea11733277e84450e2124593216c6721ef7a4d539 + checksum/configuration: 9207564b9b5f0358f7b8507232200ac759f58ae16af8561f72a4488274629eaf checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1414,7 +1414,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: b56109123fc8874c06e091766f6e86bd5ffdc4cdc86ea7861092332870babf0e + checksum/secret: ad16b8a2ae1014673d354d27f8f9e4307e588f439534e5894ecb7b0e4c8c7692 labels: app: docker-registry release: flyte-sandbox @@ -1757,7 +1757,7 @@ spec: value: minio - name: FLYTE_AWS_SECRET_ACCESS_KEY value: miniostorage - image: cr.flyte.org/flyteorg/flyteagent:1.13.0 + image: cr.flyte.org/flyteorg/flyteagent:1.13.3 imagePullPolicy: IfNotPresent name: flyteagent ports: diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index db86678c70..ea327339fb 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -458,7 +458,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.13.0" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.13.1-rc0" k8s-array: logs: config: @@ -797,7 +797,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: dGZGeUR3THl1ODFKV2ROdA== + haSharedSecret: UFppYnRpOVNHMVdlZkp0TA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1195,7 +1195,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 401b476b6dd95fe6fcfb0d91b6cb42550e272bc66f07de218d192ad374c1baf0 + checksum/configuration: 6bc1ee22a1eb899398b82b56862cfb1aa09ed96f467d4eae11f2738c284115c2 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1361,7 +1361,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 953e371a21c81445990201881e1603b35558441bea4c3702cc1f3dbdbc66c6b7 + checksum/secret: bfe262b4ef6c387db539e0d2b93d9557907a4a4b5aef3cec954b1ce593d364d9 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index bdd5143bb3..1f55acef66 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: clA0NmJsaW9TUHNJYjRYTA== + haSharedSecret: MEV1QmRqTlVpVHljaU9FeQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 51051b8a95112e2aa1aed0e472540340697cf3481a144444d65dfaa220ae471d + checksum/secret: 9ac72b0cb595456c3e96447f44a0377762ab17d663e80e47079203fcbd518a34 labels: app: docker-registry release: flyte-sandbox diff --git a/docs/conf.py b/docs/conf.py index 6ef6e93a1c..35b21a09ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,7 +36,7 @@ # The short X.Y version version = "" # The full version, including alpha/beta/rc tags -release = "1.13.0" +release = "1.13.1-rc0" # -- General configuration --------------------------------------------------- diff --git a/docs/deployment/configuration/generated/flyteadmin_config.rst b/docs/deployment/configuration/generated/flyteadmin_config.rst index 106c5b57ef..162cbc4d1d 100644 --- a/docs/deployment/configuration/generated/flyteadmin_config.rst +++ b/docs/deployment/configuration/generated/flyteadmin_config.rst @@ -2181,6 +2181,18 @@ A URL pointing to the flyteconsole instance used to hit this flyteadmin instance "" +useOffloadedInputs (bool) +------------------------------------------------------------------------------------------------------------------------ + +Use offloaded inputs for workflows. + +**Default Value**: + +.. code-block:: yaml + + "false" + + interfaces.FeatureGates ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/deployment/configuration/generated/scheduler_config.rst b/docs/deployment/configuration/generated/scheduler_config.rst index 06def34abc..923e3db898 100644 --- a/docs/deployment/configuration/generated/scheduler_config.rst +++ b/docs/deployment/configuration/generated/scheduler_config.rst @@ -2181,6 +2181,18 @@ A URL pointing to the flyteconsole instance used to hit this flyteadmin instance "" +useOffloadedInputs (bool) +------------------------------------------------------------------------------------------------------------------------ + +Use offloaded inputs for workflows. + +**Default Value**: + +.. code-block:: yaml + + "false" + + interfaces.FeatureGates ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 3d387295a950791d5cca2f36e2e0d9a45e81c1c6 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Wed, 14 Aug 2024 20:27:30 +0200 Subject: [PATCH 22/23] Add offloaded path to literal (#5660) Signed-off-by: Katrina Rogan --- flyteidl/clients/go/assets/admin.swagger.json | 9 + .../gen/pb-es/flyteidl/core/literals_pb.ts | 18 +- .../gen/pb-go/flyteidl/core/literals.pb.go | 185 ++++++++++-------- .../cacheservice/cacheservice.swagger.json | 9 + .../datacatalog/datacatalog.swagger.json | 9 + .../flyteidl/service/admin.swagger.json | 9 + .../flyteidl/service/agent.swagger.json | 9 + .../flyteidl/service/dataproxy.swagger.json | 9 + .../external_plugin_service.swagger.json | 9 + .../flyteidl/service/signal.swagger.json | 9 + flyteidl/gen/pb-js/flyteidl.d.ts | 12 ++ flyteidl/gen/pb-js/flyteidl.js | 34 ++++ .../pb_python/flyteidl/core/literals_pb2.py | 52 ++--- .../pb_python/flyteidl/core/literals_pb2.pyi | 8 +- flyteidl/gen/pb_rust/flyteidl.core.rs | 6 + flyteidl/protos/flyteidl/core/literals.proto | 6 + 16 files changed, 282 insertions(+), 111 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 7e6e7d3367..6ebfd70f8d 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -7428,6 +7428,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts index 6cf6b07ca2..4dd6f2467e 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts @@ -4,7 +4,7 @@ // @ts-nocheck import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; -import { Duration, Message, proto3, Struct, Timestamp } from "@bufbuild/protobuf"; +import { Duration, Message, proto3, protoInt64, Struct, Timestamp } from "@bufbuild/protobuf"; import { BlobType, Error, LiteralType, OutputReference, SchemaType, StructuredDatasetType } from "./types_pb.js"; /** @@ -584,6 +584,20 @@ export class Literal extends Message { */ metadata: { [key: string]: string } = {}; + /** + * If this literal is offloaded, this field will contain metadata including the offload location. + * + * @generated from field: string uri = 6; + */ + uri = ""; + + /** + * Includes information about the size of the literal. + * + * @generated from field: uint64 size_bytes = 7; + */ + sizeBytes = protoInt64.zero; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -597,6 +611,8 @@ export class Literal extends Message { { no: 3, name: "map", kind: "message", T: LiteralMap, oneof: "value" }, { no: 4, name: "hash", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 5, name: "metadata", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "scalar", T: 9 /* ScalarType.STRING */} }, + { no: 6, name: "uri", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 7, name: "size_bytes", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): Literal { diff --git a/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go b/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go index 40b24e30a5..897ffc0d72 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go @@ -785,6 +785,10 @@ type Literal struct { Hash string `protobuf:"bytes,4,opt,name=hash,proto3" json:"hash,omitempty"` // Additional metadata for literals. Metadata map[string]string `protobuf:"bytes,5,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // If this literal is offloaded, this field will contain metadata including the offload location. + Uri string `protobuf:"bytes,6,opt,name=uri,proto3" json:"uri,omitempty"` + // Includes information about the size of the literal. + SizeBytes uint64 `protobuf:"varint,7,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` } func (x *Literal) Reset() { @@ -861,6 +865,20 @@ func (x *Literal) GetMetadata() map[string]string { return nil } +func (x *Literal) GetUri() string { + if x != nil { + return x.Uri + } + return "" +} + +func (x *Literal) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + type isLiteral_Value interface { isLiteral_Value() } @@ -1513,7 +1531,7 @@ var file_flyteidl_core_literals_proto_rawDesc = []byte{ 0x74, 0x12, 0x2c, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x42, - 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xc9, 0x02, 0x0a, 0x07, 0x4c, 0x69, 0x74, + 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xfa, 0x02, 0x0a, 0x07, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x48, 0x00, 0x52, 0x06, 0x73, @@ -1529,90 +1547,93 @@ var file_flyteidl_core_literals_proto_rawDesc = []byte{ 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x3b, - 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x22, 0x47, 0x0a, 0x11, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x08, 0x6c, 0x69, 0x74, - 0x65, 0x72, 0x61, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x6c, - 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, - 0x72, 0x61, 0x6c, 0x52, 0x08, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x22, 0xa6, 0x01, - 0x0a, 0x0a, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x12, 0x43, 0x0a, 0x08, - 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x10, + 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, + 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x73, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x1a, + 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x07, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x47, 0x0a, 0x11, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, + 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x08, 0x6c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, + 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x52, 0x08, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x22, 0xa6, + 0x01, 0x0a, 0x0a, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x12, 0x43, 0x0a, + 0x08, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x27, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, + 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x6c, 0x73, 0x1a, 0x53, 0x0a, 0x0d, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4f, 0x0a, 0x15, 0x42, 0x69, 0x6e, 0x64, 0x69, + 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x36, 0x0a, 0x08, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x08, + 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x0e, 0x42, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x4d, 0x61, 0x70, 0x12, 0x47, 0x0a, 0x08, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, + 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x4d, 0x61, 0x70, 0x2e, 0x42, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x62, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x57, 0x0a, 0x0d, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, + 0x74, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x47, 0x0a, + 0x09, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3a, 0x0a, 0x0a, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, - 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, - 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, - 0x73, 0x1a, 0x53, 0x0a, 0x0d, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4f, 0x0a, 0x15, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, - 0x67, 0x44, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x36, 0x0a, 0x08, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x08, 0x62, - 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x0e, 0x42, 0x69, 0x6e, 0x64, - 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x4d, 0x61, 0x70, 0x12, 0x47, 0x0a, 0x08, 0x62, 0x69, - 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x66, - 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, - 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x4d, 0x61, 0x70, 0x2e, 0x42, 0x69, 0x6e, 0x64, - 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x62, 0x69, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x73, 0x1a, 0x57, 0x0a, 0x0d, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xae, 0x02, 0x0a, 0x0b, 0x42, 0x69, 0x6e, 0x64, 0x69, + 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x48, 0x00, 0x52, + 0x06, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x12, 0x46, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x6c, + 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x3a, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1e, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x48, 0x00, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x03, 0x6d, + 0x61, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x44, 0x61, 0x74, 0x61, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x12, 0x2e, + 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x6e, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x42, 0x07, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x51, 0x0a, 0x07, 0x42, 0x69, 0x6e, 0x64, 0x69, + 0x6e, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x76, 0x61, 0x72, 0x12, 0x34, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, - 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x47, 0x0a, 0x09, - 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3a, 0x0a, 0x0a, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, - 0x74, 0x65, 0x72, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xae, 0x02, 0x0a, 0x0b, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, - 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x48, 0x00, 0x52, 0x06, - 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x12, 0x46, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x6c, 0x79, - 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, - 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1e, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x48, - 0x00, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x03, 0x6d, 0x61, - 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, - 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, - 0x61, 0x74, 0x61, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x12, 0x2e, 0x0a, - 0x05, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, - 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x6e, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x42, 0x07, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x51, 0x0a, 0x07, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, - 0x67, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x76, 0x61, 0x72, 0x12, 0x34, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, - 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x65, 0x79, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x22, 0x29, 0x0a, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x79, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, - 0x67, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x42, 0xb3, 0x01, 0x0a, - 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x42, 0x0d, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, - 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, - 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, - 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, - 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x65, + 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x22, 0x29, 0x0a, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x79, 0x53, 0x74, 0x72, 0x61, 0x74, + 0x65, 0x67, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x42, 0xb3, 0x01, + 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x42, 0x0d, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, + 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, + 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, + 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json index 79be0ba117..c000257abc 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json @@ -227,6 +227,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json index 1f2f91d28b..3173437543 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json @@ -171,6 +171,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 7e6e7d3367..6ebfd70f8d 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -7428,6 +7428,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index a0e8cfed39..ef57647773 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1265,6 +1265,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json index 4798fc6c2d..ea24c34e3e 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json @@ -287,6 +287,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index f488a49c00..097504b251 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -581,6 +581,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json index 94c859bfcd..6d0bd1f15a 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json @@ -363,6 +363,15 @@ "type": "string" }, "description": "Additional metadata for literals." + }, + "uri": { + "type": "string", + "description": "If this literal is offloaded, this field will contain metadata including the offload location." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index ada71f1f09..b0f382ee88 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -3390,6 +3390,12 @@ export namespace flyteidl { /** Literal metadata */ metadata?: ({ [k: string]: string }|null); + + /** Literal uri */ + uri?: (string|null); + + /** Literal sizeBytes */ + sizeBytes?: (Long|null); } /** Represents a Literal. */ @@ -3416,6 +3422,12 @@ export namespace flyteidl { /** Literal metadata. */ public metadata: { [k: string]: string }; + /** Literal uri. */ + public uri: string; + + /** Literal sizeBytes. */ + public sizeBytes: Long; + /** Literal value. */ public value?: ("scalar"|"collection"|"map"); diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index 8f446b4aba..6b78cbc030 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -8105,6 +8105,8 @@ * @property {flyteidl.core.ILiteralMap|null} [map] Literal map * @property {string|null} [hash] Literal hash * @property {Object.|null} [metadata] Literal metadata + * @property {string|null} [uri] Literal uri + * @property {Long|null} [sizeBytes] Literal sizeBytes */ /** @@ -8163,6 +8165,22 @@ */ Literal.prototype.metadata = $util.emptyObject; + /** + * Literal uri. + * @member {string} uri + * @memberof flyteidl.core.Literal + * @instance + */ + Literal.prototype.uri = ""; + + /** + * Literal sizeBytes. + * @member {Long} sizeBytes + * @memberof flyteidl.core.Literal + * @instance + */ + Literal.prototype.sizeBytes = $util.Long ? $util.Long.fromBits(0,0,true) : 0; + // OneOf field names bound to virtual getters and setters var $oneOfFields; @@ -8212,6 +8230,10 @@ if (message.metadata != null && message.hasOwnProperty("metadata")) for (var keys = Object.keys(message.metadata), i = 0; i < keys.length; ++i) writer.uint32(/* id 5, wireType 2 =*/42).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.metadata[keys[i]]).ldelim(); + if (message.uri != null && message.hasOwnProperty("uri")) + writer.uint32(/* id 6, wireType 2 =*/50).string(message.uri); + if (message.sizeBytes != null && message.hasOwnProperty("sizeBytes")) + writer.uint32(/* id 7, wireType 0 =*/56).uint64(message.sizeBytes); return writer; }; @@ -8253,6 +8275,12 @@ reader.pos++; message.metadata[key] = reader.string(); break; + case 6: + message.uri = reader.string(); + break; + case 7: + message.sizeBytes = reader.uint64(); + break; default: reader.skipType(tag & 7); break; @@ -8312,6 +8340,12 @@ if (!$util.isString(message.metadata[key[i]])) return "metadata: string{k:string} expected"; } + if (message.uri != null && message.hasOwnProperty("uri")) + if (!$util.isString(message.uri)) + return "uri: string expected"; + if (message.sizeBytes != null && message.hasOwnProperty("sizeBytes")) + if (!$util.isInteger(message.sizeBytes) && !(message.sizeBytes && $util.isInteger(message.sizeBytes.low) && $util.isInteger(message.sizeBytes.high))) + return "sizeBytes: integer|Long expected"; return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.py b/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.py index 77bc3ea3f0..dc2f436e8e 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.py @@ -17,7 +17,7 @@ from flyteidl.core import types_pb2 as flyteidl_dot_core_dot_types__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/core/literals.proto\x12\rflyteidl.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x19\x66lyteidl/core/types.proto\"\x87\x02\n\tPrimitive\x12\x1a\n\x07integer\x18\x01 \x01(\x03H\x00R\x07integer\x12!\n\x0b\x66loat_value\x18\x02 \x01(\x01H\x00R\nfloatValue\x12#\n\x0cstring_value\x18\x03 \x01(\tH\x00R\x0bstringValue\x12\x1a\n\x07\x62oolean\x18\x04 \x01(\x08H\x00R\x07\x62oolean\x12\x38\n\x08\x64\x61tetime\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x08\x64\x61tetime\x12\x37\n\x08\x64uration\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00R\x08\x64urationB\x07\n\x05value\"\x06\n\x04Void\"Q\n\x04\x42lob\x12\x37\n\x08metadata\x18\x01 \x01(\x0b\x32\x1b.flyteidl.core.BlobMetadataR\x08metadata\x12\x10\n\x03uri\x18\x03 \x01(\tR\x03uri\";\n\x0c\x42lobMetadata\x12+\n\x04type\x18\x01 \x01(\x0b\x32\x17.flyteidl.core.BlobTypeR\x04type\"0\n\x06\x42inary\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value\x12\x10\n\x03tag\x18\x02 \x01(\tR\x03tag\"I\n\x06Schema\x12\x10\n\x03uri\x18\x01 \x01(\tR\x03uri\x12-\n\x04type\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.SchemaTypeR\x04type\"e\n\x05Union\x12,\n\x05value\x18\x01 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value\x12.\n\x04type\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.LiteralTypeR\x04type\"y\n\x19StructuredDatasetMetadata\x12\\\n\x17structured_dataset_type\x18\x01 \x01(\x0b\x32$.flyteidl.core.StructuredDatasetTypeR\x15structuredDatasetType\"k\n\x11StructuredDataset\x12\x10\n\x03uri\x18\x01 \x01(\tR\x03uri\x12\x44\n\x08metadata\x18\x02 \x01(\x0b\x32(.flyteidl.core.StructuredDatasetMetadataR\x08metadata\"\xf0\x03\n\x06Scalar\x12\x38\n\tprimitive\x18\x01 \x01(\x0b\x32\x18.flyteidl.core.PrimitiveH\x00R\tprimitive\x12)\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x13.flyteidl.core.BlobH\x00R\x04\x62lob\x12/\n\x06\x62inary\x18\x03 \x01(\x0b\x32\x15.flyteidl.core.BinaryH\x00R\x06\x62inary\x12/\n\x06schema\x18\x04 \x01(\x0b\x32\x15.flyteidl.core.SchemaH\x00R\x06schema\x12\x32\n\tnone_type\x18\x05 \x01(\x0b\x32\x13.flyteidl.core.VoidH\x00R\x08noneType\x12,\n\x05\x65rror\x18\x06 \x01(\x0b\x32\x14.flyteidl.core.ErrorH\x00R\x05\x65rror\x12\x33\n\x07generic\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\x07generic\x12Q\n\x12structured_dataset\x18\x08 \x01(\x0b\x32 .flyteidl.core.StructuredDatasetH\x00R\x11structuredDataset\x12,\n\x05union\x18\t \x01(\x0b\x32\x14.flyteidl.core.UnionH\x00R\x05unionB\x07\n\x05value\"\xc9\x02\n\x07Literal\x12/\n\x06scalar\x18\x01 \x01(\x0b\x32\x15.flyteidl.core.ScalarH\x00R\x06scalar\x12\x42\n\ncollection\x18\x02 \x01(\x0b\x32 .flyteidl.core.LiteralCollectionH\x00R\ncollection\x12-\n\x03map\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapH\x00R\x03map\x12\x12\n\x04hash\x18\x04 \x01(\tR\x04hash\x12@\n\x08metadata\x18\x05 \x03(\x0b\x32$.flyteidl.core.Literal.MetadataEntryR\x08metadata\x1a;\n\rMetadataEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x07\n\x05value\"G\n\x11LiteralCollection\x12\x32\n\x08literals\x18\x01 \x03(\x0b\x32\x16.flyteidl.core.LiteralR\x08literals\"\xa6\x01\n\nLiteralMap\x12\x43\n\x08literals\x18\x01 \x03(\x0b\x32\'.flyteidl.core.LiteralMap.LiteralsEntryR\x08literals\x1aS\n\rLiteralsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value:\x02\x38\x01\"O\n\x15\x42indingDataCollection\x12\x36\n\x08\x62indings\x18\x01 \x03(\x0b\x32\x1a.flyteidl.core.BindingDataR\x08\x62indings\"\xb2\x01\n\x0e\x42indingDataMap\x12G\n\x08\x62indings\x18\x01 \x03(\x0b\x32+.flyteidl.core.BindingDataMap.BindingsEntryR\x08\x62indings\x1aW\n\rBindingsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x30\n\x05value\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.BindingDataR\x05value:\x02\x38\x01\"G\n\tUnionInfo\x12:\n\ntargetType\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.LiteralTypeR\ntargetType\"\xae\x02\n\x0b\x42indingData\x12/\n\x06scalar\x18\x01 \x01(\x0b\x32\x15.flyteidl.core.ScalarH\x00R\x06scalar\x12\x46\n\ncollection\x18\x02 \x01(\x0b\x32$.flyteidl.core.BindingDataCollectionH\x00R\ncollection\x12:\n\x07promise\x18\x03 \x01(\x0b\x32\x1e.flyteidl.core.OutputReferenceH\x00R\x07promise\x12\x31\n\x03map\x18\x04 \x01(\x0b\x32\x1d.flyteidl.core.BindingDataMapH\x00R\x03map\x12.\n\x05union\x18\x05 \x01(\x0b\x32\x18.flyteidl.core.UnionInfoR\x05unionB\x07\n\x05value\"Q\n\x07\x42inding\x12\x10\n\x03var\x18\x01 \x01(\tR\x03var\x12\x34\n\x07\x62inding\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.BindingDataR\x07\x62inding\"6\n\x0cKeyValuePair\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\")\n\rRetryStrategy\x12\x18\n\x07retries\x18\x05 \x01(\rR\x07retriesB\xb3\x01\n\x11\x63om.flyteidl.coreB\rLiteralsProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/core/literals.proto\x12\rflyteidl.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x19\x66lyteidl/core/types.proto\"\x87\x02\n\tPrimitive\x12\x1a\n\x07integer\x18\x01 \x01(\x03H\x00R\x07integer\x12!\n\x0b\x66loat_value\x18\x02 \x01(\x01H\x00R\nfloatValue\x12#\n\x0cstring_value\x18\x03 \x01(\tH\x00R\x0bstringValue\x12\x1a\n\x07\x62oolean\x18\x04 \x01(\x08H\x00R\x07\x62oolean\x12\x38\n\x08\x64\x61tetime\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x08\x64\x61tetime\x12\x37\n\x08\x64uration\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00R\x08\x64urationB\x07\n\x05value\"\x06\n\x04Void\"Q\n\x04\x42lob\x12\x37\n\x08metadata\x18\x01 \x01(\x0b\x32\x1b.flyteidl.core.BlobMetadataR\x08metadata\x12\x10\n\x03uri\x18\x03 \x01(\tR\x03uri\";\n\x0c\x42lobMetadata\x12+\n\x04type\x18\x01 \x01(\x0b\x32\x17.flyteidl.core.BlobTypeR\x04type\"0\n\x06\x42inary\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value\x12\x10\n\x03tag\x18\x02 \x01(\tR\x03tag\"I\n\x06Schema\x12\x10\n\x03uri\x18\x01 \x01(\tR\x03uri\x12-\n\x04type\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.SchemaTypeR\x04type\"e\n\x05Union\x12,\n\x05value\x18\x01 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value\x12.\n\x04type\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.LiteralTypeR\x04type\"y\n\x19StructuredDatasetMetadata\x12\\\n\x17structured_dataset_type\x18\x01 \x01(\x0b\x32$.flyteidl.core.StructuredDatasetTypeR\x15structuredDatasetType\"k\n\x11StructuredDataset\x12\x10\n\x03uri\x18\x01 \x01(\tR\x03uri\x12\x44\n\x08metadata\x18\x02 \x01(\x0b\x32(.flyteidl.core.StructuredDatasetMetadataR\x08metadata\"\xf0\x03\n\x06Scalar\x12\x38\n\tprimitive\x18\x01 \x01(\x0b\x32\x18.flyteidl.core.PrimitiveH\x00R\tprimitive\x12)\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x13.flyteidl.core.BlobH\x00R\x04\x62lob\x12/\n\x06\x62inary\x18\x03 \x01(\x0b\x32\x15.flyteidl.core.BinaryH\x00R\x06\x62inary\x12/\n\x06schema\x18\x04 \x01(\x0b\x32\x15.flyteidl.core.SchemaH\x00R\x06schema\x12\x32\n\tnone_type\x18\x05 \x01(\x0b\x32\x13.flyteidl.core.VoidH\x00R\x08noneType\x12,\n\x05\x65rror\x18\x06 \x01(\x0b\x32\x14.flyteidl.core.ErrorH\x00R\x05\x65rror\x12\x33\n\x07generic\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\x07generic\x12Q\n\x12structured_dataset\x18\x08 \x01(\x0b\x32 .flyteidl.core.StructuredDatasetH\x00R\x11structuredDataset\x12,\n\x05union\x18\t \x01(\x0b\x32\x14.flyteidl.core.UnionH\x00R\x05unionB\x07\n\x05value\"\xfa\x02\n\x07Literal\x12/\n\x06scalar\x18\x01 \x01(\x0b\x32\x15.flyteidl.core.ScalarH\x00R\x06scalar\x12\x42\n\ncollection\x18\x02 \x01(\x0b\x32 .flyteidl.core.LiteralCollectionH\x00R\ncollection\x12-\n\x03map\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapH\x00R\x03map\x12\x12\n\x04hash\x18\x04 \x01(\tR\x04hash\x12@\n\x08metadata\x18\x05 \x03(\x0b\x32$.flyteidl.core.Literal.MetadataEntryR\x08metadata\x12\x10\n\x03uri\x18\x06 \x01(\tR\x03uri\x12\x1d\n\nsize_bytes\x18\x07 \x01(\x04R\tsizeBytes\x1a;\n\rMetadataEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x07\n\x05value\"G\n\x11LiteralCollection\x12\x32\n\x08literals\x18\x01 \x03(\x0b\x32\x16.flyteidl.core.LiteralR\x08literals\"\xa6\x01\n\nLiteralMap\x12\x43\n\x08literals\x18\x01 \x03(\x0b\x32\'.flyteidl.core.LiteralMap.LiteralsEntryR\x08literals\x1aS\n\rLiteralsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value:\x02\x38\x01\"O\n\x15\x42indingDataCollection\x12\x36\n\x08\x62indings\x18\x01 \x03(\x0b\x32\x1a.flyteidl.core.BindingDataR\x08\x62indings\"\xb2\x01\n\x0e\x42indingDataMap\x12G\n\x08\x62indings\x18\x01 \x03(\x0b\x32+.flyteidl.core.BindingDataMap.BindingsEntryR\x08\x62indings\x1aW\n\rBindingsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x30\n\x05value\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.BindingDataR\x05value:\x02\x38\x01\"G\n\tUnionInfo\x12:\n\ntargetType\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.LiteralTypeR\ntargetType\"\xae\x02\n\x0b\x42indingData\x12/\n\x06scalar\x18\x01 \x01(\x0b\x32\x15.flyteidl.core.ScalarH\x00R\x06scalar\x12\x46\n\ncollection\x18\x02 \x01(\x0b\x32$.flyteidl.core.BindingDataCollectionH\x00R\ncollection\x12:\n\x07promise\x18\x03 \x01(\x0b\x32\x1e.flyteidl.core.OutputReferenceH\x00R\x07promise\x12\x31\n\x03map\x18\x04 \x01(\x0b\x32\x1d.flyteidl.core.BindingDataMapH\x00R\x03map\x12.\n\x05union\x18\x05 \x01(\x0b\x32\x18.flyteidl.core.UnionInfoR\x05unionB\x07\n\x05value\"Q\n\x07\x42inding\x12\x10\n\x03var\x18\x01 \x01(\tR\x03var\x12\x34\n\x07\x62inding\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.BindingDataR\x07\x62inding\"6\n\x0cKeyValuePair\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\")\n\rRetryStrategy\x12\x18\n\x07retries\x18\x05 \x01(\rR\x07retriesB\xb3\x01\n\x11\x63om.flyteidl.coreB\rLiteralsProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -53,29 +53,29 @@ _globals['_SCALAR']._serialized_start=1048 _globals['_SCALAR']._serialized_end=1544 _globals['_LITERAL']._serialized_start=1547 - _globals['_LITERAL']._serialized_end=1876 - _globals['_LITERAL_METADATAENTRY']._serialized_start=1808 - _globals['_LITERAL_METADATAENTRY']._serialized_end=1867 - _globals['_LITERALCOLLECTION']._serialized_start=1878 - _globals['_LITERALCOLLECTION']._serialized_end=1949 - _globals['_LITERALMAP']._serialized_start=1952 - _globals['_LITERALMAP']._serialized_end=2118 - _globals['_LITERALMAP_LITERALSENTRY']._serialized_start=2035 - _globals['_LITERALMAP_LITERALSENTRY']._serialized_end=2118 - _globals['_BINDINGDATACOLLECTION']._serialized_start=2120 - _globals['_BINDINGDATACOLLECTION']._serialized_end=2199 - _globals['_BINDINGDATAMAP']._serialized_start=2202 - _globals['_BINDINGDATAMAP']._serialized_end=2380 - _globals['_BINDINGDATAMAP_BINDINGSENTRY']._serialized_start=2293 - _globals['_BINDINGDATAMAP_BINDINGSENTRY']._serialized_end=2380 - _globals['_UNIONINFO']._serialized_start=2382 - _globals['_UNIONINFO']._serialized_end=2453 - _globals['_BINDINGDATA']._serialized_start=2456 - _globals['_BINDINGDATA']._serialized_end=2758 - _globals['_BINDING']._serialized_start=2760 - _globals['_BINDING']._serialized_end=2841 - _globals['_KEYVALUEPAIR']._serialized_start=2843 - _globals['_KEYVALUEPAIR']._serialized_end=2897 - _globals['_RETRYSTRATEGY']._serialized_start=2899 - _globals['_RETRYSTRATEGY']._serialized_end=2940 + _globals['_LITERAL']._serialized_end=1925 + _globals['_LITERAL_METADATAENTRY']._serialized_start=1857 + _globals['_LITERAL_METADATAENTRY']._serialized_end=1916 + _globals['_LITERALCOLLECTION']._serialized_start=1927 + _globals['_LITERALCOLLECTION']._serialized_end=1998 + _globals['_LITERALMAP']._serialized_start=2001 + _globals['_LITERALMAP']._serialized_end=2167 + _globals['_LITERALMAP_LITERALSENTRY']._serialized_start=2084 + _globals['_LITERALMAP_LITERALSENTRY']._serialized_end=2167 + _globals['_BINDINGDATACOLLECTION']._serialized_start=2169 + _globals['_BINDINGDATACOLLECTION']._serialized_end=2248 + _globals['_BINDINGDATAMAP']._serialized_start=2251 + _globals['_BINDINGDATAMAP']._serialized_end=2429 + _globals['_BINDINGDATAMAP_BINDINGSENTRY']._serialized_start=2342 + _globals['_BINDINGDATAMAP_BINDINGSENTRY']._serialized_end=2429 + _globals['_UNIONINFO']._serialized_start=2431 + _globals['_UNIONINFO']._serialized_end=2502 + _globals['_BINDINGDATA']._serialized_start=2505 + _globals['_BINDINGDATA']._serialized_end=2807 + _globals['_BINDING']._serialized_start=2809 + _globals['_BINDING']._serialized_end=2890 + _globals['_KEYVALUEPAIR']._serialized_start=2892 + _globals['_KEYVALUEPAIR']._serialized_end=2946 + _globals['_RETRYSTRATEGY']._serialized_start=2948 + _globals['_RETRYSTRATEGY']._serialized_end=2989 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.pyi index 62622203bd..0dc2af20da 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/core/literals_pb2.pyi @@ -104,7 +104,7 @@ class Scalar(_message.Message): def __init__(self, primitive: _Optional[_Union[Primitive, _Mapping]] = ..., blob: _Optional[_Union[Blob, _Mapping]] = ..., binary: _Optional[_Union[Binary, _Mapping]] = ..., schema: _Optional[_Union[Schema, _Mapping]] = ..., none_type: _Optional[_Union[Void, _Mapping]] = ..., error: _Optional[_Union[_types_pb2.Error, _Mapping]] = ..., generic: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., structured_dataset: _Optional[_Union[StructuredDataset, _Mapping]] = ..., union: _Optional[_Union[Union, _Mapping]] = ...) -> None: ... class Literal(_message.Message): - __slots__ = ["scalar", "collection", "map", "hash", "metadata"] + __slots__ = ["scalar", "collection", "map", "hash", "metadata", "uri", "size_bytes"] class MetadataEntry(_message.Message): __slots__ = ["key", "value"] KEY_FIELD_NUMBER: _ClassVar[int] @@ -117,12 +117,16 @@ class Literal(_message.Message): MAP_FIELD_NUMBER: _ClassVar[int] HASH_FIELD_NUMBER: _ClassVar[int] METADATA_FIELD_NUMBER: _ClassVar[int] + URI_FIELD_NUMBER: _ClassVar[int] + SIZE_BYTES_FIELD_NUMBER: _ClassVar[int] scalar: Scalar collection: LiteralCollection map: LiteralMap hash: str metadata: _containers.ScalarMap[str, str] - def __init__(self, scalar: _Optional[_Union[Scalar, _Mapping]] = ..., collection: _Optional[_Union[LiteralCollection, _Mapping]] = ..., map: _Optional[_Union[LiteralMap, _Mapping]] = ..., hash: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ...) -> None: ... + uri: str + size_bytes: int + def __init__(self, scalar: _Optional[_Union[Scalar, _Mapping]] = ..., collection: _Optional[_Union[LiteralCollection, _Mapping]] = ..., map: _Optional[_Union[LiteralMap, _Mapping]] = ..., hash: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., uri: _Optional[str] = ..., size_bytes: _Optional[int] = ...) -> None: ... class LiteralCollection(_message.Message): __slots__ = ["literals"] diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index bb3a04d75a..0876c70d6f 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -488,6 +488,12 @@ pub struct Literal { /// Additional metadata for literals. #[prost(map="string, string", tag="5")] pub metadata: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + /// If this literal is offloaded, this field will contain metadata including the offload location. + #[prost(string, tag="6")] + pub uri: ::prost::alloc::string::String, + /// Includes information about the size of the literal. + #[prost(uint64, tag="7")] + pub size_bytes: u64, #[prost(oneof="literal::Value", tags="1, 2, 3")] pub value: ::core::option::Option, } diff --git a/flyteidl/protos/flyteidl/core/literals.proto b/flyteidl/protos/flyteidl/core/literals.proto index f886873ffb..00b03f9456 100644 --- a/flyteidl/protos/flyteidl/core/literals.proto +++ b/flyteidl/protos/flyteidl/core/literals.proto @@ -111,6 +111,12 @@ message Literal { // Additional metadata for literals. map metadata = 5; + + // If this literal is offloaded, this field will contain metadata including the offload location. + string uri = 6; + + // Includes information about the size of the literal. + uint64 size_bytes = 7; } // A collection of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field. From bed761c33c40af23750467c828afea553c0b80a0 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 15 Aug 2024 04:03:44 +0800 Subject: [PATCH 23/23] Improve error messaging for invalid arguments (#5658) * Improve error messaging for invalid arguments Signed-off-by: Kevin Su * update tests Signed-off-by: Kevin Su * fix tests Signed-off-by: Kevin Su * Define a separate identifier for the launchplan test Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Kevin Su Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- flyteadmin/pkg/errors/errors.go | 11 +++++------ flyteadmin/pkg/errors/errors_test.go | 15 ++++++++++----- .../pkg/manager/impl/workflow_manager_test.go | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/flyteadmin/pkg/errors/errors.go b/flyteadmin/pkg/errors/errors.go index 78727a7305..0fd9542ba7 100644 --- a/flyteadmin/pkg/errors/errors.go +++ b/flyteadmin/pkg/errors/errors.go @@ -112,7 +112,7 @@ func NewIncompatibleClusterError(ctx context.Context, errorMsg, curCluster strin } func compareJsons(jsonArray1 jsondiff.Patch, jsonArray2 jsondiff.Patch) []string { - results := []string{} + var results []string map1 := make(map[string]jsondiff.Operation) for _, obj := range jsonArray1 { map1[obj.Path] = obj @@ -120,7 +120,7 @@ func compareJsons(jsonArray1 jsondiff.Patch, jsonArray2 jsondiff.Patch) []string for _, obj := range jsonArray2 { if val, ok := map1[obj.Path]; ok { - result := fmt.Sprintf("\t\t- %v: %v -> %v", obj.Path, obj.Value, val.Value) + result := fmt.Sprintf("%v:\n\t- %v \n\t+ %v", obj.Path, obj.Value, val.Value) results = append(results, result) } } @@ -128,7 +128,7 @@ func compareJsons(jsonArray1 jsondiff.Patch, jsonArray2 jsondiff.Patch) []string } func NewTaskExistsDifferentStructureError(ctx context.Context, request *admin.TaskCreateRequest, oldSpec *core.CompiledTask, newSpec *core.CompiledTask) FlyteAdminError { - errorMsg := fmt.Sprintf("%v task with different structure already exists:\n", request.Id.Name) + errorMsg := fmt.Sprintf("%v task with different structure already exists. (Please register a new version of the task):\n", request.Id.Name) diff, _ := jsondiff.Compare(oldSpec, newSpec) rdiff, _ := jsondiff.Compare(newSpec, oldSpec) rs := compareJsons(diff, rdiff) @@ -136,7 +136,6 @@ func NewTaskExistsDifferentStructureError(ctx context.Context, request *admin.Ta errorMsg += strings.Join(rs, "\n") return NewFlyteAdminErrorf(codes.InvalidArgument, errorMsg) - } func NewTaskExistsIdenticalStructureError(ctx context.Context, request *admin.TaskCreateRequest) FlyteAdminError { @@ -145,7 +144,7 @@ func NewTaskExistsIdenticalStructureError(ctx context.Context, request *admin.Ta } func NewWorkflowExistsDifferentStructureError(ctx context.Context, request *admin.WorkflowCreateRequest, oldSpec *core.CompiledWorkflowClosure, newSpec *core.CompiledWorkflowClosure) FlyteAdminError { - errorMsg := fmt.Sprintf("%v workflow with different structure already exists:\n", request.Id.Name) + errorMsg := fmt.Sprintf("%v workflow with different structure already exists. (Please register a new version of the workflow):\n", request.Id.Name) diff, _ := jsondiff.Compare(oldSpec, newSpec) rdiff, _ := jsondiff.Compare(newSpec, oldSpec) rs := compareJsons(diff, rdiff) @@ -183,7 +182,7 @@ func NewWorkflowExistsIdenticalStructureError(ctx context.Context, request *admi } func NewLaunchPlanExistsDifferentStructureError(ctx context.Context, request *admin.LaunchPlanCreateRequest, oldSpec *admin.LaunchPlanSpec, newSpec *admin.LaunchPlanSpec) FlyteAdminError { - errorMsg := "launch plan with different structure already exists:\n" + errorMsg := fmt.Sprintf("%v launch plan with different structure already exists. (Please register a new version of the launch plan):\n", request.Id.Name) diff, _ := jsondiff.Compare(oldSpec, newSpec) rdiff, _ := jsondiff.Compare(newSpec, oldSpec) rs := compareJsons(diff, rdiff) diff --git a/flyteadmin/pkg/errors/errors_test.go b/flyteadmin/pkg/errors/errors_test.go index daaa060340..18c76992b5 100644 --- a/flyteadmin/pkg/errors/errors_test.go +++ b/flyteadmin/pkg/errors/errors_test.go @@ -74,7 +74,7 @@ func TestJsonDifferHasDiffError(t *testing.T) { diff, _ := jsondiff.Compare(oldSpec, newSpec) rdiff, _ := jsondiff.Compare(newSpec, oldSpec) rs := compareJsons(diff, rdiff) - assert.Equal(t, "\t\t- /four: 4 -> 0", strings.Join(rs, "\n")) + assert.Equal(t, "/four:\n\t- 4 \n\t+ 0", strings.Join(rs, "\n")) } func TestJsonDifferNoDiffError(t *testing.T) { @@ -144,7 +144,7 @@ func TestNewTaskExistsDifferentStructureError(t *testing.T) { s, ok := status.FromError(statusErr) assert.True(t, ok) assert.Equal(t, codes.InvalidArgument, s.Code()) - assert.Equal(t, "t1 task with different structure already exists:\n\t\t- /template/Target/Container/resources/requests/0/value: 150m -> 250m", s.Message()) + assert.Equal(t, "t1 task with different structure already exists. (Please register a new version of the task):\n/template/Target/Container/resources/requests/0/value:\n\t- 150m \n\t+ 250m", s.Message()) } func TestNewTaskExistsIdenticalStructureError(t *testing.T) { @@ -160,7 +160,7 @@ func TestNewTaskExistsIdenticalStructureError(t *testing.T) { } func TestNewWorkflowExistsDifferentStructureError(t *testing.T) { - identifier = core.Identifier{ + identifier := core.Identifier{ ResourceType: core.ResourceType_WORKFLOW, Project: "testProj", Domain: "domain", @@ -225,7 +225,7 @@ func TestNewWorkflowExistsDifferentStructureError(t *testing.T) { s, ok := status.FromError(statusErr) assert.True(t, ok) assert.Equal(t, codes.InvalidArgument, s.Code()) - assert.Equal(t, "hello workflow with different structure already exists:\n\t\t- /primary/connections/upstream/bar: -> map[ids:[start-node]]\n\t\t- /primary/connections/upstream/end-node/ids/0: foo -> bar\n\t\t- /primary/connections/upstream/foo: map[ids:[start-node]] -> \n\t\t- /primary/template/nodes/0/id: foo -> bar", s.Message()) + assert.Equal(t, "hello workflow with different structure already exists. (Please register a new version of the workflow):\n/primary/connections/upstream/bar:\n\t- \n\t+ map[ids:[start-node]]\n/primary/connections/upstream/end-node/ids/0:\n\t- foo \n\t+ bar\n/primary/connections/upstream/foo:\n\t- map[ids:[start-node]] \n\t+ \n/primary/template/nodes/0/id:\n\t- foo \n\t+ bar", s.Message()) details, ok := s.Details()[0].(*admin.CreateWorkflowFailureReason) assert.True(t, ok) @@ -251,6 +251,11 @@ func TestNewWorkflowExistsIdenticalStructureError(t *testing.T) { } func TestNewLaunchPlanExistsDifferentStructureError(t *testing.T) { + identifier := core.Identifier{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Name: "lp_name", + } + req := &admin.LaunchPlanCreateRequest{ Id: &identifier, } @@ -284,7 +289,7 @@ func TestNewLaunchPlanExistsDifferentStructureError(t *testing.T) { s, ok := status.FromError(statusErr) assert.True(t, ok) assert.Equal(t, codes.InvalidArgument, s.Code()) - assert.Equal(t, "launch plan with different structure already exists:\n\t\t- /workflow_id/version: ver1 -> ver2", s.Message()) + assert.Equal(t, "lp_name launch plan with different structure already exists. (Please register a new version of the launch plan):\n/workflow_id/version:\n\t- ver1 \n\t+ ver2", s.Message()) } func TestNewLaunchPlanExistsIdenticalStructureError(t *testing.T) { diff --git a/flyteadmin/pkg/manager/impl/workflow_manager_test.go b/flyteadmin/pkg/manager/impl/workflow_manager_test.go index 60e0187e07..33626cae84 100644 --- a/flyteadmin/pkg/manager/impl/workflow_manager_test.go +++ b/flyteadmin/pkg/manager/impl/workflow_manager_test.go @@ -187,7 +187,7 @@ func TestCreateWorkflow_ExistingWorkflow(t *testing.T) { getMockWorkflowConfigProvider(), getMockWorkflowCompiler(), mockStorageClient, storagePrefix, mockScope.NewTestScope()) request := testutils.GetWorkflowRequest() response, err := workflowManager.CreateWorkflow(context.Background(), request) - assert.EqualError(t, err, "name workflow with different structure already exists:\n\t\t- /primary/template/id: -> map[domain:domain name:name project:project resource_type:2 version:version]\n\t\t- /primary/template/interface/inputs: -> map[variables:map[foo:map[type:map[Type:map[Simple:3]]]]]\n\t\t- /primary/template/interface/outputs: -> map[variables:map[bar:map[type:map[Type:map[Simple:3]]]]]\n\t\t- /primary/template/nodes: -> [map[Target: id:node 1] map[Target: id:node 2]]") + assert.EqualError(t, err, "name workflow with different structure already exists. (Please register a new version of the workflow):\n/primary/template/id:\n\t- \n\t+ map[domain:domain name:name project:project resource_type:2 version:version]\n/primary/template/interface/inputs:\n\t- \n\t+ map[variables:map[foo:map[type:map[Type:map[Simple:3]]]]]\n/primary/template/interface/outputs:\n\t- \n\t+ map[variables:map[bar:map[type:map[Type:map[Simple:3]]]]]\n/primary/template/nodes:\n\t- \n\t+ [map[Target: id:node 1] map[Target: id:node 2]]") assert.Nil(t, response) } @@ -205,7 +205,7 @@ func TestCreateWorkflow_ExistingWorkflow_Different(t *testing.T) { request := testutils.GetWorkflowRequest() response, err := workflowManager.CreateWorkflow(context.Background(), request) - assert.EqualError(t, err, "name workflow with different structure already exists:\n\t\t- /primary/template/id: -> map[domain:domain name:name project:project resource_type:2 version:version]\n\t\t- /primary/template/interface/inputs: -> map[variables:map[foo:map[type:map[Type:map[Simple:3]]]]]\n\t\t- /primary/template/interface/outputs: -> map[variables:map[bar:map[type:map[Type:map[Simple:3]]]]]\n\t\t- /primary/template/nodes: -> [map[Target: id:node 1] map[Target: id:node 2]]") + assert.EqualError(t, err, "name workflow with different structure already exists. (Please register a new version of the workflow):\n/primary/template/id:\n\t- \n\t+ map[domain:domain name:name project:project resource_type:2 version:version]\n/primary/template/interface/inputs:\n\t- \n\t+ map[variables:map[foo:map[type:map[Type:map[Simple:3]]]]]\n/primary/template/interface/outputs:\n\t- \n\t+ map[variables:map[bar:map[type:map[Type:map[Simple:3]]]]]\n/primary/template/nodes:\n\t- \n\t+ [map[Target: id:node 1] map[Target: id:node 2]]") flyteErr := err.(flyteErrors.FlyteAdminError) assert.Equal(t, codes.InvalidArgument, flyteErr.Code()) assert.Nil(t, response)