From 7076e894a174c9bafdf36f59964199bc3dfc13f3 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 19 Jan 2023 12:56:07 -0800 Subject: [PATCH 1/4] wip Signed-off-by: Kevin Su --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e9476fbf5..5a043d421 100644 --- a/go.mod +++ b/go.mod @@ -147,3 +147,5 @@ require ( ) replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d + +replace github.com/flyteorg/flyteplugins => github.com/flyteorg/flyteplugins v1.0.29-0.20230119205439-b0571dade5b2 diff --git a/go.sum b/go.sum index 348308946..b027f7ef1 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flyteorg/flyteidl v1.3.2 h1:s4DC8go2ou5LtZ+CFcS31r0mhv3baelNV81C1KZS26U= github.com/flyteorg/flyteidl v1.3.2/go.mod h1:OJAq333OpInPnMhvVz93AlEjmlQ+t0FAD4aakIYE4OU= -github.com/flyteorg/flyteplugins v1.0.28 h1:w9XeA3NKMSoOyKKVPdkGi5u64p77sIhRTICOCFDvHTc= -github.com/flyteorg/flyteplugins v1.0.28/go.mod h1:aef2mGG/Wb6UnOqz2tiNcp+NNJNvxZRz2xV7JVojq6k= +github.com/flyteorg/flyteplugins v1.0.29-0.20230119205439-b0571dade5b2 h1:B7aSI/3Gk++b8rnQvqcbuIA1UwYBQfW8L68fwP0+o3M= +github.com/flyteorg/flyteplugins v1.0.29-0.20230119205439-b0571dade5b2/go.mod h1:aef2mGG/Wb6UnOqz2tiNcp+NNJNvxZRz2xV7JVojq6k= github.com/flyteorg/flytestdlib v1.0.0/go.mod h1:QSVN5wIM1lM9d60eAEbX7NwweQXW96t5x4jbyftn89c= github.com/flyteorg/flytestdlib v1.0.11 h1:f7B8x2/zMuimEVi4Jx0zqzvNhdi7aq7+ZWoqHsbp4F4= github.com/flyteorg/flytestdlib v1.0.11/go.mod h1:nIBmBHtjTJvhZEn3e/EwVC/iMkR2tUX8hEiXjRBpH/s= From 2eca4bc315d5131cec4dad6c66c2e1d8ebe278ab Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 19 Jan 2023 22:18:48 -0800 Subject: [PATCH 2/4] wip Signed-off-by: Kevin Su --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5a043d421..719ddfce9 100644 --- a/go.mod +++ b/go.mod @@ -148,4 +148,4 @@ require ( replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d -replace github.com/flyteorg/flyteplugins => github.com/flyteorg/flyteplugins v1.0.29-0.20230119205439-b0571dade5b2 +replace github.com/flyteorg/flyteplugins => github.com/flyteorg/flyteplugins v1.0.29-0.20230120065115-b91bc370b248 diff --git a/go.sum b/go.sum index b027f7ef1..c12153799 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flyteorg/flyteidl v1.3.2 h1:s4DC8go2ou5LtZ+CFcS31r0mhv3baelNV81C1KZS26U= github.com/flyteorg/flyteidl v1.3.2/go.mod h1:OJAq333OpInPnMhvVz93AlEjmlQ+t0FAD4aakIYE4OU= -github.com/flyteorg/flyteplugins v1.0.29-0.20230119205439-b0571dade5b2 h1:B7aSI/3Gk++b8rnQvqcbuIA1UwYBQfW8L68fwP0+o3M= -github.com/flyteorg/flyteplugins v1.0.29-0.20230119205439-b0571dade5b2/go.mod h1:aef2mGG/Wb6UnOqz2tiNcp+NNJNvxZRz2xV7JVojq6k= +github.com/flyteorg/flyteplugins v1.0.29-0.20230120065115-b91bc370b248 h1:Kw5ccH9fcl10kREjN9Qldi60OzKGNIP11L6JZUCUXrk= +github.com/flyteorg/flyteplugins v1.0.29-0.20230120065115-b91bc370b248/go.mod h1:aef2mGG/Wb6UnOqz2tiNcp+NNJNvxZRz2xV7JVojq6k= github.com/flyteorg/flytestdlib v1.0.0/go.mod h1:QSVN5wIM1lM9d60eAEbX7NwweQXW96t5x4jbyftn89c= github.com/flyteorg/flytestdlib v1.0.11 h1:f7B8x2/zMuimEVi4Jx0zqzvNhdi7aq7+ZWoqHsbp4F4= github.com/flyteorg/flytestdlib v1.0.11/go.mod h1:nIBmBHtjTJvhZEn3e/EwVC/iMkR2tUX8hEiXjRBpH/s= From 34738774335162e05ce7b49ae27ffbd50fb7cd44 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 20 Jan 2023 00:47:04 -0800 Subject: [PATCH 3/4] Transition to failed state if the k8s resourc cannot be created Signed-off-by: Kevin Su --- .../nodes/task/k8s/plugin_manager.go | 6 ++--- .../nodes/task/k8s/plugin_manager_test.go | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/controller/nodes/task/k8s/plugin_manager.go b/pkg/controller/nodes/task/k8s/plugin_manager.go index e0d786858..08a14cb5a 100644 --- a/pkg/controller/nodes/task/k8s/plugin_manager.go +++ b/pkg/controller/nodes/task/k8s/plugin_manager.go @@ -188,19 +188,19 @@ func (e *PluginManager) LaunchResource(ctx context.Context, tCtx pluginsCore.Tas tmpl, err := tCtx.TaskReader().Read(ctx) if err != nil { - return pluginsCore.Transition{}, err + return pluginsCore.DoTransition(pluginsCore.PhaseInfoFailure("Failed to read task template", err.Error(), nil)), nil } k8sTaskCtxMetadata, err := newTaskExecutionMetadata(tCtx.TaskExecutionMetadata(), tmpl) if err != nil { - return pluginsCore.Transition{}, err + return pluginsCore.DoTransition(pluginsCore.PhaseInfoFailure("Failed to create task context", err.Error(), nil)), nil } k8sTaskCtx := newTaskExecutionContext(tCtx, k8sTaskCtxMetadata) o, err := e.plugin.BuildResource(ctx, k8sTaskCtx) if err != nil { - return pluginsCore.UnknownTransition, err + return pluginsCore.DoTransition(pluginsCore.PhaseInfoFailure("Failed to build k8s resource", err.Error(), nil)), nil } e.AddObjectMetadata(k8sTaskCtxMetadata, o, config.GetK8sPluginConfig()) diff --git a/pkg/controller/nodes/task/k8s/plugin_manager_test.go b/pkg/controller/nodes/task/k8s/plugin_manager_test.go index 160dc335f..511b06bde 100644 --- a/pkg/controller/nodes/task/k8s/plugin_manager_test.go +++ b/pkg/controller/nodes/task/k8s/plugin_manager_test.go @@ -306,6 +306,28 @@ func TestK8sTaskExecutor_Handle_LaunchResource(t *testing.T) { assert.NoError(t, fakeClient.Delete(ctx, createdPod)) }) + t.Run("failed to build k8s resource", func(t *testing.T) { + tCtx := getMockTaskContext(PluginPhaseNotStarted, PluginPhaseStarted) + // common setup code + mockResourceHandler := &pluginsk8sMock.Plugin{} + mockResourceHandler.OnGetProperties().Return(k8s.PluginProperties{}) + mockResourceHandler.OnBuildResourceMatch(mock.Anything, mock.Anything).Return(nil, errors.New("failed")) + fakeClient := fake.NewClientBuilder().WithRuntimeObjects().Build() + pluginManager, err := NewPluginManager(ctx, dummySetupContext(fakeClient), k8s.PluginEntry{ + ID: "x", + ResourceToWatch: &v1.Pod{}, + Plugin: mockResourceHandler, + }, NewResourceMonitorIndex()) + assert.NoError(t, err) + + transition, err := pluginManager.Handle(ctx, tCtx) + assert.NoError(t, err) + assert.NotNil(t, transition) + transitionInfo := transition.Info() + assert.NotNil(t, transitionInfo) + assert.Equal(t, pluginsCore.PhasePermanentFailure, transitionInfo.Phase()) + }) + t.Run("jobAlreadyExists", func(t *testing.T) { tctx := getMockTaskContext(PluginPhaseNotStarted, PluginPhaseStarted) // common setup code From 3b566aa9e2fb9dba84276cdd9d6956a8de7bd966 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 20 Jan 2023 00:53:44 -0800 Subject: [PATCH 4/4] nit Signed-off-by: Kevin Su --- go.mod | 2 -- go.sum | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 719ddfce9..e9476fbf5 100644 --- a/go.mod +++ b/go.mod @@ -147,5 +147,3 @@ require ( ) replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d - -replace github.com/flyteorg/flyteplugins => github.com/flyteorg/flyteplugins v1.0.29-0.20230120065115-b91bc370b248 diff --git a/go.sum b/go.sum index c12153799..348308946 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flyteorg/flyteidl v1.3.2 h1:s4DC8go2ou5LtZ+CFcS31r0mhv3baelNV81C1KZS26U= github.com/flyteorg/flyteidl v1.3.2/go.mod h1:OJAq333OpInPnMhvVz93AlEjmlQ+t0FAD4aakIYE4OU= -github.com/flyteorg/flyteplugins v1.0.29-0.20230120065115-b91bc370b248 h1:Kw5ccH9fcl10kREjN9Qldi60OzKGNIP11L6JZUCUXrk= -github.com/flyteorg/flyteplugins v1.0.29-0.20230120065115-b91bc370b248/go.mod h1:aef2mGG/Wb6UnOqz2tiNcp+NNJNvxZRz2xV7JVojq6k= +github.com/flyteorg/flyteplugins v1.0.28 h1:w9XeA3NKMSoOyKKVPdkGi5u64p77sIhRTICOCFDvHTc= +github.com/flyteorg/flyteplugins v1.0.28/go.mod h1:aef2mGG/Wb6UnOqz2tiNcp+NNJNvxZRz2xV7JVojq6k= github.com/flyteorg/flytestdlib v1.0.0/go.mod h1:QSVN5wIM1lM9d60eAEbX7NwweQXW96t5x4jbyftn89c= github.com/flyteorg/flytestdlib v1.0.11 h1:f7B8x2/zMuimEVi4Jx0zqzvNhdi7aq7+ZWoqHsbp4F4= github.com/flyteorg/flytestdlib v1.0.11/go.mod h1:nIBmBHtjTJvhZEn3e/EwVC/iMkR2tUX8hEiXjRBpH/s=