Merge branch 'master' into agent-metadata-service-error-message-improve

flyteorg · Jan 5, 2024 · 3bba4c1 · 3bba4c1
2 parents 7982102 + b96419c
commit 3bba4c1
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 13 deletions.
diff --git a/charts/flyte-sandbox/templates/buildkit/deployment.yaml b/charts/flyte-sandbox/templates/buildkit/deployment.yaml
@@ -13,6 +13,7 @@ spec:
     metadata:
       labels: {{- include "flyte-sandbox.buildkitSelectorLabels" . | nindent 8 }}
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       containers:
         - name: buildkit

diff --git a/docker/sandbox-bundled/Makefile b/docker/sandbox-bundled/Makefile
@@ -18,8 +18,9 @@ flyte:
 .PHONY: manifests
 manifests:
 	mkdir -p manifests
-	helm dependency update ../../charts/flyte-sandbox
 	helm dependency update ../../charts/flyteagent
+	helm dependency update ../../charts/flyte-binary
+	helm dependency update ../../charts/flyte-sandbox
 	kustomize build \
 		--enable-helm \
 		--load-restrictor=LoadRestrictionsNone \

diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -816,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: SlF2M0tXNXdGTUpHZzdvNg==
+  haSharedSecret: RkZLd2xsVEZSNXNHMExjeg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1388,6 +1388,7 @@ spec:
           periodSeconds: 30
         securityContext:
           privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
 ---
 apiVersion: apps/v1
@@ -1411,7 +1412,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: b76914839daf95717da7a8ef31769c3b97f6c3c46a09776e5d0f8cc3fe9b1e9a
+        checksum/secret: 54b8e8ed67f9cf2f4b8c83b917ba11b0ed9f81f453df70376c332e202522643e
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -796,7 +796,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: QU5jN3V6ZjA4Y0tFbzZ0dw==
+  haSharedSecret: SENrVEZGc09ob3RKdFJRSA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1336,6 +1336,7 @@ spec:
           periodSeconds: 30
         securityContext:
           privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
 ---
 apiVersion: apps/v1
@@ -1359,7 +1360,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 436cdf2d6630635a2652233c83f71c617facb2fda8edabb639ab7dfa0f15e46f
+        checksum/secret: 7c61351bdfcc86e008feb9f8d023e02d3aa7e570e1759790f8d7c3f6a4fd9c86
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: MGNlbmFPenJydE1hNlB0Sw==
+  haSharedSecret: RjVjd3lFdXVJRllyTzlTNA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -910,6 +910,7 @@ spec:
           periodSeconds: 30
         securityContext:
           privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
 ---
 apiVersion: apps/v1
@@ -933,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 05cc4dc3bf13796e3a35e79da36baf286c81f80b207628f1f64197c80a04130c
+        checksum/secret: 048f8e7e067f646dce51250b72fe31919755c8fa62f7f4f4ccc054e20a719d9e
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/flyteplugins/go/tasks/pluginmachinery/k8s/client.go b/flyteplugins/go/tasks/pluginmachinery/k8s/client.go
@@ -3,8 +3,6 @@
 package k8s
 
 import (
-	"net/http"
-
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -40,9 +38,14 @@ type Options struct {
 // NewKubeClient creates a new KubeClient that caches reads and falls back to
 // make API calls on failure. Write calls are not cached.
 func NewKubeClient(config *rest.Config, options Options) (core.KubeClient, error) {
+	httpClient, err := rest.HTTPClientFor(config)
+	if err != nil {
+		return nil, err
+	}
+
 	if options.MapperProvider == nil {
 		options.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) {
-			return apiutil.NewDynamicRESTMapper(config, http.DefaultClient)
+			return apiutil.NewDynamicRESTMapper(config, httpClient)
 		}
 	}
 
@@ -53,7 +56,7 @@ func NewKubeClient(config *rest.Config, options Options) (core.KubeClient, error
 
 	if options.CacheOptions == nil {
 		options.CacheOptions = &cache.Options{
-			HTTPClient: http.DefaultClient,
+			HTTPClient: httpClient,
 			Mapper:     mapper,
 		}
 	}
@@ -64,7 +67,10 @@ func NewKubeClient(config *rest.Config, options Options) (core.KubeClient, error
 	}
 
 	if options.ClientOptions == nil {
-		options.ClientOptions = &client.Options{Mapper: mapper}
+		options.ClientOptions = &client.Options{
+			HTTPClient: httpClient,
+			Mapper: mapper,
+		}
 	}
 
 	client, err := client.New(config, *options.ClientOptions)

diff --git a/flytepropeller/pkg/controller/nodes/array/handler.go b/flytepropeller/pkg/controller/nodes/array/handler.go
@@ -633,7 +633,7 @@ func (a *arrayNodeHandler) buildArrayNodeContext(ctx context.Context, nCtx inter
 	// currently just mocking based on node phase -> which works for all k8s plugins
 	// we can not pre-allocated a bit array because max size is 256B and with 5k fanout node state = 1.28MB
 	pluginStateBytes := a.pluginStateBytesStarted
-	if taskPhase == int(core.PhaseUndefined) || taskPhase == int(core.PhaseRetryableFailure) {
+	if taskPhase == int(core.PhaseUndefined) || taskPhase == int(core.PhaseRetryableFailure) || taskPhase == int(core.PhaseWaitingForResources) {
 		pluginStateBytes = a.pluginStateBytesNotStarted
 	}
 

diff --git a/flytepropeller/pkg/controller/nodes/array/handler_test.go b/flytepropeller/pkg/controller/nodes/array/handler_test.go
@@ -507,6 +507,24 @@ func TestHandleArrayNodePhaseExecuting(t *testing.T) {
 			expectedTransitionPhase:        handler.EPhaseRunning,
 			expectedExternalResourcePhases: []idlcore.TaskExecution_Phase{idlcore.TaskExecution_RUNNING},
 		},
+		{
+			name: "StartSubNodesNewAttempts",
+			subNodePhases: []v1alpha1.NodePhase{
+				v1alpha1.NodePhaseQueued,
+				v1alpha1.NodePhaseQueued,
+			},
+			subNodeTaskPhases: []core.Phase{
+				core.PhaseRetryableFailure,
+				core.PhaseWaitingForResources,
+			},
+			subNodeTransitions: []handler.Transition{
+				handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoRunning(&handler.ExecutionInfo{})),
+				handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoRunning(&handler.ExecutionInfo{})),
+			},
+			expectedArrayNodePhase:         v1alpha1.ArrayNodePhaseExecuting,
+			expectedTransitionPhase:        handler.EPhaseRunning,
+			expectedExternalResourcePhases: []idlcore.TaskExecution_Phase{idlcore.TaskExecution_RUNNING, idlcore.TaskExecution_RUNNING},
+		},
 		{
 			name: "AllSubNodesSuccedeed",
 			subNodePhases: []v1alpha1.NodePhase{